diff --git a/.codespell.skip b/.codespell.skip index 833490cbdc7..b873c7082f2 100644 --- a/.codespell.skip +++ b/.codespell.skip @@ -24,6 +24,7 @@ ./openbb_platform/extensions/economy/openbb_economy/economy_views.py ./openbb_platform/providers/fmp/openbb_fmp/models/calendar_ipo.py ./openbb_platform/providers/imf/openbb_imf/models/consumer_price_index.py +./openbb_platform/providers/government_us/openbb_government_us/utils/psd_codes.py ./desktop/src-tauri/src/tauri_handlers/startup.rs ./desktop/src/routes/environments.tsx ./desktop/src/tests/routes/installation-progress.test.tsx diff --git a/openbb_platform/core/openbb_core/provider/standard_models/commodity_psd_data.py b/openbb_platform/core/openbb_core/provider/standard_models/commodity_psd_data.py new file mode 100644 index 00000000000..2d90fe5097e --- /dev/null +++ b/openbb_platform/core/openbb_core/provider/standard_models/commodity_psd_data.py @@ -0,0 +1,39 @@ +"""Commodity Production Supply & Demand Data Standard Model.""" + +from openbb_core.provider.abstract.data import Data +from openbb_core.provider.abstract.query_params import QueryParams +from pydantic import Field + + +class CommodityPsdDataQueryParams(QueryParams): + """Commodity Production Supply & Demand Data Query.""" + + +class CommodityPsdData(Data): + """Commodity Production Supply & Demand Data.""" + + region: str | None = Field(default=None, description="Region group category.") + country: str | None = Field( + default=None, + description="Country or area name.", + ) + commodity: str | None = Field( + default=None, + description="Commodity name.", + ) + attribute: str | None = Field( + default=None, + description="Name of the row value.", + ) + marketing_year: str | None = Field( + default=None, + description="Marketing year for the commodity.", + ) + value: float | int | None = Field( + default=None, + description="Value for the commodity attribute in the given marketing year.", + ) + unit: str | None = Field( + default=None, + description="Unit of measurement for the value.", + ) diff --git a/openbb_platform/core/openbb_core/provider/standard_models/commodity_psd_report.py b/openbb_platform/core/openbb_core/provider/standard_models/commodity_psd_report.py new file mode 100644 index 00000000000..ac533004198 --- /dev/null +++ b/openbb_platform/core/openbb_core/provider/standard_models/commodity_psd_report.py @@ -0,0 +1,29 @@ +"""Commodity Production Supply & Distribution Report Standard Model.""" + +from openbb_core.provider.abstract.data import Data +from openbb_core.provider.abstract.query_params import QueryParams +from pydantic import Field + + +class CommodityPsdReportQueryParams(QueryParams): + """Commodity Production Supply & Distribution Report Query.""" + + commodity: str = Field( + description="Commodity for the report.", + ) + year: int = Field( + description="Year of the report.", + ) + month: int = Field( + description="Month of the report.", + ge=1, + le=12, + ) + + +class CommodityPsdReportData(Data): + """Commodity Production Supply & Distribution Report Data.""" + + content: str = Field( + description="Base64 encoded content.", + ) diff --git a/openbb_platform/core/openbb_core/provider/standard_models/weather_bulletin.py b/openbb_platform/core/openbb_core/provider/standard_models/weather_bulletin.py new file mode 100644 index 00000000000..35c72cbf4b3 --- /dev/null +++ b/openbb_platform/core/openbb_core/provider/standard_models/weather_bulletin.py @@ -0,0 +1,42 @@ +"""Weather Bulletin Standard Model.""" + +from datetime import datetime + +from openbb_core.provider.abstract.data import Data +from openbb_core.provider.abstract.query_params import QueryParams +from pydantic import Field + + +class WeatherBulletinQueryParams(QueryParams): + """Weather Bulletin Query.""" + + year: int = Field( + description="Year of the data. Default is the current year.", + default=datetime.now().year, + ) + month: int | None = Field( + description="Month of the data. If not provided, data for the entire year is returned.", + ge=1, + le=12, + default=None, + ) + week: int | None = Field( + description="Numeric week of the data, relative to the month." + + " If not provided, data for the entire month is returned.", + ge=1, + le=5, + default=None, + ) + + +class WeatherBulletinData(Data): + """Weather Bulletin Data.""" + + label: str | None = Field( + default=None, + description="Label representing the weather bulletin file.", + ) + value: str | None = Field( + default=None, + description="URL to the weather bulletin document.", + ) diff --git a/openbb_platform/core/openbb_core/provider/standard_models/weather_bulletin_download.py b/openbb_platform/core/openbb_core/provider/standard_models/weather_bulletin_download.py new file mode 100644 index 00000000000..07cdb60ea29 --- /dev/null +++ b/openbb_platform/core/openbb_core/provider/standard_models/weather_bulletin_download.py @@ -0,0 +1,36 @@ +"""Weather Bulletin Download Standard Model.""" + +from openbb_core.provider.abstract.data import Data +from openbb_core.provider.abstract.query_params import QueryParams +from pydantic import Field, field_validator + + +class WeatherBulletinDownloadQueryParams(QueryParams): + """Weather Bulletin Query.""" + + urls: str | dict | list = Field( + kw_only=True, + description="URLs for reports to download.", + ) + + @field_validator("urls", mode="before", check_fields=False) + @classmethod + def _validate_urls(cls, v): + """Validate URLs input.""" + if isinstance(v, str): + if "," in v: + return v.split(",") + return [v] + if isinstance(v, dict) and "urls" in v: + return v["urls"] + if isinstance(v, list): + return v + raise ValueError("Invalid format for URLs. Must be str, dict, or list.") + + +class WeatherBulletinDownloadData(Data): + """Weather Bulletin Data.""" + + content: str = Field( + description="Base64 encoded content of the weather bulletin document.", + ) diff --git a/openbb_platform/extensions/commodity/integration/test_commodity_api.py b/openbb_platform/extensions/commodity/integration/test_commodity_api.py index c63b3a4c54f..0ee44d1859c 100644 --- a/openbb_platform/extensions/commodity/integration/test_commodity_api.py +++ b/openbb_platform/extensions/commodity/integration/test_commodity_api.py @@ -110,3 +110,123 @@ def test_commodity_short_term_energy_outlook(params, headers): result = requests.get(url, headers=headers, timeout=10) assert isinstance(result, requests.Response) assert result.status_code == 200 + + +@pytest.mark.parametrize( + "params", + [ + ( + { + "commodity": "sugar", + "year": 2025, + "month": 5, + "provider": "government_us", + } + ), + ], +) +@pytest.mark.integration +def test_commodity_psd_report(params, headers): + """Test the Commodity PSD Report endpoint.""" + params = {p: v for p, v in params.items() if v} + + query_str = get_querystring(params, []) + url = f"http://0.0.0.0:8000/api/v1/commodity/psd_report?{query_str}" + result = requests.get(url, headers=headers, timeout=10) + assert isinstance(result, requests.Response) + assert result.status_code == 200 + + +@pytest.mark.parametrize( + "params", + [ + ( + { + "urls": "https://esmis.nal.usda.gov/sites/default/release-files/cj82k728n/vx023b997/z890tr81b/wwcb1825.pdf" + + ",https://esmis.nal.usda.gov/sites/default/release-files/cj82k728n/w6635s43r/x059dz29h/wwcb1924.pdf", + "provider": "government_us", + } + ), + ], +) +@pytest.mark.integration +def test_commodity_weather_bulletins_download(params, headers): + """Test the Commodity Weather Bulletin Download endpoint.""" + params = {p: v for p, v in params.items() if v} + urls = params.pop("urls", "") + query_str = get_querystring(params, []) + url = f"http://0.0.0.0:8000/api/v1/commodity/weather_bulletins_download?{query_str}" + result = requests.post(url, headers=headers, json=urls, timeout=30) + assert isinstance(result, requests.Response) + assert result.status_code == 200 + + +@pytest.mark.parametrize( + "params", + [ + ( + { + "year": 2025, + "month": 5, + "week": 2, + "provider": "government_us", + } + ), + ], +) +@pytest.mark.integration +def test_commodity_weather_bulletins(params, headers): + """Test the Commodity Weather Bulletins endpoint.""" + params = {p: v for p, v in params.items() if v} + + query_str = get_querystring(params, []) + url = f"http://0.0.0.0:8000/api/v1/commodity/weather_bulletins?{query_str}" + result = requests.get(url, headers=headers, timeout=10) + assert isinstance(result, requests.Response) + assert result.status_code == 200 + + +@pytest.mark.parametrize( + "params", + [ + ( + { + "provider": "government_us", + } + ), + ( + { + "provider": "government_us", + "report_id": "coffee_summary", + "commodity": None, + "country": None, + "attribute": None, + "start_year": None, + "end_year": None, + "aggregate_regions": False, + } + ), + ( + { + "report_id": "world_crop_production_summary", # ignored if commodity is set + "commodity": "corn", + "country": "united_states,argentina", + "attribute": "exports", + "start_year": 2025, + "end_year": 2025, + "provider": "government_us", + "aggregate_regions": False, + } + ), + ], +) +@pytest.mark.integration +def test_commodity_psd_data(params, headers): + """Test the Commodity PSD Data endpoint.""" + params = {p: v for p, v in params.items() if v} + + query_str = get_querystring(params, []) + url = f"http://0.0.0.0:8000/api/v1/commodity/psd_data?{query_str}" + result = requests.get(url, headers=headers, timeout=10) + assert isinstance(result, requests.Response) + assert result.status_code == 200 diff --git a/openbb_platform/extensions/commodity/integration/test_commodity_python.py b/openbb_platform/extensions/commodity/integration/test_commodity_python.py index f955c6417ea..db8446e21c9 100644 --- a/openbb_platform/extensions/commodity/integration/test_commodity_python.py +++ b/openbb_platform/extensions/commodity/integration/test_commodity_python.py @@ -98,3 +98,122 @@ def test_commodity_short_term_energy_outlook(params, obb): assert result assert isinstance(result, OBBject) assert len(result.results) > 0 + + +@pytest.mark.parametrize( + "params", + [ + ( + { + "commodity": "sugar", + "year": 2025, + "month": 5, + "provider": "government_us", + } + ), + ], +) +@pytest.mark.integration +def test_commodity_psd_report(params, obb): + """Test Commodity PSD Report endpoint.""" + result = obb.commodity.psd_report(**params) + assert result + assert isinstance(result, dict) + assert result["data_format"]["data_type"] == "pdf" + + +@pytest.mark.parametrize( + "params", + [ + ( + { + "year": 2025, + "month": 5, + "week": 2, + "provider": "government_us", + } + ), + ], +) +@pytest.mark.integration +def test_commodity_weather_bulletins(params, obb): + """Test Commodity Weather Bulletins endpoint.""" + result = obb.commodity.weather_bulletins(**params) + assert result + assert isinstance(result, list) + assert len(result) > 0 + for bulletin in result: + assert bulletin["label"] + assert bulletin["value"] + + +@pytest.mark.parametrize( + "params", + [ + ( + { + "urls": [ + "https://esmis.nal.usda.gov/sites/default/release-files/cj82k728n/vx023b997/z890tr81b/wwcb1825.pdf", + "https://esmis.nal.usda.gov/sites/default/release-files/cj82k728n/w6635s43r/x059dz29h/wwcb1924.pdf", + ], + "provider": "government_us", + } + ), + ], +) +@pytest.mark.integration +def test_commodity_weather_bulletins_download(params, obb): + """Test Commodity Weather Bulletins Download endpoint.""" + result = obb.commodity.weather_bulletins_download(**params) + assert result + assert isinstance(result, list) + assert len(result) > 0 + for bulletin in result: + assert isinstance(bulletin, dict) + assert bulletin["content"] + assert bulletin["data_format"]["data_type"] == "pdf" + + +@pytest.mark.parametrize( + "params", + [ + ( + { + "provider": "government_us", + } + ), + ( + { + "provider": "government_us", + "report_id": "coffee_summary", + "commodity": None, + "country": None, + "attribute": None, + "start_year": None, + "end_year": None, + "aggregate_regions": False, + } + ), + ( + { + "report_id": "world_crop_production_summary", # ignored if commodity is set + "commodity": "corn", + "country": "united_states,argentina", + "attribute": "exports", + "start_year": 2025, + "end_year": 2025, + "provider": "government_us", + "aggregate_regions": False, + } + ), + ], +) +@pytest.mark.integration +def test_commodity_psd_data(params, obb): + """Test Commodity PSD Data endpoint.""" + params = {p: v for p, v in params.items() if v is not None} + + result = obb.commodity.psd_data(**params) + assert result + assert isinstance(result, OBBject) + assert len(result.results) > 0 diff --git a/openbb_platform/extensions/commodity/openbb_commodity/commodity_router.py b/openbb_platform/extensions/commodity/openbb_commodity/commodity_router.py index a5a080b56af..4ac3df54f8c 100644 --- a/openbb_platform/extensions/commodity/openbb_commodity/commodity_router.py +++ b/openbb_platform/extensions/commodity/openbb_commodity/commodity_router.py @@ -5,8 +5,10 @@ # pylint: disable=unused-argument +from datetime import datetime + from openbb_core.app.model.command_context import CommandContext -from openbb_core.app.model.example import APIEx +from openbb_core.app.model.example import APIEx, PythonEx from openbb_core.app.model.obbject import OBBject from openbb_core.app.provider_interface import ( ExtraParams, @@ -15,13 +17,13 @@ from openbb_core.app.provider_interface import ( ) from openbb_core.app.query import Query from openbb_core.app.router import Router +from openbb_core.app.service.system_service import SystemService from openbb_commodity.price.price_router import router as price_router router = Router(prefix="", description="Commodity market data.") - - router.include_router(price_router) +api_prefix = SystemService().system_settings.api_settings.prefix @router.command( @@ -78,3 +80,288 @@ async def short_term_energy_outlook( Source: www.eia.gov/steo/ """ return await OBBject.from_query(Query(**locals())) + + +@router.command( + model="CommodityPsdData", + examples=[ + APIEx( + description="Get the World Crop Production Summary table.", + parameters={ + "provider": "government_us", + }, + ), + APIEx( + description="Get the current Corn World Trade table from the PDS report.", + parameters={ + "provider": "government_us", + "report_id": "corn_world_trade", + }, + ), + APIEx( + description="Get all attributes for Coffee globally, for a single year.", + parameters={ + "provider": "government_us", + "commodity": "coffee", + "start_year": 2025, + "end_year": 2025, + }, + ), + APIEx( + description="Compare Brazil coffee exports versus the world from 2010 to present.", + parameters={ + "provider": "government_us", + "commodity": "coffee", + "country": "brazil", + "attribute": "exports", + "aggregate_regions": True, + "start_year": 2010, + }, + ), + APIEx( + description="Get historical production of corn in the US from 2020.", + parameters={ + "provider": "government_us", + "commodity": "corn", + "country": "united_states", + "attribute": "production", + "start_year": 2020, + }, + ), + APIEx( + description="Get regional aggregates for wheat beginning and ending stocks from 2020.", + parameters={ + "provider": "government_us", + "commodity": "wheat", + "country": "world", + "attribute": "beginning_stocks,ending_stocks", + "aggregate_regions": True, + "start_year": 2020, + }, + ), + ], +) +async def psd_data( + cc: CommandContext, + provider_choices: ProviderChoices, + standard_params: StandardParams, + extra_params: ExtraParams, +) -> OBBject: + """Get data tables and historical time series from the USDA FAS Production, Supply, and Distribution (PSD) Reports.""" + return await OBBject.from_query(Query(**locals())) + + +@router.command( + model="CommodityPsdReport", + no_validate=True, + widget_config={ + "name": "USDA FAS Commodity Production Supply & Distribution Reports", + "description": "Monthly publications released by the USDA Foreign Agriculture Service.", + "type": "pdf", + "refetchInterval": False, + "gridData": { + "w": 20, + "h": 30, + }, + "category": "Commodity", + "subCategory": "Agriculture", + "source": ["USDA", "FAS"], + }, + examples=[ + APIEx( + parameters={ + "provider": "government_us", + "commodity": "sugar", + "year": 2022, + "month": 5, + } + ), + APIEx( + description="Get the PSD report for coffee for March 2023.", + parameters={ + "provider": "government_us", + "commodity": "coffee", + "year": 2023, + "month": 3, + }, + ), + ], +) +async def psd_report( + cc: CommandContext, + provider_choices: ProviderChoices, + standard_params: StandardParams, + extra_params: ExtraParams, +) -> OBBject: + """Agriculture commodity production, supply, and distribution PDF reports (World Agricultural Outlook). + + This command returns only the results portion of the OBBject response. + It contains a dictionary where the PDF content is base64 encoded under the 'content' key. + """ + response = await OBBject.from_query(Query(**locals())) + return response.model_dump().get("results", {}) + + +@router.command( + model="WeatherBulletin", + no_validate=True, + widget_config={"exclude": True}, + examples=[ + APIEx( + description="Get weather bulletins for the current year.", + parameters={ + "provider": "government_us", + }, + ), + APIEx( + description="Get weather bulletins for May 2023, week 2.", + parameters={ + "provider": "government_us", + "year": 2023, + "month": 5, + "week": 2, + }, + ), + PythonEx( + description="Get URLs for comparing versus 1 year ago and download the base64-encoded PDF content to memory.", + code=[ + "from datetime import datetime", + "urls = []", + "for year in [datetime.now().year, datetime.now().year - 1]:", + " urls.append(obb.commodity.weather_bulletins(year=year, month=5, week=2)[0]['value'])", + "pdfs = obb.commodity.weather_bulletins_download(urls=urls)", + "# PDFs are now in a list where each item has 'content' and 'data_format' keys", + ], + ), + ], +) +async def weather_bulletins( + cc: CommandContext, + provider_choices: ProviderChoices, + standard_params: StandardParams, + extra_params: ExtraParams, +) -> OBBject: + """Get current and historical weather bulletins with their PDF links. + + This command returns only the results portion of the OBBject response. + It contains a list of dictionaries where each dictionary has 'label' and 'value' keys. + + Use this endpoint to programmatically access the list of available weather bulletins. + Suitable for dropdown selections in a UI. + """ + response = await OBBject.from_query(Query(**locals())) + return response.model_dump().get("results", {}) + + +@router.command( + methods=["POST"], + model="WeatherBulletinDownload", + no_validate=True, + widget_config={ + "name": "USDA Weather & Crop Bulletin", + "description": "Weekly Weather and Crop Bulletin from the USDA.", + "type": "multi_file_viewer", + "refetchInterval": False, + "gridData": { + "w": 20, + "h": 30, + }, + "category": "Commodity", + "subCategory": "Agriculture", + "source": ["USDA", "WAOB"], + "params": [ + { + "paramName": "urls", + "type": "endpoint", + "optionsEndpoint": f"{api_prefix}/commodity/weather_bulletins", + "optionsParams": { + "year": "$year", + "month": "$month", + "week": "$week", + "provider": "government_us", + }, + "show": False, + "multiSelect": True, + "roles": ["fileSelector"], + }, + { + "paramName": "year", + "type": "number", + "label": "Year", + "value": datetime.now().year, + "options": [ + {"value": year, "label": str(year)} + for year in sorted( + list(range(1974, datetime.now().year + 1)), + reverse=True, + ) + ], + }, + { + "paramName": "month", + "type": "number", + "label": "Month", + "value": None, + "options": [ + {"value": i, "label": month} + for i, month in enumerate( + [ + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December", + ], + start=1, + ) + ] + + [{"value": None, "label": "All Months"}], + }, + { + "paramName": "week", + "type": "number", + "label": "Week", + "value": None, + "options": [{"value": week, "label": str(week)} for week in range(1, 6)] + + [{"value": None, "label": "All Weeks"}], + }, + { + "paramName": "provider", + "show": False, + "value": "government_us", + "type": "text", + "options": [{"value": "government_us", "label": "government_us"}], + }, + ], + }, + examples=[ + APIEx( + parameters={ + "provider": "government_us", + "urls": [ + "https://esmis.nal.usda.gov/sites/default/release-files/cj82k728n/9w033w568/x059f4232/wwcb0125.pdf" + ], + } + ), + ], +) +async def weather_bulletins_download( + cc: CommandContext, + provider_choices: ProviderChoices, + standard_params: StandardParams, + extra_params: ExtraParams, +) -> OBBject: + """Download one, or more, weather bulletin documents. + + This command returns only the results portion of the OBBject response. + It contains a list of dictionaries where the base64 encoded content of the document is under the 'content' key. + """ + response = await OBBject.from_query(Query(**locals())) + return response.model_dump().get("results", {}) diff --git a/openbb_platform/providers/government_us/README.md b/openbb_platform/providers/government_us/README.md index 1fecd39fd8c..e4ac9fda9dc 100644 --- a/openbb_platform/providers/government_us/README.md +++ b/openbb_platform/providers/government_us/README.md @@ -10,4 +10,17 @@ To install the extension: pip install openbb-us-government ``` -Documentation available [here](https://docs.openbb.co/platform/developer_guide/contributing). +See the OpenBB Documentation [here](https://docs.openbb.co/python/) for details on getting started. + +## Coverage + +This extension covers services with no registration requirement. + +- `obb.commodity.psd_report` +- `obb.commodity.psd_data` +- `obb.commodity.weather_bulletins` +- `obb.commodity.weather_bulletins_download` +- `obb.fixed_income.government.treasury_auctions` +- `obb.fixed_income.government.treasury_prices` + +See the [reference](https://docs.openbb.co/python/reference) for breakdowns of each endpoint. diff --git a/openbb_platform/providers/government_us/openbb_government_us/__init__.py b/openbb_platform/providers/government_us/openbb_government_us/__init__.py index 2a15283a7a9..165f6ddae91 100644 --- a/openbb_platform/providers/government_us/openbb_government_us/__init__.py +++ b/openbb_platform/providers/government_us/openbb_government_us/__init__.py @@ -1,12 +1,24 @@ """Government US provider module.""" from openbb_core.provider.abstract.provider import Provider +from openbb_government_us.models.commodity_psd_data import ( + GovernmentUsCommodityPsdDataFetcher, +) +from openbb_government_us.models.commodity_psd_report import ( + GovernmentUsCommodityPsdReportFetcher, +) from openbb_government_us.models.treasury_auctions import ( GovernmentUSTreasuryAuctionsFetcher, ) from openbb_government_us.models.treasury_prices import ( GovernmentUSTreasuryPricesFetcher, ) +from openbb_government_us.models.weather_bulletin import ( + GovernmentUsWeatherBulletinFetcher, +) +from openbb_government_us.models.weather_bulletin_download import ( + GovernmentUsWeatherBulletinDownloadFetcher, +) government_us_provider = Provider( name="government_us", @@ -15,10 +27,14 @@ government_us_provider = Provider( It provides access to datasets published by agencies across the federal government. Data.gov is intended to provide access to government open data to the public, achieve agency missions, drive innovation, fuel economic activity, and uphold the ideals of -an open and transparent government.""", +an open and transparent government. https://api.data.gov/signup/""", fetcher_dict={ + "CommodityPsdData": GovernmentUsCommodityPsdDataFetcher, + "CommodityPsdReport": GovernmentUsCommodityPsdReportFetcher, "TreasuryAuctions": GovernmentUSTreasuryAuctionsFetcher, "TreasuryPrices": GovernmentUSTreasuryPricesFetcher, + "WeatherBulletin": GovernmentUsWeatherBulletinFetcher, + "WeatherBulletinDownload": GovernmentUsWeatherBulletinDownloadFetcher, }, repr_name="Data.gov | United States Government", ) diff --git a/openbb_platform/providers/government_us/openbb_government_us/models/commodity_psd_data.py b/openbb_platform/providers/government_us/openbb_government_us/models/commodity_psd_data.py new file mode 100644 index 00000000000..3c1d7c5764e --- /dev/null +++ b/openbb_platform/providers/government_us/openbb_government_us/models/commodity_psd_data.py @@ -0,0 +1,411 @@ +"""US Government Production Supply & Distribution Report Data & Time Series Model.""" + +# pylint: disable=unused-argument + +from datetime import datetime +from typing import Any + +from openbb_core.app.model.abstract.error import OpenBBError +from openbb_core.provider.abstract.fetcher import Fetcher +from openbb_core.provider.standard_models.commodity_psd_data import ( + CommodityPsdData, + CommodityPsdDataQueryParams, +) +from openbb_government_us.utils.psd_codes import ( + ATTRIBUTES, + COMMODITIES, + COUNTRIES, + PSD_REPORT_NAMES, + REGIONS, +) +from pydantic import ConfigDict, Field, field_validator, model_validator + +COUNTRY_CHOICES = { + **{k: v for k, v in COUNTRIES.items() if k != "world"}, + **REGIONS, +} + + +class GovernmentUsCommodityPsdDataQueryParams(CommodityPsdDataQueryParams): + """US Government Commodity PSD Data Query Params. + + Source: https://apps.fas.usda.gov/psdonline/app/index.html#/app/downloads + """ + + __json_schema_extra__ = { + "report_id": { + "x-widget_config": { + "description": "Report ID to retrieve. Gets the current report for the given commodity and subject. " + + "Ignored if 'commodity' is provided.", + "value": "world_crop_production_summary", + "options": [ + { + "label": name.replace("_", " ") + .title() + .replace("Eu", "EU") + .replace("Us", "US"), + "value": name, + } + for name in sorted(list(PSD_REPORT_NAMES)) + ], + "style": {"popupWidth": 350}, + "row": 1, + }, + }, + "commodity": { + "x-widget_config": { + "description": "Commodity name to filter the data. " + + "If provided, overrides the Report ID and retrieves time series data for the given commodity.", + "value": None, + "options": [{"label": "Report Mode", "value": None}] + + [ + {"label": comm.replace("_", " ").title(), "value": comm} + for comm in sorted(list(COMMODITIES)) + ], + "row": 1, + }, + }, + "attribute": { + "multiple_items_allowed": True, + "x-widget_config": { + "description": "Attribute to filter the data. Ignored when commodity is Report Mode. " + + "If None, retrieves all available attributes for the commodity.", + "options": [{"label": "All Attributes", "value": None}] + + [ + { + "label": attr.replace("_", " ") + .title() + .replace("Us", "US") + .replace("Ty", "TY") + .replace("Fsi", "FSI"), + "value": attr, + } + for attr in sorted(list(ATTRIBUTES)) + ], + "row": 1, + }, + }, + "country": { + "multiple_items_allowed": True, + "x-widget_config": { + "description": "Country code(s) to filter the data. Ignored when commodity is Report Mode. " + + "If None, retrieves data for all countries.", + "options": [{"label": "All Countries", "value": None}] + + [ + {"label": name.replace("_", " ").title(), "value": code} + for name, code in COUNTRY_CHOICES.items() + ], + "row": 1, + }, + }, + "start_year": { + "x-widget_config": { + "description": "Start year for filtering time series data. Ignored when commodity is Report Mode. " + + "None returns from the beginning of the series.", + "type": "number", + "options": [{"label": "Start Year", "value": None}] + + [ + {"label": str(year), "value": year} + for year in sorted( + list(range(1960, datetime.now().year + 1)), reverse=True + ) + ], + "row": 1, + }, + }, + "end_year": { + "x-widget_config": { + "description": "End year for filtering time series data." + + " Ignored when commodity is Report Mode. If None, returns up to the most recent year.", + "type": "number", + "options": [{"label": "End Year", "value": None}] + + [ + {"label": str(year), "value": year} + for year in sorted( + list(range(1960, datetime.now().year + 1)), reverse=True + ) + ], + "row": 1, + }, + }, + "aggregate_regions": { + "x-widget_config": { + "description": "Whether to include regional and world aggregates in the data." + + " Ignored when 'commodity' is Report Mode.", + "type": "boolean", + "value": False, + "row": 1, + }, + }, + } + + report_id: str | None = Field( + default="world_crop_production_summary", + description="Report ID to retrieve. Gets the current report for the given commodity and subject. " + + "These are predefined tables that are part of the PDF publication data. " + + "This parameter is ignored if 'commodity' is provided. " + + "Use the 'commodity' parameter for time series data. " + + "Valid reports are:\n " + + ", ".join(sorted(list(PSD_REPORT_NAMES.keys()))) + + "\n", + ) + commodity: str | None = Field( + default=None, + description="Commodity name to filter the data. If provided, retrieves time series data for the given commodity. " + + "Supplying both 'report_id' and 'commodity' will prioritize 'commodity' for time series data. " + + "Valid commodities are:\n " + + ", ".join(sorted(list(COMMODITIES))) + + "\n", + ) + attribute: str | list[str] | None = Field( + default=None, + description="Attribute to filter the data. If None, retrieves all available attributes for the commodity.\n" + + "Parameter is ignored when commodity is None. Valid attributes depend on the commodity, " + + "an invalid choice will show the available attributes for the entered commodity.\n" + + "All attributes choices are:\n" + + ", ".join(sorted(list(ATTRIBUTES))) + + "\n", + ) + country: str | list[str] | None = Field( + default=None, + description="Country code(s) to filter the data. If None, retrieves data for all countries.\n" + + "Parameter is ignored when commodity is None. Valid country codes include:\n" + + ", ".join( + sorted(list(COUNTRY_CHOICES)), + ) + + "\n", + ) + aggregate_regions: bool = Field( + default=False, + description="Whether to include regional and world aggregates in the data. " + + "Parameter is ignored when 'commodity' is None.", + ) + start_year: int | None = Field( + default=None, + description="Start year for filtering time series data. None returns from the beginning of the series.\n" + + "Parameter is ignored when 'commodity' is None.", + ) + end_year: int | None = Field( + default=None, + description="End year for filtering time series data. If None, returns up to the most recent year.\n" + + "Parameter is ignored when 'commodity' is None.", + ) + + @field_validator("report_id", mode="before", check_fields=False) + @classmethod + def _validate_report_id(cls, v): + """Validate report_id.""" + if not v: + return "world_crop_production_summary" + + if v not in PSD_REPORT_NAMES: + raise ValueError( + f"Invalid report_id '{v}'. Valid report IDs are: " + + ", ".join(sorted(list(PSD_REPORT_NAMES.keys()))) + ) + return v + + @field_validator("commodity", mode="before", check_fields=False) + @classmethod + def _validate_commodity(cls, v): + """Validate commodity.""" + if not v: + return None + if v and v not in COMMODITIES: + raise ValueError( + f"Invalid commodity '{v}'. Valid commodities are: " + + ", ".join(sorted(list(COMMODITIES))) + ) + return v + + @field_validator("attribute", mode="before", check_fields=False) + @classmethod + def _validate_attribute(cls, v): + """Validate attribute.""" + if not v: + return None + + if v and isinstance(v, list) and not v[0]: + return None + + attributes = v + + if isinstance(attributes, str): + attributes = ( + [attributes] if "," not in attributes else attributes.split(",") + ) + + if ( + isinstance(attributes, list) + and len(attributes) == 1 + and "," in attributes[0] + ): + attributes = attributes[0].split(",") + + if not isinstance(attributes, list): + raise ValueError( + f"Attribute must be a string or list of strings. Got {type(v)}" + ) + + invalid_attrs = [attr for attr in attributes if attr not in ATTRIBUTES] + + if invalid_attrs: + raise ValueError( + f"Invalid attribute(s) '{', '.join(invalid_attrs)}'. Valid attributes are: " + + ", ".join(sorted(list(ATTRIBUTES))) + ) + + return attributes + + @field_validator("country", mode="before", check_fields=False) + @classmethod + def _validate_country(cls, v): + """Validate country.""" + if not v: + return None + + if v and isinstance(v, list) and not v[0]: + return None + + countries = v + + if isinstance(countries, str): + countries = [countries] if "," not in countries else countries.split(",") + + if not isinstance(countries, list): + raise ValueError( + f"Country must be a string or list of strings. Got {type(v)}" + ) + + if isinstance(countries, list) and len(countries) == 1 and "," in countries[0]: + countries = countries[0].split(",") + + invalid_countries = [ + country + for country in countries + if country + and country not in COUNTRY_CHOICES.values() + and country not in COUNTRY_CHOICES + ] + + if invalid_countries: + raise ValueError( + f"Invalid country code(s) '{', '.join(invalid_countries)}'. Valid country codes are: " + + ", ".join( + sorted(list(COUNTRY_CHOICES)), + ) + ) + + return v + + @field_validator("start_year", mode="before", check_fields=False) + @classmethod + def _validate_start_year(cls, v): + """Validate start_year.""" + if v and v < 1960: + raise ValueError("Earliest possible year is 1960.") + return v + + @model_validator(mode="after") + def _validate_model(self): + """Validate that start_year is less than or equal to end_year.""" + report_id = getattr(self, "report_id", None) + commodity = getattr(self, "commodity", None) + + if not report_id and not commodity: + raise ValueError( + "Either 'report_id' or 'commodity' must be provided." + + " If both are provided, 'commodity' takes precedence." + ) + + return self + + +class GovernmentUsCommodityPsdData(CommodityPsdData): + """US Government Commodity PSD Data.""" + + model_config = ConfigDict( + json_schema_extra={ + "x-widget_config": { + "$.name": "USDA FAS Commodity Production Supply & Distribution Data", + "$.description": "Predefined presentation tables, or time series data by commodity." + + " Source: USDA Foreign Agricultural Service (FAS) Production Supply & Distribution Online.", + "$.source": ["USDA", "FAS"], + "$.category": "Commodity", + "$.subCategory": "Agriculture", + "$.runButton": True, + }, + } + ) + + +class GovernmentUsCommodityPsdDataFetcher( + Fetcher[ + GovernmentUsCommodityPsdDataQueryParams, + list[GovernmentUsCommodityPsdData], + ] +): + """US Government Commodity PSD Data Fetcher.""" + + @staticmethod + def transform_query( + params: dict[str, Any], + ) -> GovernmentUsCommodityPsdDataQueryParams: + """Transform the query parameters into the model.""" + return GovernmentUsCommodityPsdDataQueryParams(**params) + + @staticmethod + async def aextract_data( + query: GovernmentUsCommodityPsdDataQueryParams, + credentials: dict[str, Any] | None, + **kwargs: Any, + ) -> list: + """Extract data from the provider.""" + # pylint: disable=import-outside-toplevel + from openbb_government_us.utils.psd_data_downloader import ( + get_psd_report_data, + get_timeseries, + ) + + if query.commodity is not None: + # Time series data + try: + timeseries_data = get_timeseries( + commodity=query.commodity, + attribute=query.attribute, + country=query.country, + aggregate_region=query.aggregate_regions, + start_year=query.start_year, + end_year=query.end_year, + ) + return timeseries_data + except (ValueError, OpenBBError) as e: + raise OpenBBError(e) from None + + report_id = PSD_REPORT_NAMES[query.report_id or "world_crop_production_summary"] + + try: + report_data = await get_psd_report_data(report_id) + + if not report_data.get("data", []): + raise OpenBBError( + "The request was successful but no data was returned. This might be a bug or a network error." + + f" -> Report ID: {query.report_id}" + ) + if report_data.get("error", ""): + raise OpenBBError( + f"Error processing data: {report_data['error']} -> Report ID: {query.report_id}" + ) + except OpenBBError as e: + raise e from e + + return report_data.get("data", []) + + @staticmethod + def transform_data( + query: GovernmentUsCommodityPsdDataQueryParams, + data: list, + **kwargs: Any, + ) -> list[GovernmentUsCommodityPsdData]: + """Transform and validate the data.""" + return [GovernmentUsCommodityPsdData.model_validate(item) for item in data] diff --git a/openbb_platform/providers/government_us/openbb_government_us/models/commodity_psd_report.py b/openbb_platform/providers/government_us/openbb_government_us/models/commodity_psd_report.py new file mode 100644 index 00000000000..429735046b3 --- /dev/null +++ b/openbb_platform/providers/government_us/openbb_government_us/models/commodity_psd_report.py @@ -0,0 +1,197 @@ +"""US Government Production Supply & Distribution Publications Model.""" + +# pylint: disable=unused-argument + +from typing import Any + +from openbb_core.app.model.abstract.error import OpenBBError +from openbb_core.provider.abstract.fetcher import Fetcher +from openbb_core.provider.standard_models.commodity_psd_report import ( + CommodityPsdReportData, + CommodityPsdReportQueryParams, +) +from pydantic import Field, field_validator + +COMMODITIES = { + "citrus": "Citrus", + "coffee": "Coffee", + "cotton": "Cotton", + "dairy": "Dairy", + "fruit": "Fruit", + "grain": "Grain", + "livestock": "Livestock_poultry", + "oilseeds": "Oilseeds", + "stone_fruit": "StoneFruit", + "sugar": "Sugar", + "tree_nuts": "TreeNuts", + "world_production": "production", +} + + +class GovernmentUsCommodityPsdReportQueryParams(CommodityPsdReportQueryParams): + """US Government Commodity PSD Report Query Params. + + Source: https://apps.fas.usda.gov/psdonline/app/index.html#/app/downloads + """ + + __json_schema_extra__ = { + "year": { + "x-widget_config": { + "description": "Year of the report to retrieve. Data is available from 2006 onwards.", + "type": "number", + "row": 1, + } + }, + "month": { + "x-widget_config": { + "description": "Month of the report to retrieve.", + "row": 1, + "options": [ + {"value": i, "label": month} + for i, month in enumerate( + [ + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December", + ], + start=1, + ) + ], + } + }, + "commodity": { + "multiple_items_allowed": False, + "choices": list(COMMODITIES), + "x-widget_config": { + "row": 1, + "description": "Commodity for the report to retrieve.", + "options": [ + {"value": commodity, "label": commodity.replace("_", " ").title()} + for commodity in COMMODITIES + ], + }, + }, + } + + @field_validator("year", mode="before", check_fields=False) + @classmethod + def _validate_year(cls, v): + """Validate year is >= 2006.""" + if v < 2006: + raise OpenBBError(ValueError("Data is only available from 2006 onwards.")) + return v + + @field_validator("commodity", mode="before", check_fields=False) + @classmethod + def _validate_commodity(cls, v): + """Validate commodity is in the list.""" + if v.lower() not in COMMODITIES: + raise OpenBBError( + ValueError( + f"Commodity '{v}' is not supported. " + + "Please choose from the available commodities -> " + + f"{', '.join(COMMODITIES.keys())}" + ) + ) + return v.lower() + + +class GovernmentUsCommodityPsdReportData(CommodityPsdReportData): + """US Government Commodity PSD Report Data.""" + + data_format: dict[str, str] = Field( + description="Data format information.", + default_factory=lambda: { + "data_type": "pdf", + "filename": "commodity_psd_report.pdf", + }, + ) + + +class GovernmentUsCommodityPsdReportFetcher( + Fetcher[ + GovernmentUsCommodityPsdReportQueryParams, + GovernmentUsCommodityPsdReportData, + ] +): + """US Government Commodity PSD Report Fetcher.""" + + @staticmethod + def transform_query( + params: dict[str, Any], + ) -> GovernmentUsCommodityPsdReportQueryParams: + """Transform params into the query params model.""" + return GovernmentUsCommodityPsdReportQueryParams(**params) + + @staticmethod + def extract_data( + query: GovernmentUsCommodityPsdReportQueryParams, + credentials: dict[str, Any] | None, + **kwargs: Any, + ) -> dict: + """Extract data from the provider.""" + # pylint: disable=import-outside-toplevel + from openbb_core.provider.utils.helpers import make_request + + api_commodity = COMMODITIES.get(query.commodity.lower(), query.commodity) + month = f"{query.month:02d}" + url = f"https://apps.fas.usda.gov/PSDOnline/CircularDownloader.ashx?year={query.year}&month={month}&commodity={api_commodity}" + output = b"" + + try: + response = make_request(url) + + if response.status_code == 404 or not response.content: + raise OpenBBError( + ValueError( + "No data found for the given parameters. Please check the commodity, year, and month." + ) + ) + + if response.status_code == 200 and response.content[:4] == b"%PDF": + output = response.content + else: + raise OpenBBError( + ValueError( + "Failed to retrieve data. Unexpected response from the server." + + f" Status code: {response.status_code} -> " + + f"Response content: {response.content.decode('utf-8', errors='ignore')}" + ) + ) + except Exception as e: + raise OpenBBError(e) from e + + return {"content": output} + + @staticmethod + def transform_data( + query: GovernmentUsCommodityPsdReportQueryParams, + data: dict, + **kwargs: Any, + ) -> GovernmentUsCommodityPsdReportData: + """Transform data into the encoded format.""" + # pylint: disable=import-outside-toplevel + import base64 + + if not data: + raise OpenBBError("The request was successful but no data was returned.") + + encoded_content = base64.b64encode(data.get("content", b"")).decode("utf-8") # type: ignore + filename = f"psd_report_{query.commodity}_{query.year}_{query.month:02d}.pdf" + data_format = { + "data_type": "pdf", + "filename": filename, + } + + return GovernmentUsCommodityPsdReportData( + content=encoded_content, data_format=data_format + ) diff --git a/openbb_platform/providers/government_us/openbb_government_us/models/weather_bulletin.py b/openbb_platform/providers/government_us/openbb_government_us/models/weather_bulletin.py new file mode 100644 index 00000000000..585f9b719ba --- /dev/null +++ b/openbb_platform/providers/government_us/openbb_government_us/models/weather_bulletin.py @@ -0,0 +1,191 @@ +"""Government US Weather Bulletin Model""" + +# pylint: disable=unused-argument +from datetime import datetime +from typing import Any + +from openbb_core.app.model.abstract.error import OpenBBError +from openbb_core.provider.abstract.fetcher import Fetcher +from openbb_core.provider.standard_models.weather_bulletin import ( + WeatherBulletinData, + WeatherBulletinQueryParams, +) +from pydantic import ConfigDict, field_validator + + +class GovernmentUsWeatherBulletinQueryParams(WeatherBulletinQueryParams): + """US Government Weather Bulletin Query Params. + + Source: https://esmis.nal.usda.gov/publication/weekly-weather-and-crop-bulletin + """ + + @field_validator("year") + @classmethod + def _validate_year(cls, value: int) -> int: + """Validate year.""" + if value < 1974: + raise ValueError("Year must not be before 1974.") + if value > datetime.now().year: + raise OpenBBError(ValueError("Invalid year. Year cannot be in the future.")) + return value + + +class GovernmentUsWeatherBulletinData(WeatherBulletinData): + """US Government Weather Bulletin Data. + + The Weekly Weather and Crop Bulletin is released by 4:00 p.m. on the second workday of each week. + + The Bulletin includes the National Summary, State Stories, current data for weather, temperature and precipitation + and international agricultural weather. The full report is jointly published by the National Oceanic + and Atmospheric Administration of the U.S. Department of Commerce, the National Agricultural Statistics Service, + and the World Agricultural Outlook Board. + """ + + model_config = ConfigDict( + extra="ignore", + ) + + +class GovernmentUsWeatherBulletinFetcher( + Fetcher[ + GovernmentUsWeatherBulletinQueryParams, + list[GovernmentUsWeatherBulletinData], + ] +): + """US Government Weather Bulletin Fetcher.""" + + @staticmethod + def transform_query( + params: dict[str, Any], + ) -> GovernmentUsWeatherBulletinQueryParams: + """Transform params into the query params model.""" + return GovernmentUsWeatherBulletinQueryParams(**params) + + @staticmethod + async def aextract_data( + query: GovernmentUsWeatherBulletinQueryParams, + credentials: dict[str, str] | None = None, + **kwargs: Any, + ) -> list: + """Asynchronously extract data from the US Government Weather Bulletin API.""" + # pylint: disable=import-outside-toplevel + import asyncio # noqa + import re + from openbb_core.provider.utils.helpers import get_async_requests_session + + BASE_URL = "https://esmis.nal.usda.gov" + PUBLICATION_URL = f"{BASE_URL}/publication/weekly-weather-and-crop-bulletin" + months_to_fetch = ( + [query.month] if query.month is not None else list(range(1, 13)) + ) + year = query.year + week = query.week + month = query.month + + try: + async with await get_async_requests_session() as session: + session._max_field_size = 32768 # pylint: disable=protected-access + + async def fetch_month(m: int) -> list[dict]: + """Fetch bulletins for a single month.""" + url = f"{PUBLICATION_URL}?date={year}-{m:02d}" + bulletins = [] + + try: + resp = await session.get(url) + if resp.status != 200: + return [] + html = await resp.text() + except Exception: + return [] + + pattern = ( + r'href="(/sites/default/release-files/[^"]+\.pdf)"[^>]*>.*?' + r'