From 67cc3bb3c80bcc2bfe75490bb3f7bd8a48c70320 Mon Sep 17 00:00:00 2001
From: montezdesousa <79287829+montezdesousa@users.noreply.github.com>
Date: Mon, 25 Sep 2023 12:46:26 +0100
Subject: [PATCH 1/4] return None instead of nan, nan breaks API (#5475)

---
 openbb_sdk/providers/fred/openbb_fred/models/ameribor_rates.py  | 2 +-
 openbb_sdk/providers/fred/openbb_fred/models/estr_rates.py      | 2 +-
 openbb_sdk/providers/fred/openbb_fred/models/fed_rates.py       | 2 +-
 openbb_sdk/providers/fred/openbb_fred/models/iorb_rates.py      | 2 +-
 openbb_sdk/providers/fred/openbb_fred/models/sofr_rates.py      | 2 +-
 openbb_sdk/providers/fred/openbb_fred/models/sonia_rates.py     | 2 +-
 .../provider/openbb_provider/standard_models/ameribor_rates.py  | 2 +-
 .../sdk/provider/openbb_provider/standard_models/estr_rates.py  | 2 +-
 .../sdk/provider/openbb_provider/standard_models/fed_rates.py   | 2 +-
 .../sdk/provider/openbb_provider/standard_models/iorb_rates.py  | 2 +-
 .../sdk/provider/openbb_provider/standard_models/sofr_rates.py  | 2 +-
 .../sdk/provider/openbb_provider/standard_models/sonia_rates.py | 2 +-
 12 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/openbb_sdk/providers/fred/openbb_fred/models/ameribor_rates.py b/openbb_sdk/providers/fred/openbb_fred/models/ameribor_rates.py
index 540c6e4f9c9..16d2d4dd939 100644
--- a/openbb_sdk/providers/fred/openbb_fred/models/ameribor_rates.py
+++ b/openbb_sdk/providers/fred/openbb_fred/models/ameribor_rates.py
@@ -59,7 +59,7 @@ class FREDAMERIBORData(AMERIBORData):
         try:
             return float(v)
         except ValueError:
-            return float("nan")
+            return None
 
 
 class FREDAMERIBORFetcher(
diff --git a/openbb_sdk/providers/fred/openbb_fred/models/estr_rates.py b/openbb_sdk/providers/fred/openbb_fred/models/estr_rates.py
index fc95787f723..0883b77a588 100644
--- a/openbb_sdk/providers/fred/openbb_fred/models/estr_rates.py
+++ b/openbb_sdk/providers/fred/openbb_fred/models/estr_rates.py
@@ -50,7 +50,7 @@ class FREDESTRData(ESTRData):
         try:
             return float(v)
         except ValueError:
-            return float("nan")
+            return None
 
 
 class FREDESTRFetcher(
diff --git a/openbb_sdk/providers/fred/openbb_fred/models/fed_rates.py b/openbb_sdk/providers/fred/openbb_fred/models/fed_rates.py
index 1385653000c..7461e75e565 100644
--- a/openbb_sdk/providers/fred/openbb_fred/models/fed_rates.py
+++ b/openbb_sdk/providers/fred/openbb_fred/models/fed_rates.py
@@ -51,7 +51,7 @@ class FREDFEDData(FEDData):
         try:
             return float(v)
         except ValueError:
-            return float("nan")
+            return None
 
 
 class FREDFEDFetcher(Fetcher[FREDFEDQueryParams, List[Dict[str, List[FREDFEDData]]]]):
diff --git a/openbb_sdk/providers/fred/openbb_fred/models/iorb_rates.py b/openbb_sdk/providers/fred/openbb_fred/models/iorb_rates.py
index 3a7d644cb83..26e9381229d 100644
--- a/openbb_sdk/providers/fred/openbb_fred/models/iorb_rates.py
+++ b/openbb_sdk/providers/fred/openbb_fred/models/iorb_rates.py
@@ -31,7 +31,7 @@ class FREDIORBData(IORBData):
         try:
             return float(v)
         except ValueError:
-            return float("nan")
+            return None
 
 
 class FREDIORBFetcher(
diff --git a/openbb_sdk/providers/fred/openbb_fred/models/sofr_rates.py b/openbb_sdk/providers/fred/openbb_fred/models/sofr_rates.py
index 12dffcebc94..5847aa355c3 100644
--- a/openbb_sdk/providers/fred/openbb_fred/models/sofr_rates.py
+++ b/openbb_sdk/providers/fred/openbb_fred/models/sofr_rates.py
@@ -40,7 +40,7 @@ class FREDSOFRData(SOFRData):
         try:
             return float(v)
         except ValueError:
-            return float("nan")
+            return None
 
 
 class FREDSOFRFetcher(
diff --git a/openbb_sdk/providers/fred/openbb_fred/models/sonia_rates.py b/openbb_sdk/providers/fred/openbb_fred/models/sonia_rates.py
index 0fa3a16933f..88f40f17907 100644
--- a/openbb_sdk/providers/fred/openbb_fred/models/sonia_rates.py
+++ b/openbb_sdk/providers/fred/openbb_fred/models/sonia_rates.py
@@ -48,7 +48,7 @@ class FREDSONIAData(SONIAData):
         try:
             return float(v)
         except ValueError:
-            return float("nan")
+            return None
 
 
 class FREDSONIAFetcher(
diff --git a/openbb_sdk/sdk/provider/openbb_provider/standard_models/ameribor_rates.py b/openbb_sdk/sdk/provider/openbb_provider/standard_models/ameribor_rates.py
index 2ca783ab5cc..7a714642c9e 100644
--- a/openbb_sdk/sdk/provider/openbb_provider/standard_models/ameribor_rates.py
+++ b/openbb_sdk/sdk/provider/openbb_provider/standard_models/ameribor_rates.py
@@ -28,4 +28,4 @@ class AMERIBORData(Data):
     """Return Treasury Rates Data."""
 
     date: dateType = Field(description=DATA_DESCRIPTIONS.get("date", ""))
-    rate: float = Field(description="AMERIBOR rate.")
+    rate: Optional[float] = Field(description="AMERIBOR rate.")
diff --git a/openbb_sdk/sdk/provider/openbb_provider/standard_models/estr_rates.py b/openbb_sdk/sdk/provider/openbb_provider/standard_models/estr_rates.py
index da915ffe1d4..406810cb3d7 100644
--- a/openbb_sdk/sdk/provider/openbb_provider/standard_models/estr_rates.py
+++ b/openbb_sdk/sdk/provider/openbb_provider/standard_models/estr_rates.py
@@ -28,4 +28,4 @@ class ESTRData(Data):
     """Return Treasury Rates Data."""
 
     date: dateType = Field(description=DATA_DESCRIPTIONS.get("date", ""))
-    rate: float = Field(description="ESTR rate.")
+    rate: Optional[float] = Field(description="ESTR rate.")
diff --git a/openbb_sdk/sdk/provider/openbb_provider/standard_models/fed_rates.py b/openbb_sdk/sdk/provider/openbb_provider/standard_models/fed_rates.py
index afee46070e0..a8eeef75311 100644
--- a/openbb_sdk/sdk/provider/openbb_provider/standard_models/fed_rates.py
+++ b/openbb_sdk/sdk/provider/openbb_provider/standard_models/fed_rates.py
@@ -28,4 +28,4 @@ class FEDData(Data):
     """Return Treasury Rates Data."""
 
     date: dateType = Field(description=DATA_DESCRIPTIONS.get("date", ""))
-    rate: float = Field(description="FED rate.")
+    rate: Optional[float] = Field(description="FED rate.")
diff --git a/openbb_sdk/sdk/provider/openbb_provider/standard_models/iorb_rates.py b/openbb_sdk/sdk/provider/openbb_provider/standard_models/iorb_rates.py
index 316041f6824..695c63d8616 100644
--- a/openbb_sdk/sdk/provider/openbb_provider/standard_models/iorb_rates.py
+++ b/openbb_sdk/sdk/provider/openbb_provider/standard_models/iorb_rates.py
@@ -28,4 +28,4 @@ class IORBData(Data):
     """Return Treasury Rates Data."""
 
     date: dateType = Field(description=DATA_DESCRIPTIONS.get("date", ""))
-    rate: float = Field(description="IORB rate.")
+    rate: Optional[float] = Field(description="IORB rate.")
diff --git a/openbb_sdk/sdk/provider/openbb_provider/standard_models/sofr_rates.py b/openbb_sdk/sdk/provider/openbb_provider/standard_models/sofr_rates.py
index c2e73778222..f3027fb3e91 100644
--- a/openbb_sdk/sdk/provider/openbb_provider/standard_models/sofr_rates.py
+++ b/openbb_sdk/sdk/provider/openbb_provider/standard_models/sofr_rates.py
@@ -28,4 +28,4 @@ class SOFRData(Data):
     """Return Treasury Rates Data."""
 
     date: dateType = Field(description=DATA_DESCRIPTIONS.get("date", ""))
-    rate: float = Field(description="SOFR rate.")
+    rate: Optional[float] = Field(description="SOFR rate.")
diff --git a/openbb_sdk/sdk/provider/openbb_provider/standard_models/sonia_rates.py b/openbb_sdk/sdk/provider/openbb_provider/standard_models/sonia_rates.py
index 469f8cb3eb2..9bf05047806 100644
--- a/openbb_sdk/sdk/provider/openbb_provider/standard_models/sonia_rates.py
+++ b/openbb_sdk/sdk/provider/openbb_provider/standard_models/sonia_rates.py
@@ -28,4 +28,4 @@ class SONIAData(Data):
     """Return Treasury Rates Data."""
 
     date: dateType = Field(description=DATA_DESCRIPTIONS.get("date", ""))
-    rate: float = Field(description="SONIA rate.")
+    rate: Optional[float] = Field(description="SONIA rate.")

From 7d933e3f774832dcd896198a725f88568af9e08f Mon Sep 17 00:00:00 2001
From: Diogo Sousa <montezdesousa@gmail.com>
Date: Mon, 25 Sep 2023 14:31:25 +0100
Subject: [PATCH 2/4] fix stocks.fa.transcript and rebuild

---
 openbb_sdk/openbb/package/stocks_fa.py                   | 8 ++++----
 .../standard_models/earnings_call_transcript.py          | 9 ++++++---
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/openbb_sdk/openbb/package/stocks_fa.py b/openbb_sdk/openbb/package/stocks_fa.py
index 3478551ec72..b1d18dd62fc 100644
--- a/openbb_sdk/openbb/package/stocks_fa.py
+++ b/openbb_sdk/openbb/package/stocks_fa.py
@@ -855,7 +855,7 @@ class ROUTER_stocks_fa(Container):
             Name and position of the executive.
         year : Optional[int]
             Year of the compensation.
-        salary : Optional[PositiveFloat]
+        salary : Optional[NonNegativeFloat]
             Salary of the executive.
         bonus : Optional[NonNegativeFloat]
             Bonus of the executive.
@@ -865,7 +865,7 @@ class ROUTER_stocks_fa(Container):
             Incentive plan compensation of the executive.
         all_other_compensation : Optional[NonNegativeFloat]
             All other compensation of the executive.
-        total : Optional[PositiveFloat]
+        total : Optional[NonNegativeFloat]
             Total compensation of the executive.
         url : Optional[str]
             URL of the filing data."""  # noqa: E501
@@ -3205,7 +3205,7 @@ class ROUTER_stocks_fa(Container):
             OpenBBCustomParameter(description="Year of the earnings call transcript."),
         ],
         quarter: typing_extensions.Annotated[
-            Literal[1, 2, 3, 4],
+            int,
             OpenBBCustomParameter(
                 description="Quarter of the earnings call transcript."
             ),
@@ -3221,7 +3221,7 @@ class ROUTER_stocks_fa(Container):
             Symbol to get data for.
         year : int
             Year of the earnings call transcript.
-        quarter : Literal[1, 2, 3, 4]
+        quarter : int
             Quarter of the earnings call transcript.
         provider : Union[Literal['fmp'], None]
             The provider to use for the query, by default None.
diff --git a/openbb_sdk/sdk/provider/openbb_provider/standard_models/earnings_call_transcript.py b/openbb_sdk/sdk/provider/openbb_provider/standard_models/earnings_call_transcript.py
index cbc9098dc37..7f4d35455e5 100644
--- a/openbb_sdk/sdk/provider/openbb_provider/standard_models/earnings_call_transcript.py
+++ b/openbb_sdk/sdk/provider/openbb_provider/standard_models/earnings_call_transcript.py
@@ -2,7 +2,7 @@
 
 
 from datetime import datetime
-from typing import List, Literal, Set, Union
+from typing import List, Set, Union
 
 from pydantic import Field, validator
 
@@ -16,8 +16,11 @@ class EarningsCallTranscriptQueryParams(QueryParams):
 
     symbol: str = Field(description=QUERY_DESCRIPTIONS.get("symbol", ""))
     year: int = Field(description="Year of the earnings call transcript.")
-    quarter: Literal[1, 2, 3, 4] = Field(
-        default=1, description="Quarter of the earnings call transcript."
+    quarter: int = Field(
+        default=1,
+        gt=0,
+        lt=5,
+        description="Quarter of the earnings call transcript.",
     )
 
     @validator("symbol", pre=True, check_fields=False, always=True)

From d516315c625d797522e394e41506990b94793d54 Mon Sep 17 00:00:00 2001
From: Diogo Sousa <montezdesousa@gmail.com>
Date: Mon, 25 Sep 2023 15:11:28 +0100
Subject: [PATCH 3/4] fix projections command

---
 .../fred/openbb_fred/utils/fred_helpers.py        |  4 ++--
 .../standard_models/fed_projections.py            | 15 ++++++++-------
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/openbb_sdk/providers/fred/openbb_fred/utils/fred_helpers.py b/openbb_sdk/providers/fred/openbb_fred/utils/fred_helpers.py
index e746d01d97a..b5e8121a339 100644
--- a/openbb_sdk/providers/fred/openbb_fred/utils/fred_helpers.py
+++ b/openbb_sdk/providers/fred/openbb_fred/utils/fred_helpers.py
@@ -67,9 +67,9 @@ def process_projections(data: Dict) -> List[Dict]:
         for key, value in data.items():
             val = [item["value"] for item in value if item["date"] == date]
             if val:
-                entry[key] = float(val[0]) if val != "." else float("nan")
+                entry[key] = float(val[0]) if val[0] != "." else None
             else:
-                entry[key] = float("nan")
+                entry[key] = None
         ldata.append(entry)
 
     return ldata
diff --git a/openbb_sdk/sdk/provider/openbb_provider/standard_models/fed_projections.py b/openbb_sdk/sdk/provider/openbb_provider/standard_models/fed_projections.py
index 750e60a001a..e3896050742 100644
--- a/openbb_sdk/sdk/provider/openbb_provider/standard_models/fed_projections.py
+++ b/openbb_sdk/sdk/provider/openbb_provider/standard_models/fed_projections.py
@@ -2,6 +2,7 @@
 
 
 from datetime import date as dateType
+from typing import Optional
 
 from pydantic import Field
 
@@ -18,16 +19,16 @@ class PROJECTIONData(Data):
     """Return Treasury Rates Data."""
 
     date: dateType = Field(description=DATA_DESCRIPTIONS.get("date", ""))
-    range_high: float = Field(description="High projection of rates.")
-    central_tendency_high: float = Field(
+    range_high: Optional[float] = Field(description="High projection of rates.")
+    central_tendency_high: Optional[float] = Field(
         description="Central tendency of high projection of rates."
     )
-    median: float = Field(description="Median projection of rates.")
-    range_midpoint: float = Field(description="Midpoint projection of rates.")
-    central_tendency_midpoint: float = Field(
+    median: Optional[float] = Field(description="Median projection of rates.")
+    range_midpoint: Optional[float] = Field(description="Midpoint projection of rates.")
+    central_tendency_midpoint: Optional[float] = Field(
         description="Central tendency of midpoint projection of rates."
     )
-    range_low: float = Field(description="Low projection of rates.")
-    central_tendency_low: float = Field(
+    range_low: Optional[float] = Field(description="Low projection of rates.")
+    central_tendency_low: Optional[float] = Field(
         description="Central tendency of low projection of rates."
     )

From 65b32f2a50706921a8b227eb504d970ffc8902b9 Mon Sep 17 00:00:00 2001
From: Pratyush Shukla <ps4534@nyu.edu>
Date: Tue, 26 Sep 2023 00:07:27 +0530
Subject: [PATCH 4/4] yfianance `FuturesHistorical` fix date issue

---
 .../models/futures_historical.py              | 50 +++++++++++--------
 .../standard_models/futures_historical.py     |  6 +++
 2 files changed, 35 insertions(+), 21 deletions(-)

diff --git a/openbb_sdk/providers/yfinance/openbb_yfinance/models/futures_historical.py b/openbb_sdk/providers/yfinance/openbb_yfinance/models/futures_historical.py
index f0f497307bb..b7a704e2eff 100644
--- a/openbb_sdk/providers/yfinance/openbb_yfinance/models/futures_historical.py
+++ b/openbb_sdk/providers/yfinance/openbb_yfinance/models/futures_historical.py
@@ -2,7 +2,7 @@
 # ruff: noqa: SIM105
 
 
-from datetime import datetime
+from datetime import datetime, timedelta
 from typing import Any, Dict, List, Optional
 
 from dateutil.relativedelta import relativedelta
@@ -14,6 +14,7 @@ from openbb_provider.standard_models.futures_historical import (
 from openbb_provider.utils.descriptions import QUERY_DESCRIPTIONS
 from openbb_yfinance.utils.helpers import get_futures_data
 from openbb_yfinance.utils.references import INTERVALS, MONTHS, PERIODS
+from pandas import Timestamp, to_datetime
 from pydantic import Field, validator
 from yfinance import Ticker
 
@@ -40,10 +41,12 @@ class YFinanceFuturesHistoricalQueryParams(FuturesHistoricalQueryParams):
 class YFinanceFuturesHistoricalData(FuturesHistoricalData):
     """YFinance Futures End of Day Data."""
 
-    @validator("Date", pre=True, check_fields=False)
+    @validator("date", pre=True, check_fields=False)
     def date_validate(cls, v):  # pylint: disable=E0213
-        """Return datetime object from string."""
-        return datetime.strptime(v, "%Y-%m-%dT%H:%M:%S")
+        """Return formatted datetime."""
+        if isinstance(v, Timestamp):
+            return v.to_pydatetime()
+        return v
 
 
 class YFinanceFuturesHistoricalFetcher(
@@ -63,23 +66,9 @@ class YFinanceFuturesHistoricalFetcher(
             now = datetime.now().date()
             if params.get("start_date") is None:
                 transformed_params["start_date"] = now - relativedelta(years=1)
-            else:
-                try:
-                    transformed_params["start_date"] = datetime.strptime(
-                        params["start_date"], "%Y-%m-%d"
-                    ).date()
-                except TypeError:
-                    pass
 
             if params.get("end_date") is None:
                 transformed_params["end_date"] = now
-            else:
-                try:
-                    transformed_params["end_date"] = datetime.strptime(
-                        params["end_date"], "%Y-%m-%d"
-                    ).date()
-                except TypeError:
-                    pass
 
         return YFinanceFuturesHistoricalQueryParams(**transformed_params)
 
@@ -124,10 +113,29 @@ class YFinanceFuturesHistoricalFetcher(
                 raise_errors=True,
             )
 
-        data = data.reset_index()
-        data["Date"] = (
-            data["Date"].dt.tz_localize(None).dt.strftime("%Y-%m-%dT%H:%M:%S")
+        query.end_date = (
+            datetime.now().date() if query.end_date is None else query.end_date
         )
+        days = (
+            1
+            if query.interval in ["1m", "2m", "5m", "15m", "30m", "60m", "1h", "90m"]
+            else 0
+        )
+        if query.start_date:
+            data.set_index("date", inplace=True)
+            data.index = to_datetime(data.index)
+
+            start_date_dt = datetime.combine(query.start_date, datetime.min.time())
+            end_date_dt = datetime.combine(query.end_date, datetime.min.time())
+
+            data = data[
+                (data.index >= start_date_dt + timedelta(days=days))
+                & (data.index <= end_date_dt)
+            ]
+
+        data.reset_index(inplace=True)
+        data.rename(columns={"index": "date"}, inplace=True)
+
         return data.to_dict("records")
 
     @staticmethod
diff --git a/openbb_sdk/sdk/provider/openbb_provider/standard_models/futures_historical.py b/openbb_sdk/sdk/provider/openbb_provider/standard_models/futures_historical.py
index 7c75f082c89..db21baf4af8 100644
--- a/openbb_sdk/sdk/provider/openbb_provider/standard_models/futures_historical.py
+++ b/openbb_sdk/sdk/provider/openbb_provider/standard_models/futures_historical.py
@@ -4,6 +4,7 @@
 from datetime import date, datetime
 from typing import List, Optional, Set, Union
 
+from dateutil import parser
 from pydantic import Field, validator
 
 from openbb_provider.abstract.data import Data
@@ -45,3 +46,8 @@ class FuturesHistoricalData(Data):
     low: float = Field(description=DATA_DESCRIPTIONS.get("low", ""))
     close: float = Field(description=DATA_DESCRIPTIONS.get("close", ""))
     volume: float = Field(description=DATA_DESCRIPTIONS.get("volume", ""))
+
+    @validator("date", pre=True, check_fields=False)
+    def date_validate(cls, v):  # pylint: disable=E0213
+        """Return formatted datetime."""
+        return parser.isoparse(str(v))