Fix Stocks Load (#4579)

* fix load * Standardise yfinance, fix EODHD * ruff * YF automatic date adjustments * Add TODO comment * Fix date message * Fix AV date message * merge conflicts * Update stocks_helper.py * tests recorder --------- Co-authored-by: Danglewood <85772166+deeleeramone@users.noreply.github.com> Co-authored-by: teh_coderer <me@tehcoderer.com>
2026-10-28 04:52:07 +00:00 · 2023-03-30 21:26:34 +02:00
parent 1f6ecc34be
commit 2b4cb0b02e
9 changed files with 2734 additions and 6004 deletions
--- a/openbb_terminal/stocks/stocks_helper.py
+++ b/openbb_terminal/stocks/stocks_helper.py
@@ -250,7 +250,7 @@ def search(
    return df


-def load(
+def load(  # pylint: disable=too-many-return-statements
    symbol: str,
    start_date: Optional[Union[datetime, str]] = None,
    interval: int = 1440,
@@ -333,7 +333,9 @@ def load(
    # Daily
    if int(interval) == 1440:
        if source == "AlphaVantage":
-            df_stock_candidate = load_stock_av(symbol, int_string, start_date, end_date)
+            df_stock_candidate: pd.DataFrame = load_stock_av(
+                symbol, int_string, start_date, end_date
+            )

        elif source == "YahooFinance":
            df_stock_candidate = load_stock_yf(
@@ -341,9 +343,19 @@ def load(
            )

        elif source == "EODHD":
-            df_stock_candidate = load_stock_eodhd(
-                symbol, start_date, end_date, weekly, monthly
-            )
+            try:
+                df_stock_candidate = load_stock_eodhd(
+                    symbol,
+                    start_date,
+                    end_date,
+                    weekly,
+                    monthly,
+                )
+            except KeyError:
+                console.print(
+                    "[red]Invalid symbol for EODHD. Please check your subscription.[/red]\n"
+                )
+                return pd.DataFrame()

        elif source == "Polygon":
            df_stock_candidate = load_stock_polygon(
@@ -376,27 +388,48 @@ def load(
            s_start = start_date
            int_string = "Minute"
            s_interval = f"{interval}min"
+            if end_date:
+                end_date = (end_date + timedelta(days=1)).strftime("%Y-%m-%d")
+                end_date = datetime.strptime(end_date, "%Y-%m-%d")
            df_stock_candidate = load_stock_av(
                symbol, int_string, start_date, end_date, s_interval
            )
+            s_start = df_stock_candidate.index[0]

        elif source == "YahooFinance":
            s_int = str(interval) + "m"
            s_interval = s_int + "in"
-            d_granularity = {"1m": 6, "5m": 59, "15m": 59, "30m": 59, "60m": 729}

-            s_start_dt = datetime.utcnow() - timedelta(days=d_granularity[s_int])
-            s_date_start = s_start_dt.strftime("%Y-%m-%d")
+            # add 1 day to end_date to include the last day
+            if end_date:
+                end_date = (end_date + timedelta(days=1)).strftime("%Y-%m-%d")
+            else:
+                end_date = (datetime.now() + timedelta(days=1)).strftime("%Y-%m-%d")

            df_stock_candidate = yf.download(
                symbol,
-                start=s_date_start
-                if s_start_dt > start_date
-                else start_date.strftime("%Y-%m-%d"),
+                start=start_date.strftime("%Y-%m-%d"),
+                end=end_date,
                progress=False,
                interval=s_int,
                prepost=prepost,
+                show_errors=False,
            )
+            # Handle the case when start and end dates aren't explicitly set
+            # TODO: This is a temporary fix, need to find a better way to handle this
+            if df_stock_candidate.empty:
+                d_granularity = {"1m": 6, "5m": 59, "15m": 59, "30m": 59, "60m": 729}
+                s_start_dt = datetime.utcnow() - timedelta(days=d_granularity[s_int])
+                s_date_start = s_start_dt.strftime("%Y-%m-%d")
+                df_stock_candidate = yf.download(
+                    symbol,
+                    start=s_date_start
+                    if s_start_dt > start_date
+                    else start_date.strftime("%Y-%m-%d"),
+                    progress=False,
+                    interval=s_int,
+                    prepost=prepost,
+                )

            # Check that loading a stock was not successful
            if df_stock_candidate.empty:
@@ -406,6 +439,8 @@ def load(
                df_stock_candidate.index
            ).tz_localize(None)

+            s_start_dt = df_stock_candidate.index[0]
+
            s_start = (
                pytz.utc.localize(s_start_dt) if s_start_dt > start_date else start_date
            )
@@ -472,6 +507,30 @@ def load(
            )
            s_interval = f"{interval}min"

+        elif source == "EODHD":
+            df_stock_candidate = load_stock_eodhd(
+                symbol, start_date, end_date, weekly, monthly, intraday=True
+            )
+
+            if df_stock_candidate.empty:
+                return pd.DataFrame()
+
+            df_stock_candidate.index = df_stock_candidate.index.tz_convert(
+                "US/Eastern"
+            ).tz_localize(None)
+
+            s_start_dt = df_stock_candidate.index[0]
+
+            s_start = (
+                pytz.utc.localize(s_start_dt) if s_start_dt > start_date else start_date
+            )
+
+            s_interval = f"{interval}min"
+
+        else:
+            console.print("[red]Invalid intraday data source[/red]")
+            return pd.DataFrame()
+
        if not prepost:
            df_stock_candidate = df_stock_candidate.between_time("9:30", "16:00")

--- a/openbb_terminal/stocks/stocks_model.py
+++ b/openbb_terminal/stocks/stocks_model.py
@@ -73,7 +73,7 @@ def load_stock_av(
        )
        if interval == "Minute":
            df_stock_candidate: pd.DataFrame = ts.get_intraday(
-                symbol=symbol, interval=interval_min
+                symbol=symbol, interval=interval_min, outputsize="full"
            )[0]
        elif interval == "Daily":
            df_stock_candidate = ts.get_daily_adjusted(
@@ -102,15 +102,15 @@ def load_stock_av(
        console.print("No data found.")
        return pd.DataFrame()

-    df_stock_candidate.index = df_stock_candidate.index.tz_localize(None)
-
    df_stock_candidate.sort_index(ascending=True, inplace=True)

-    # Slice dataframe from the starting date YYYY-MM-DD selected
+    df_stock_candidate.index = df_stock_candidate.index.tz_localize(None)
+
    df_stock_candidate = df_stock_candidate[
-        (df_stock_candidate.index >= start_date.strftime("%Y-%m-%d"))
-        & (df_stock_candidate.index <= end_date.strftime("%Y-%m-%d"))
+        (df_stock_candidate.index >= start_date)
+        & (df_stock_candidate.index <= end_date)
    ]
+
    return df_stock_candidate


@@ -133,6 +133,9 @@ def load_stock_yf(
            1970, 1, 2
        )  # 1 day buffer in case of timezone adjustments

+    # add 1 day to end_date to include the last day
+    end_date = end_date + pd.Timedelta(days=1)
+
    # Adding a dropna for weekly and monthly because these include weird NaN columns.
    df_stock_candidate = yf.download(
        symbol,
@@ -167,18 +170,28 @@ def load_stock_yf(


 def load_stock_eodhd(
-    symbol: str, start_date: datetime, end_date: datetime, weekly: bool, monthly: bool
+    symbol: str,
+    start_date: datetime,
+    end_date: datetime,
+    weekly: bool,
+    monthly: bool,
+    intraday: bool = False,
 ) -> pd.DataFrame:
+    request_url = "https://eodhistoricaldata.com/api/eod/"
+
    int_ = "d"
    if weekly:
        int_ = "w"
    elif monthly:
        int_ = "m"
+    elif intraday:
+        int_ = "1m"
+        request_url = "https://eodhistoricaldata.com/api/intraday/"

    request_url = (
-        f"https://eodhistoricaldata.com/api/eod/"
+        f"{request_url}"
        f"{symbol.upper()}?"
-        f"{start_date.strftime('%Y-%m-%d')}&"
+        f"from={start_date.strftime('%Y-%m-%d')}&"
        f"to={end_date.strftime('%Y-%m-%d')}&"
        f"period={int_}&"
        f"api_token={get_current_user().credentials.API_EODHD_KEY}&"
--- a/tests/openbb_terminal/stocks/cassettes/test_stocks_helper/test_load[1440-AlphaVantage].yaml
+++ b/tests/openbb_terminal/stocks/cassettes/test_stocks_helper/test_load[1440-AlphaVantage].yaml
--- a/tests/openbb_terminal/stocks/cassettes/test_stocks_helper/test_load[1440-Intrinio].yaml
+++ b/tests/openbb_terminal/stocks/cassettes/test_stocks_helper/test_load[1440-Intrinio].yaml
@@ -22,7 +22,7 @@ interactions:
      Content-Type:
      - application/json
      Date:
-      - Sat, 25 Feb 2023 23:29:23 GMT
+      - Mon, 27 Mar 2023 20:16:56 GMT
      Vary:
      - Origin,Accept-Encoding
    status:
--- a/tests/openbb_terminal/stocks/cassettes/test_stocks_helper/test_load[1440-YahooFinance].yaml
+++ b/tests/openbb_terminal/stocks/cassettes/test_stocks_helper/test_load[1440-YahooFinance].yaml
@@ -15,7 +15,7 @@ interactions:
    uri: https://query2.finance.yahoo.com/v8/finance/chart/GME?interval=1d&range=1d
  response:
    body:
-      string: '{"chart":{"result":[{"meta":{"currency":"USD","symbol":"GME","exchangeName":"NYQ","instrumentType":"EQUITY","firstTradeDate":1013610600,"regularMarketTime":1678222802,"gmtoffset":-18000,"timezone":"EST","exchangeTimezoneName":"America/New_York","regularMarketPrice":18.06,"chartPreviousClose":18.65,"priceHint":2,"currentTradingPeriod":{"pre":{"timezone":"EST","start":1678266000,"end":1678285800,"gmtoffset":-18000},"regular":{"timezone":"EST","start":1678285800,"end":1678309200,"gmtoffset":-18000},"post":{"timezone":"EST","start":1678309200,"end":1678323600,"gmtoffset":-18000}},"dataGranularity":"1d","range":"1d","validRanges":["1d","5d","1mo","3mo","6mo","1y","2y","5y","10y","ytd","max"]},"timestamp":[1678199400],"indicators":{"quote":[{"volume":[2368700],"high":[18.84000015258789],"low":[18.059999465942383],"open":[18.540000915527344],"close":[18.059999465942383]}],"adjclose":[{"adjclose":[18.059999465942383]}]}}],"error":null}}'
+      string: '{"chart":{"result":[{"meta":{"currency":"USD","symbol":"GME","exchangeName":"NYQ","instrumentType":"EQUITY","firstTradeDate":1013610600,"regularMarketTime":1679947202,"gmtoffset":-14400,"timezone":"EDT","exchangeTimezoneName":"America/New_York","regularMarketPrice":23.0,"chartPreviousClose":23.98,"priceHint":2,"currentTradingPeriod":{"pre":{"timezone":"EDT","end":1679923800,"start":1679904000,"gmtoffset":-14400},"regular":{"timezone":"EDT","end":1679947200,"start":1679923800,"gmtoffset":-14400},"post":{"timezone":"EDT","end":1679961600,"start":1679947200,"gmtoffset":-14400}},"dataGranularity":"1d","range":"1d","validRanges":["1d","5d","1mo","3mo","6mo","1y","2y","5y","10y","ytd","max"]},"timestamp":[1679947202],"indicators":{"quote":[{"volume":[4542129],"low":[22.700000762939453],"open":[23.989999771118164],"close":[23.0],"high":[24.059999465942383]}],"adjclose":[{"adjclose":[23.0]}]}}],"error":null}}'
    headers:
      Age:
      - '0'
@@ -36,11 +36,11 @@ interactions:
      cache-control:
      - public, max-age=10, stale-while-revalidate=20
      content-length:
-      - '942'
+      - '914'
      content-type:
      - application/json;charset=utf-8
      date:
-      - Wed, 08 Mar 2023 14:01:51 GMT
+      - Mon, 27 Mar 2023 20:16:56 GMT
      server:
      - ATS
      vary:
@@ -48,13 +48,13 @@ interactions:
      x-envoy-decorator-operation:
      - finance-chart-api--mtls-production-ir2.finance-k8s.svc.yahoo.local:4080/*
      x-envoy-upstream-service-time:
-      - '10'
+      - '15'
      x-request-id:
-      - 70ce00b1-ce30-4ddc-8b80-5ebd018835df
+      - c8ecec37-bcdb-4cde-bf4a-6ddb76a823d5
      x-yahoo-request-id:
-      - aflvfchi0h5af
+      - 46eh975i23udo
      y-rid:
-      - aflvfchi0h5af
+      - 46eh975i23udo
    status:
      code: 200
      message: OK
@@ -74,7 +74,7 @@ interactions:
    uri: https://query2.finance.yahoo.com/v8/finance/chart/GME?events=div%2Csplits%2CcapitalGains&includePrePost=False&interval=1d&period1=MOCK_PERIOD_1&period2=MOCK_PERIOD_2
  response:
    body:
-      string: '{"chart":{"result":[{"meta":{"currency":"USD","symbol":"GME","exchangeName":"NYQ","instrumentType":"EQUITY","firstTradeDate":1013610600,"regularMarketTime":1678222802,"gmtoffset":-18000,"timezone":"EST","exchangeTimezoneName":"America/New_York","regularMarketPrice":18.06,"chartPreviousClose":49.053,"priceHint":2,"currentTradingPeriod":{"pre":{"timezone":"EST","start":1678266000,"end":1678285800,"gmtoffset":-18000},"regular":{"timezone":"EST","start":1678285800,"end":1678309200,"gmtoffset":-18000},"post":{"timezone":"EST","start":1678309200,"end":1678323600,"gmtoffset":-18000}},"dataGranularity":"1d","range":"","validRanges":["1d","5d","1mo","3mo","6mo","1y","2y","5y","10y","ytd","max"]},"timestamp":[1638369000],"indicators":{"quote":[{"high":[50.02750015258789],"volume":[9406000],"close":[44.959999084472656],"open":[49.849998474121094],"low":[44.252498626708984]}],"adjclose":[{"adjclose":[44.959999084472656]}]}}],"error":null}}'
+      string: '{"chart":{"result":[{"meta":{"currency":"USD","symbol":"GME","exchangeName":"NYQ","instrumentType":"EQUITY","firstTradeDate":1013610600,"regularMarketTime":1679947202,"gmtoffset":-14400,"timezone":"EDT","exchangeTimezoneName":"America/New_York","regularMarketPrice":23.0,"chartPreviousClose":49.053,"priceHint":2,"currentTradingPeriod":{"pre":{"timezone":"EDT","end":1679923800,"start":1679904000,"gmtoffset":-14400},"regular":{"timezone":"EDT","end":1679947200,"start":1679923800,"gmtoffset":-14400},"post":{"timezone":"EDT","end":1679961600,"start":1679947200,"gmtoffset":-14400}},"dataGranularity":"1d","range":"","validRanges":["1d","5d","1mo","3mo","6mo","1y","2y","5y","10y","ytd","max"]},"timestamp":[1638369000,1638455400],"indicators":{"quote":[{"volume":[9406000,8348400],"low":[44.252498626708984,43.26750183105469],"open":[49.849998474121094,46.25],"close":[44.959999084472656,45.38999938964844],"high":[50.02750015258789,46.85749816894531]}],"adjclose":[{"adjclose":[44.959999084472656,45.38999938964844]}]}}],"error":null}}'
    headers:
      Age:
      - '0'
@@ -95,25 +95,25 @@ interactions:
      cache-control:
      - public, max-age=10, stale-while-revalidate=20
      content-length:
-      - '941'
+      - '1037'
      content-type:
      - application/json;charset=utf-8
      date:
-      - Wed, 08 Mar 2023 14:01:51 GMT
+      - Mon, 27 Mar 2023 20:16:56 GMT
      server:
      - ATS
      vary:
      - Origin,Accept-Encoding
      x-envoy-decorator-operation:
-      - finance-chart-api--mtls-canary-production-ir2.finance-k8s.svc.yahoo.local:4080/*
+      - finance-chart-api--mtls-production-ir2.finance-k8s.svc.yahoo.local:4080/*
      x-envoy-upstream-service-time:
      - '14'
      x-request-id:
-      - 450be85f-39ed-408d-8861-4c6763df0d8e
+      - 3b7f7f5e-24c0-46f9-9929-b68e9514954b
      x-yahoo-request-id:
-      - 4ird1b9i0h5af
+      - 7dkipili23udo
      y-rid:
-      - 4ird1b9i0h5af
+      - 7dkipili23udo
    status:
      code: 200
      message: OK
--- a/tests/openbb_terminal/stocks/cassettes/test_stocks_helper/test_load[60-YahooFinance].yaml
+++ b/tests/openbb_terminal/stocks/cassettes/test_stocks_helper/test_load[60-YahooFinance].yaml
--- a/tests/openbb_terminal/stocks/csv/test_stocks_helper/test_load[1440-YahooFinance].csv
+++ b/tests/openbb_terminal/stocks/csv/test_stocks_helper/test_load[1440-YahooFinance].csv
@@ -1,2 +1,3 @@
 date,Open,High,Low,Close,Adj Close,Volume,Dividends,Stock Splits
 2021-12-01,49.849998474121094,50.02750015258789,44.252498626708984,44.959999084472656,44.959999084472656,9406000,0.0,0.0
+2021-12-02,46.25,46.85749816894531,43.26750183105469,45.38999938964844,45.38999938964844,8348400,0.0,0.0
--- a/tests/openbb_terminal/stocks/csv/test_stocks_helper/test_load[60-YahooFinance].csv
+++ b/tests/openbb_terminal/stocks/csv/test_stocks_helper/test_load[60-YahooFinance].csv
--- a/tests/openbb_terminal/stocks/test_stocks_helper.py
+++ b/tests/openbb_terminal/stocks/test_stocks_helper.py
@@ -7,10 +7,7 @@ from datetime import datetime
 import pytest

 # IMPORTATION INTERNAL
-from openbb_terminal.core.session.current_user import (
-    PreferencesModel,
-    copy_user,
-)
+from openbb_terminal.core.session.current_user import PreferencesModel, copy_user

 # IMPORTATION INTERNAL
 from openbb_terminal.stocks import stocks_helper, stocks_view