Fix for DCF glitch (#1932)

* Added fixes

* Added fixes

Co-authored-by: didierlopes.eth <dro.lopes@campus.fct.unl.pt>
Co-authored-by: Theodore Aptekarev <aptekarev@gmail.com>

bots/requirements.txt

@@ -86,6 +86,7 @@ grpcio==1.46.3; python_full_version >= "3.7.1" and python_full_version < "4.0.0"
 h11==0.13.0; python_version >= "3.7" and python_version < "4.0" and python_full_version >= "3.7.0"
 hijri-converter==2.2.4; python_version >= "3.6"
 holidays==0.11.3.1; python_version >= "3.6"
+html5lib==1.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
 identify==2.5.1; python_version >= "3.7"
 idna==3.3; python_full_version >= "3.8.0" and python_version >= "3.8" and python_version < "4" and python_full_version < "4.0.0" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0") and (python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.7") and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6")
 imagesize==1.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
@@ -312,7 +313,7 @@ vcrpy==4.1.1; python_version >= "3.5"
 virtualenv==20.14.1; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.7"
 voila==0.3.5; python_version >= "3.7"
 wcwidth==0.2.5; python_full_version >= "3.6.2" and python_version >= "3.8"
-webencodings==0.5.1; python_version >= "3.7"
+webencodings==0.5.1; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.7"
 websocket-client==1.3.2; python_version >= "3.8" and python_version < "4.0"
 websockets==10.3; python_version >= "3.7"
 widgetsnbextension==3.6.0

openbb_terminal/stocks/fundamental_analysis/dcf_model.py

@@ -3,7 +3,6 @@ __docformat__ = "numpy"
 
 import logging
 import os
-import re
 from io import BytesIO
 from pathlib import Path
 from typing import Any, Dict, List, Tuple, Union
@@ -408,22 +407,14 @@ def create_dataframe(ticker: str, statement: str, period: str = "annual"):
 
     if "404 - Page Not Found" in r.text:
         return pd.DataFrame(), None, None
+
+    try:
+        df = pd.read_html(r.text)[0]
+    except ValueError:
+        return pd.DataFrame(), None, None
+
     soup = BeautifulSoup(r.content, "html.parser")
-
-    table = soup.find("table", attrs={"class": re.compile("fintbl")})
-    if table is None:
-        return pd.DataFrame(), None, None
-    head = table.find("thead")
-    if head is None:
-        return pd.DataFrame(), None, None
-    columns = head.find_all("th")
-    if columns is None:
-        return pd.DataFrame(), None, None
-
-    years = [x.get_text().strip() for x in columns]
-    len_data = len(years) - 1
-
-    phrase = soup.find("div", attrs={"class": "block text-sm text-gray-600 lg:hidden"})
+    phrase = soup.find("div", attrs={"class": "info-long svelte-f7kao3"})
     phrase = phrase.get_text().lower() if phrase else ""
 
     if "thousand" in phrase:
@@ -440,23 +431,7 @@ def create_dataframe(ticker: str, statement: str, period: str = "annual"):
             statement_currency = currency
             break
 
-    body = table.find("tbody")
-    rows = body.find_all("tr")
-
-    all_data = [
-        [
-            x.get_text().strip() if x.get_text().strip() != "-" else "0"
-            for x in y.find_all("td")
-        ]
-        for y in rows
-    ]
-
-    df = pd.DataFrame(data=all_data)
-    df = df.set_index(0)
-    n = df.shape[1] - len_data
-    if n > 0:
-        df = df.iloc[:, :-n]
-    df.columns = years[1 : len(df.columns) + 1]  # noqa: E203
+    df = df.set_index("Year")
     df = df.loc[:, ~(df == "Upgrade").any()]
 
     for ignore in ignores:
@@ -556,10 +531,16 @@ def get_value(df: pd.DataFrame, row: str, column: int) -> Tuple[float, float]:
     value : List[float]
         The information in float format
     """
-    val1: str = df.at[row, df.columns[column]]
-    fin_val1: float = float(val1.replace(",", "").replace("-", "-0"))
-    val2: str = df.at[row, df.columns[column + 1]]
-    fin_val2: float = float(val2.replace(",", "").replace("-", "-0"))
+    val1 = df.at[row, df.columns[column]]
+    if isinstance(val1, str):
+        fin_val1: float = float(val1.replace(",", "").replace("-", "-0"))
+    else:
+        fin_val1 = float(val1)
+    val2 = df.at[row, df.columns[column + 1]]
+    if isinstance(val2, str):
+        fin_val2: float = float(val2.replace(",", "").replace("-", "-0"))
+    else:
+        fin_val2 = float(val2)
     return fin_val1, fin_val2
 
 

openbb_terminal/stocks/fundamental_analysis/dcf_static.py

@@ -32,6 +32,7 @@ non_gaap_is = [
     "Operating Expenses",
     "Pretax Income",
     "Revenue Growth (YoY)",
+    "Depreciation & Amortization",
 ]
 gaap_is = [
     "Revenue",

poetry.lock

@@ -1204,6 +1204,24 @@ hijri-converter = "*"
 korean-lunar-calendar = "*"
 python-dateutil = "*"
 
+[[package]]
+name = "html5lib"
+version = "1.1"
+description = "HTML parser based on the WHATWG HTML specification"
+category = "main"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
+
+[package.dependencies]
+six = ">=1.9"
+webencodings = "*"
+
+[package.extras]
+all = ["genshi", "chardet (>=2.2)", "lxml"]
+chardet = ["chardet (>=2.2)"]
+genshi = ["genshi"]
+lxml = ["lxml"]
+
 [[package]]
 name = "identify"
 version = "2.5.1"
@@ -4572,7 +4590,7 @@ prediction = ["tensorflow"]
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.8,<3.10"
-content-hash = "6f1fdd3e76f3826d325ccf5f2e0c2b5bc0ad537038784096dab0018af86a4196"
+content-hash = "d6fa69aa88f955931a3c20e39fc70c64ec3008721e46de9f485c25157e4b20a6"
 
 [metadata.files]
 absl-py = [
@@ -5463,6 +5481,10 @@ holidays = [
     {file = "holidays-0.11.3.1-py3-none-any.whl", hash = "sha256:63e2578513d0877aefd43d49870caf666fd902ec3a5884a3397f403c912a8246"},
     {file = "holidays-0.11.3.1.tar.gz", hash = "sha256:4855afe0ebf428efbcf848477828b889f8515be7f4f15ae26682919369d92774"},
 ]
+html5lib = [
+    {file = "html5lib-1.1-py2.py3-none-any.whl", hash = "sha256:0d78f8fde1c230e99fe37986a60526d7049ed4bf8a9fadbad5f00e22e58e041d"},
+    {file = "html5lib-1.1.tar.gz", hash = "sha256:b2e5b40261e20f354d198eae92afc10d750afb487ed5e50f9c4eaf07c184146f"},
+]
 identify = [
     {file = "identify-2.5.1-py2.py3-none-any.whl", hash = "sha256:0dca2ea3e4381c435ef9c33ba100a78a9b40c0bab11189c7cf121f75815efeaa"},
     {file = "identify-2.5.1.tar.gz", hash = "sha256:3d11b16f3fe19f52039fb7e39c9c884b21cb1b586988114fbe42671f03de3e82"},

pyproject.toml

@@ -103,6 +103,7 @@ jupyterlab-code-formatter = {version = "^1.4.10", optional = true}
 jupyterlab-lsp = {version = "^3.10.1", optional = true}
 jedi-language-server = {version = "^0.35.1", optional = true}
 kaleido = {version = "<=0.2.1", optional = true}
+html5lib = "^1.1"
 
 
 [tool.poetry.dev-dependencies]

requirements-full.txt

@@ -90,6 +90,7 @@ h11==0.13.0; python_version >= "3.7" and python_version < "4.0" and python_full_
 h5py==3.7.0; python_version >= "3.7"
 hijri-converter==2.2.4; python_version >= "3.6"
 holidays==0.11.3.1; python_version >= "3.6"
+html5lib==1.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
 identify==2.5.1; python_version >= "3.7"
 idna==3.3; python_full_version >= "3.7.1" and python_version >= "3.8" and python_version < "4" and python_full_version < "4.0.0" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0") and (python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.7") and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6")
 imagesize==1.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
@@ -318,7 +319,7 @@ vcrpy==4.1.1; python_version >= "3.5"
 virtualenv==20.14.1; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.7"
 voila==0.3.5; python_version >= "3.7"
 wcwidth==0.2.5; python_full_version >= "3.6.2" and python_version >= "3.8"
-webencodings==0.5.1; python_version >= "3.7"
+webencodings==0.5.1; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.7"
 websocket-client==1.3.2; python_version >= "3.8" and python_version < "4.0"
 websockets==10.3; python_version >= "3.7"
 werkzeug==2.1.2; python_version >= "3.7"

requirements.txt

@@ -69,6 +69,7 @@ grpcio==1.46.3; python_full_version >= "3.7.1" and python_full_version < "4.0.0"
 h11==0.13.0; python_version >= "3.7" and python_version < "4.0" and python_full_version >= "3.7.0"
 hijri-converter==2.2.4; python_version >= "3.6"
 holidays==0.11.3.1; python_version >= "3.6"
+html5lib==1.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
 idna==3.3; python_full_version >= "3.7.1" and python_version >= "3.8" and python_version < "4" and python_full_version < "4.0.0" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0") and (python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.7")
 importlib-metadata==4.11.4; python_version < "3.10" and python_version >= "3.7"
 inflection==0.5.1; python_version >= "3.6"
@@ -240,7 +241,7 @@ vadersentiment==3.3.2
 valinvest==0.0.2; python_version >= "3.6"
 voila==0.3.5; python_version >= "3.7"
 wcwidth==0.2.5; python_full_version >= "3.6.2" and python_version >= "3.8"
-webencodings==0.5.1; python_version >= "3.7"
+webencodings==0.5.1; python_version >= "3.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.7"
 websocket-client==1.3.2; python_version >= "3.8" and python_version < "4.0"
 websockets==10.3; python_version >= "3.7"
 widgetsnbextension==3.6.0