Hotfix/3596 (#4548)

* Step one: error handle Option class

* Step one: error handle Option class

* Got upcoming working

* Refactored before bug deep dive

* Refactored before bug deep dive

* cleaned

* cleaned

openbb_terminal/common/behavioural_analysis/reddit_helpers.py

@@ -1,10 +1,27 @@
 """Reddit Helpers."""
 __docformat__ = "numpy"
 
+import logging
 import re
+import urllib.request
 from typing import List
 
+import pandas as pd
 import praw
+from prawcore.exceptions import ResponseException
+
+from openbb_terminal.core.models import UserModel
+from openbb_terminal.rich_config import console
+
+logger = logging.getLogger(__name__)
+
+reddit_requirements = [
+    "API_REDDIT_CLIENT_ID",
+    "API_REDDIT_CLIENT_SECRET",
+    "API_REDDIT_USERNAME",
+    "API_REDDIT_USER_AGENT",
+    "API_REDDIT_PASSWORD",
+]
 
 
 def find_tickers(submission: praw.models.reddit.submission.Submission) -> List[str]:
@@ -31,3 +48,124 @@ def find_tickers(submission: praw.models.reddit.submission.Submission) -> List[s
             l_tickers_found.append(s_ticker.strip())
 
     return l_tickers_found
+
+
+def get_praw_api(current_user: UserModel) -> praw.Reddit:
+    """Get praw api.
+
+    Parameters
+    ----------
+    current_user : UserModel
+        The user model
+
+    Returns
+    -------
+    praw.Reddit
+        Praw api
+    """
+    # See https://github.com/praw-dev/praw/issues/1016 regarding praw arguments
+    praw_api = praw.Reddit(
+        client_id=current_user.credentials.API_REDDIT_CLIENT_ID,
+        client_secret=current_user.credentials.API_REDDIT_CLIENT_SECRET,
+        username=current_user.credentials.API_REDDIT_USERNAME,
+        user_agent=current_user.credentials.API_REDDIT_USER_AGENT,
+        password=current_user.credentials.API_REDDIT_PASSWORD,
+        check_for_updates=False,
+        comment_kind="t1",
+        message_kind="t4",
+        redditor_kind="t2",
+        submission_kind="t3",
+        subreddit_kind="t5",
+        trophy_kind="t6",
+        oauth_url="https://oauth.reddit.com",
+        reddit_url="https://www.reddit.com",
+        short_url="https://redd.it",
+        ratelimit_seconds=5,
+        timeout=16,
+    )
+    return praw_api
+
+
+def has_author(submission_, authors) -> bool:
+    """Check if submission has author."""
+    return (
+        hasattr(submission_, "author")
+        and hasattr(submission_.author, "name")
+        and submission_.author.name not in authors
+    )
+
+
+def has_content(submission_) -> bool:
+    """Check if submission has text or title."""
+    return hasattr(submission_, "selftext") or hasattr(submission_, "title")
+
+
+class RedditResponses:
+    def __init__(self):
+        self.tickers: dict = {}
+        self.authors: List[str] = []
+
+    def gather(self, praw_api, responses) -> int:
+        count = 0
+        for submission in responses:
+            try:
+                # Get more information about post using PRAW api
+                submission_ = praw_api.submission(id=submission["id"])
+
+                # Ensure that the post hasn't been removed by moderator in the meanwhile,
+                # that there is a description and it's not just an image, that the flair is
+                # meaningful, and that we aren't re-considering same author's content
+
+                if (
+                    submission_ is not None
+                    and not submission_.removed_by_category
+                    and has_content(submission_)
+                    and has_author(submission_, self.authors)
+                ):
+                    l_tickers_found = find_tickers(submission_)
+
+                    if l_tickers_found:
+                        count += len(l_tickers_found)
+
+                        # Add another author's name to the parsed watchlists
+                        self.authors.append(submission_.author.name)
+
+                        # Lookup stock tickers within a watchlist
+                        for key in l_tickers_found:
+                            if key in self.tickers:
+                                # Increment stock ticker found
+                                self.tickers[key] += 1
+                            else:
+                                # Initialize stock ticker found
+                                self.tickers[key] = 1
+
+            except ResponseException as e:
+                logger.exception("Invalid response: %s", str(e))
+
+                if "received 401 HTTP response" in str(e):
+                    console.print("[red]Invalid API Key[/red]\n")
+                else:
+                    console.print(f"[red]Invalid response: {str(e)}[/red]\n")
+                return 0
+        return count
+
+    def validate(
+        self,
+        url: str = "https://raw.githubusercontent.com/rreichel3/US-Stock-Symbols/main/all/all_tickers.txt",
+    ) -> None:
+        """Validate tickers."""
+        valids = []
+        with urllib.request.urlopen(url) as file:
+            for line in file:
+                new_item = line.decode("utf-8").replace("\n", "").strip()
+                valids.append(new_item)
+
+        for key in list(self.tickers.keys()):
+            if key not in valids:
+                self.tickers.pop(key)
+
+    def to_df(self) -> pd.DataFrame:
+        popularity = pd.DataFrame.from_dict(self.tickers, orient="index")
+        popularity.columns = ["Mentions"]
+        popularity = popularity.sort_values(by=["Mentions"], ascending=False)
+        return popularity

openbb_terminal/common/behavioural_analysis/reddit_model.py

@@ -7,17 +7,20 @@ import warnings
 from datetime import datetime, timedelta
 from typing import List, Tuple
 
-import finviz
 import pandas as pd
 import praw
 from pmaw import PushshiftAPI
 from prawcore.exceptions import ResponseException
-from requests import HTTPError
 from sklearn.feature_extraction import _stop_words
 from tqdm import tqdm
 from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
 
-from openbb_terminal.common.behavioural_analysis.reddit_helpers import find_tickers
+from openbb_terminal.common.behavioural_analysis.reddit_helpers import (
+    RedditResponses,
+    find_tickers,
+    get_praw_api,
+    reddit_requirements,
+)
 from openbb_terminal.core.session.current_user import get_current_user
 from openbb_terminal.decorators import check_api_key, log_start_end
 from openbb_terminal.rich_config import console
@@ -37,15 +40,7 @@ l_sub_reddits = [
 
 
 @log_start_end(log=logger)
-@check_api_key(
-    [
-        "API_REDDIT_CLIENT_ID",
-        "API_REDDIT_CLIENT_SECRET",
-        "API_REDDIT_USERNAME",
-        "API_REDDIT_USER_AGENT",
-        "API_REDDIT_PASSWORD",
-    ]
-)
+@check_api_key(reddit_requirements)
 def get_popular_tickers(
     limit: int = 10, post_limit: int = 50, subreddits: str = ""
 ) -> pd.DataFrame:
@@ -71,28 +66,8 @@ def get_popular_tickers(
         if subreddits
         else l_sub_reddits
     )
-    d_watchlist_tickers: dict = {}
-    l_watchlist_author = []
 
-    praw_api = praw.Reddit(
-        client_id=current_user.credentials.API_REDDIT_CLIENT_ID,
-        client_secret=current_user.credentials.API_REDDIT_CLIENT_SECRET,
-        username=current_user.credentials.API_REDDIT_USERNAME,
-        user_agent=current_user.credentials.API_REDDIT_USER_AGENT,
-        password=current_user.credentials.API_REDDIT_PASSWORD,
-        check_for_updates=False,
-        comment_kind="t1",
-        message_kind="t4",
-        redditor_kind="t2",
-        submission_kind="t3",
-        subreddit_kind="t5",
-        trophy_kind="t6",
-        oauth_url="https://oauth.reddit.com",
-        reddit_url="https://www.reddit.com",
-        short_url="https://redd.it",
-        ratelimit_seconds=5,
-        timeout=16,
-    )
+    praw_api = get_praw_api(current_user)
     try:
         praw_api.user.me()
     except (Exception, ResponseException):
@@ -101,6 +76,7 @@ def get_popular_tickers(
 
     pmaw_api = PushshiftAPI()
 
+    responses = RedditResponses()
     for s_sub_reddit in sub_reddit_list:
         console.print(
             f"Searching for tickers in latest {post_limit} '{s_sub_reddit}' posts"
@@ -114,128 +90,15 @@ def get_popular_tickers(
             filter=["id"],
         )
 
-        n_tickers = 0
-        for submission in submissions.responses:
-            try:
-                # Get more information about post using PRAW api
-                submission_ = praw_api.submission(id=submission["id"])
+        count = responses.gather(praw_api, submissions.responses)
+        responses.validate()
+        console.print(f"  {count} potential tickers found.")
 
-                # Ensure that the post hasn't been removed by moderator in the meanwhile,
-                # that there is a description and it's not just an image, that the flair is
-                # meaningful, and that we aren't re-considering same author's content
-                def has_author(submission_) -> bool:
-                    """Check if submission has author."""
-                    return (
-                        hasattr(submission_, "author")
-                        and hasattr(submission_.author, "name")
-                        and submission_.author.name not in l_watchlist_author
-                    )
-
-                def has_content(submission_) -> bool:
-                    """Check if submission has text or title."""
-                    return hasattr(submission_, "selftext") or hasattr(
-                        submission_, "title"
-                    )
-
-                if (
-                    submission_ is not None
-                    and not submission_.removed_by_category
-                    and has_content(submission_)
-                    and has_author(submission_)
-                ):
-                    l_tickers_found = find_tickers(submission_)
-
-                    if l_tickers_found:
-                        n_tickers += len(l_tickers_found)
-
-                        # Add another author's name to the parsed watchlists
-                        l_watchlist_author.append(submission_.author.name)
-
-                        # Lookup stock tickers within a watchlist
-                        for key in l_tickers_found:
-                            if key in d_watchlist_tickers:
-                                # Increment stock ticker found
-                                d_watchlist_tickers[key] += 1
-                            else:
-                                # Initialize stock ticker found
-                                d_watchlist_tickers[key] = 1
-
-            except ResponseException as e:
-                logger.exception("Invalid response: %s", str(e))
-
-                if "received 401 HTTP response" in str(e):
-                    console.print("[red]Invalid API Key[/red]\n")
-                else:
-                    console.print(f"[red]Invalid response: {str(e)}[/red]\n")
-
-                return pd.DataFrame()
-
-        console.print(f"  {n_tickers} potential tickers found.")
-    lt_watchlist_sorted = sorted(
-        d_watchlist_tickers.items(), key=lambda item: item[1], reverse=True
-    )
-
-    if lt_watchlist_sorted:
-        n_top_stocks = 0
-        # pylint: disable=redefined-outer-name
-        popular_tickers = []
-        for t_ticker in lt_watchlist_sorted:
-            if n_top_stocks > limit:
-                break
-            try:
-                # If try doesn't trigger exception, it means that this stock exists on finviz
-                # thus we can print it.
-                stock_info = finviz.get_stock(t_ticker[0])
-                popular_tickers.append(
-                    (
-                        t_ticker[1],
-                        t_ticker[0],
-                        stock_info["Company"],
-                        stock_info["Sector"],
-                        stock_info["Price"],
-                        stock_info["Change"],
-                        stock_info["Perf Month"],
-                        f"https://finviz.com/quote.ashx?t={t_ticker[0]}",
-                    )
-                )
-                n_top_stocks += 1
-            except HTTPError as e:
-                if e.response.status_code != 404:
-                    logger.exception("Unexpected exception from Finviz: %s", str(e))
-                    console.print(f"Unexpected exception from Finviz: {e}")
-            except Exception as e:
-                logger.exception(str(e))
-                console.print(e, "\n")
-                return pd.DataFrame()
-
-        popular_tickers_df = pd.DataFrame(
-            popular_tickers,
-            columns=[
-                "Mentions",
-                "Ticker",
-                "Company",
-                "Sector",
-                "Price",
-                "Change",
-                "Perf Month",
-                "URL",
-            ],
-        )
-    else:
-        popular_tickers_df = pd.DataFrame()
-    return popular_tickers_df
+    return responses.to_df().head(limit)
 
 
 @log_start_end(log=logger)
-@check_api_key(
-    [
-        "API_REDDIT_CLIENT_ID",
-        "API_REDDIT_CLIENT_SECRET",
-        "API_REDDIT_USERNAME",
-        "API_REDDIT_USER_AGENT",
-        "API_REDDIT_PASSWORD",
-    ]
-)
+@check_api_key(reddit_requirements)
 def get_spac_community(
     limit: int = 10, popular: bool = False
 ) -> Tuple[pd.DataFrame, dict]:
@@ -256,25 +119,7 @@ def get_spac_community(
     """
     current_user = get_current_user()
 
-    praw_api = praw.Reddit(
-        client_id=current_user.credentials.API_REDDIT_CLIENT_ID,
-        client_secret=current_user.credentials.API_REDDIT_CLIENT_SECRET,
-        username=current_user.credentials.API_REDDIT_USERNAME,
-        user_agent=current_user.credentials.API_REDDIT_USER_AGENT,
-        password=current_user.credentials.API_REDDIT_PASSWORD,
-        check_for_updates=False,
-        comment_kind="t1",
-        message_kind="t4",
-        redditor_kind="t2",
-        submission_kind="t3",
-        subreddit_kind="t5",
-        trophy_kind="t6",
-        oauth_url="https://oauth.reddit.com",
-        reddit_url="https://www.reddit.com",
-        short_url="https://redd.it",
-        ratelimit_seconds=5,
-        timeout=16,
-    )
+    praw_api = get_praw_api(current_user)
     try:
         praw_api.user.me()
     except (Exception, ResponseException):
@@ -368,15 +213,7 @@ def get_spac_community(
 
 
 @log_start_end(log=logger)
-@check_api_key(
-    [
-        "API_REDDIT_CLIENT_ID",
-        "API_REDDIT_CLIENT_SECRET",
-        "API_REDDIT_USERNAME",
-        "API_REDDIT_USER_AGENT",
-        "API_REDDIT_PASSWORD",
-    ]
-)
+@check_api_key(reddit_requirements)
 def get_wsb_community(limit: int = 10, new: bool = False) -> pd.DataFrame:
     """Get wsb posts [Source: reddit].
 
@@ -394,26 +231,7 @@ def get_wsb_community(limit: int = 10, new: bool = False) -> pd.DataFrame:
     """
     current_user = get_current_user()
 
-    # See https://github.com/praw-dev/praw/issues/1016 regarding praw arguments
-    praw_api = praw.Reddit(
-        client_id=current_user.credentials.API_REDDIT_CLIENT_ID,
-        client_secret=current_user.credentials.API_REDDIT_CLIENT_SECRET,
-        username=current_user.credentials.API_REDDIT_USERNAME,
-        user_agent=current_user.credentials.API_REDDIT_USER_AGENT,
-        password=current_user.credentials.API_REDDIT_PASSWORD,
-        check_for_updates=False,
-        comment_kind="t1",
-        message_kind="t4",
-        redditor_kind="t2",
-        submission_kind="t3",
-        subreddit_kind="t5",
-        trophy_kind="t6",
-        oauth_url="https://oauth.reddit.com",
-        reddit_url="https://www.reddit.com",
-        short_url="https://redd.it",
-        ratelimit_seconds=5,
-        timeout=16,
-    )
+    praw_api = get_praw_api(current_user)
     try:
         praw_api.user.me()
     except (Exception, ResponseException):
@@ -479,15 +297,7 @@ def get_wsb_community(limit: int = 10, new: bool = False) -> pd.DataFrame:
 
 
 @log_start_end(log=logger)
-@check_api_key(
-    [
-        "API_REDDIT_CLIENT_ID",
-        "API_REDDIT_CLIENT_SECRET",
-        "API_REDDIT_USERNAME",
-        "API_REDDIT_USER_AGENT",
-        "API_REDDIT_PASSWORD",
-    ]
-)
+@check_api_key(reddit_requirements)
 def get_due_dilligence(
     limit: int = 5, n_days: int = 3, show_all_flairs: bool = False
 ) -> pd.DataFrame:
@@ -509,25 +319,7 @@ def get_due_dilligence(
     """
     current_user = get_current_user()
 
-    praw_api = praw.Reddit(
-        client_id=current_user.credentials.API_REDDIT_CLIENT_ID,
-        client_secret=current_user.credentials.API_REDDIT_CLIENT_SECRET,
-        username=current_user.credentials.API_REDDIT_USERNAME,
-        user_agent=current_user.credentials.API_REDDIT_USER_AGENT,
-        password=current_user.credentials.API_REDDIT_PASSWORD,
-        check_for_updates=False,
-        comment_kind="t1",
-        message_kind="t4",
-        redditor_kind="t2",
-        submission_kind="t3",
-        subreddit_kind="t5",
-        trophy_kind="t6",
-        oauth_url="https://oauth.reddit.com",
-        reddit_url="https://www.reddit.com",
-        short_url="https://redd.it",
-        ratelimit_seconds=5,
-        timeout=16,
-    )
+    praw_api = get_praw_api(current_user)
     try:
         praw_api.user.me()
     except (Exception, ResponseException):
@@ -632,15 +424,7 @@ def get_due_dilligence(
 
 
 @log_start_end(log=logger)
-@check_api_key(
-    [
-        "API_REDDIT_CLIENT_ID",
-        "API_REDDIT_CLIENT_SECRET",
-        "API_REDDIT_USERNAME",
-        "API_REDDIT_USER_AGENT",
-        "API_REDDIT_PASSWORD",
-    ]
-)
+@check_api_key(reddit_requirements)
 def get_posts_about(
     symbol: str,
     limit: int = 100,
@@ -675,25 +459,7 @@ def get_posts_about(
     """
     current_user = get_current_user()
 
-    praw_api = praw.Reddit(
-        client_id=current_user.credentials.API_REDDIT_CLIENT_ID,
-        client_secret=current_user.credentials.API_REDDIT_CLIENT_SECRET,
-        username=current_user.credentials.API_REDDIT_USERNAME,
-        user_agent=current_user.credentials.API_REDDIT_USER_AGENT,
-        password=current_user.credentials.API_REDDIT_PASSWORD,
-        check_for_updates=False,
-        comment_kind="t1",
-        message_kind="t4",
-        redditor_kind="t2",
-        submission_kind="t3",
-        subreddit_kind="t5",
-        trophy_kind="t6",
-        oauth_url="https://oauth.reddit.com",
-        reddit_url="https://www.reddit.com",
-        short_url="https://redd.it",
-        ratelimit_seconds=5,
-        timeout=16,
-    )
+    praw_api = get_praw_api(current_user)
     try:
         praw_api.user.me()
     except (Exception, ResponseException):
@@ -748,15 +514,7 @@ def get_posts_about(
 
 
 @log_start_end(log=logger)
-@check_api_key(
-    [
-        "API_REDDIT_CLIENT_ID",
-        "API_REDDIT_CLIENT_SECRET",
-        "API_REDDIT_USERNAME",
-        "API_REDDIT_USER_AGENT",
-        "API_REDDIT_PASSWORD",
-    ]
-)
+@check_api_key(reddit_requirements)
 def get_comments(
     post: praw.models.reddit.submission.Submission,
 ) -> List[praw.models.reddit.comment.Comment]:
@@ -819,15 +577,7 @@ def clean_reddit_text(docs: List[str]) -> List[str]:
 
 
 @log_start_end(log=logger)
-@check_api_key(
-    [
-        "API_REDDIT_CLIENT_ID",
-        "API_REDDIT_CLIENT_SECRET",
-        "API_REDDIT_USERNAME",
-        "API_REDDIT_USER_AGENT",
-        "API_REDDIT_PASSWORD",
-    ]
-)
+@check_api_key(reddit_requirements)
 def get_sentiment(post_data: List[str]) -> float:
     """Find the sentiment of a post and related comments.
 

openbb_terminal/common/behavioural_analysis/reddit_view.py

@@ -12,6 +12,9 @@ import praw
 
 from openbb_terminal import OpenBBFigure, rich_config
 from openbb_terminal.common.behavioural_analysis import reddit_model
+from openbb_terminal.common.behavioural_analysis.reddit_helpers import (
+    reddit_requirements,
+)
 from openbb_terminal.core.session.current_user import get_current_user
 from openbb_terminal.decorators import check_api_key, log_start_end
 from openbb_terminal.helper_funcs import export_data, print_rich_table
@@ -25,15 +28,7 @@ logger = logging.getLogger(__name__)
 
 
 @log_start_end(log=logger)
-@check_api_key(
-    [
-        "API_REDDIT_CLIENT_ID",
-        "API_REDDIT_CLIENT_SECRET",
-        "API_REDDIT_USERNAME",
-        "API_REDDIT_USER_AGENT",
-        "API_REDDIT_PASSWORD",
-    ]
-)
+@check_api_key(reddit_requirements)
 def print_and_record_reddit_post(
     submissions_dict: Dict, submission: praw.models.reddit.submission.Submission
 ):
@@ -87,15 +82,7 @@ def print_and_record_reddit_post(
 
 
 @log_start_end(log=logger)
-@check_api_key(
-    [
-        "API_REDDIT_CLIENT_ID",
-        "API_REDDIT_CLIENT_SECRET",
-        "API_REDDIT_USERNAME",
-        "API_REDDIT_USER_AGENT",
-        "API_REDDIT_PASSWORD",
-    ]
-)
+@check_api_key(reddit_requirements)
 def print_reddit_post(sub: tuple):
     """Print reddit submission.
 
@@ -131,15 +118,7 @@ def print_reddit_post(sub: tuple):
 
 
 @log_start_end(log=logger)
-@check_api_key(
-    [
-        "API_REDDIT_CLIENT_ID",
-        "API_REDDIT_CLIENT_SECRET",
-        "API_REDDIT_USERNAME",
-        "API_REDDIT_USER_AGENT",
-        "API_REDDIT_PASSWORD",
-    ]
-)
+@check_api_key(reddit_requirements)
 def display_popular_tickers(
     limit: int = 10,
     post_limit: int = 50,
@@ -161,16 +140,16 @@ def display_popular_tickers(
         Format to export dataframe
     """
     popular_tickers_df = reddit_model.get_popular_tickers(limit, post_limit, subreddits)
-    if not popular_tickers_df.empty:
-        print_rich_table(
-            popular_tickers_df,
-            headers=list(popular_tickers_df.columns),
-            show_index=False,
-            title=f"The following TOP {limit} tickers have been mentioned",
-            export=bool(export),
-        )
-    else:
+    if popular_tickers_df.empty:
         console.print("No tickers found")
+        return
+    print_rich_table(
+        popular_tickers_df,
+        headers=list(popular_tickers_df.columns),
+        show_index=True,
+        title=f"The following TOP {limit} tickers have been mentioned",
+        export=bool(export),
+    )
 
     export_data(
         export,
@@ -182,15 +161,7 @@ def display_popular_tickers(
 
 
 @log_start_end(log=logger)
-@check_api_key(
-    [
-        "API_REDDIT_CLIENT_ID",
-        "API_REDDIT_CLIENT_SECRET",
-        "API_REDDIT_USERNAME",
-        "API_REDDIT_USER_AGENT",
-        "API_REDDIT_PASSWORD",
-    ]
-)
+@check_api_key(reddit_requirements)
 def display_spac_community(limit: int = 10, popular: bool = False):
     """Print tickers mentioned in r/SPACs [Source: Reddit].
 
@@ -233,15 +204,7 @@ def display_spac_community(limit: int = 10, popular: bool = False):
 
 
 @log_start_end(log=logger)
-@check_api_key(
-    [
-        "API_REDDIT_CLIENT_ID",
-        "API_REDDIT_CLIENT_SECRET",
-        "API_REDDIT_USERNAME",
-        "API_REDDIT_USER_AGENT",
-        "API_REDDIT_PASSWORD",
-    ]
-)
+@check_api_key(reddit_requirements)
 def display_wsb_community(limit: int = 10, new: bool = False):
     """Print WSB posts.
 
@@ -260,15 +223,7 @@ def display_wsb_community(limit: int = 10, new: bool = False):
 
 
 @log_start_end(log=logger)
-@check_api_key(
-    [
-        "API_REDDIT_CLIENT_ID",
-        "API_REDDIT_CLIENT_SECRET",
-        "API_REDDIT_USERNAME",
-        "API_REDDIT_USER_AGENT",
-        "API_REDDIT_PASSWORD",
-    ]
-)
+@check_api_key(reddit_requirements)
 def display_due_diligence(
     limit: int = 10, n_days: int = 3, show_all_flairs: bool = False
 ):
@@ -293,15 +248,7 @@ def display_due_diligence(
 
 
 @log_start_end(log=logger)
-@check_api_key(
-    [
-        "API_REDDIT_CLIENT_ID",
-        "API_REDDIT_CLIENT_SECRET",
-        "API_REDDIT_USERNAME",
-        "API_REDDIT_USER_AGENT",
-        "API_REDDIT_PASSWORD",
-    ]
-)
+@check_api_key(reddit_requirements)
 def display_redditsent(
     symbol: str,
     sortby: str = "relevance",