diff --git a/openbb_terminal/common/behavioural_analysis/reddit_helpers.py b/openbb_terminal/common/behavioural_analysis/reddit_helpers.py index 7c087b91ca8..3d0298cef65 100644 --- a/openbb_terminal/common/behavioural_analysis/reddit_helpers.py +++ b/openbb_terminal/common/behavioural_analysis/reddit_helpers.py @@ -1,10 +1,27 @@ """Reddit Helpers.""" __docformat__ = "numpy" +import logging import re +import urllib.request from typing import List +import pandas as pd import praw +from prawcore.exceptions import ResponseException + +from openbb_terminal.core.models import UserModel +from openbb_terminal.rich_config import console + +logger = logging.getLogger(__name__) + +reddit_requirements = [ + "API_REDDIT_CLIENT_ID", + "API_REDDIT_CLIENT_SECRET", + "API_REDDIT_USERNAME", + "API_REDDIT_USER_AGENT", + "API_REDDIT_PASSWORD", +] def find_tickers(submission: praw.models.reddit.submission.Submission) -> List[str]: @@ -31,3 +48,124 @@ def find_tickers(submission: praw.models.reddit.submission.Submission) -> List[s l_tickers_found.append(s_ticker.strip()) return l_tickers_found + + +def get_praw_api(current_user: UserModel) -> praw.Reddit: + """Get praw api. + + Parameters + ---------- + current_user : UserModel + The user model + + Returns + ------- + praw.Reddit + Praw api + """ + # See https://github.com/praw-dev/praw/issues/1016 regarding praw arguments + praw_api = praw.Reddit( + client_id=current_user.credentials.API_REDDIT_CLIENT_ID, + client_secret=current_user.credentials.API_REDDIT_CLIENT_SECRET, + username=current_user.credentials.API_REDDIT_USERNAME, + user_agent=current_user.credentials.API_REDDIT_USER_AGENT, + password=current_user.credentials.API_REDDIT_PASSWORD, + check_for_updates=False, + comment_kind="t1", + message_kind="t4", + redditor_kind="t2", + submission_kind="t3", + subreddit_kind="t5", + trophy_kind="t6", + oauth_url="https://oauth.reddit.com", + reddit_url="https://www.reddit.com", + short_url="https://redd.it", + ratelimit_seconds=5, + timeout=16, + ) + return praw_api + + +def has_author(submission_, authors) -> bool: + """Check if submission has author.""" + return ( + hasattr(submission_, "author") + and hasattr(submission_.author, "name") + and submission_.author.name not in authors + ) + + +def has_content(submission_) -> bool: + """Check if submission has text or title.""" + return hasattr(submission_, "selftext") or hasattr(submission_, "title") + + +class RedditResponses: + def __init__(self): + self.tickers: dict = {} + self.authors: List[str] = [] + + def gather(self, praw_api, responses) -> int: + count = 0 + for submission in responses: + try: + # Get more information about post using PRAW api + submission_ = praw_api.submission(id=submission["id"]) + + # Ensure that the post hasn't been removed by moderator in the meanwhile, + # that there is a description and it's not just an image, that the flair is + # meaningful, and that we aren't re-considering same author's content + + if ( + submission_ is not None + and not submission_.removed_by_category + and has_content(submission_) + and has_author(submission_, self.authors) + ): + l_tickers_found = find_tickers(submission_) + + if l_tickers_found: + count += len(l_tickers_found) + + # Add another author's name to the parsed watchlists + self.authors.append(submission_.author.name) + + # Lookup stock tickers within a watchlist + for key in l_tickers_found: + if key in self.tickers: + # Increment stock ticker found + self.tickers[key] += 1 + else: + # Initialize stock ticker found + self.tickers[key] = 1 + + except ResponseException as e: + logger.exception("Invalid response: %s", str(e)) + + if "received 401 HTTP response" in str(e): + console.print("[red]Invalid API Key[/red]\n") + else: + console.print(f"[red]Invalid response: {str(e)}[/red]\n") + return 0 + return count + + def validate( + self, + url: str = "https://raw.githubusercontent.com/rreichel3/US-Stock-Symbols/main/all/all_tickers.txt", + ) -> None: + """Validate tickers.""" + valids = [] + with urllib.request.urlopen(url) as file: + for line in file: + new_item = line.decode("utf-8").replace("\n", "").strip() + valids.append(new_item) + + for key in list(self.tickers.keys()): + if key not in valids: + self.tickers.pop(key) + + def to_df(self) -> pd.DataFrame: + popularity = pd.DataFrame.from_dict(self.tickers, orient="index") + popularity.columns = ["Mentions"] + popularity = popularity.sort_values(by=["Mentions"], ascending=False) + return popularity diff --git a/openbb_terminal/common/behavioural_analysis/reddit_model.py b/openbb_terminal/common/behavioural_analysis/reddit_model.py index ef983301047..3b6e65748a7 100644 --- a/openbb_terminal/common/behavioural_analysis/reddit_model.py +++ b/openbb_terminal/common/behavioural_analysis/reddit_model.py @@ -7,17 +7,20 @@ import warnings from datetime import datetime, timedelta from typing import List, Tuple -import finviz import pandas as pd import praw from pmaw import PushshiftAPI from prawcore.exceptions import ResponseException -from requests import HTTPError from sklearn.feature_extraction import _stop_words from tqdm import tqdm from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer -from openbb_terminal.common.behavioural_analysis.reddit_helpers import find_tickers +from openbb_terminal.common.behavioural_analysis.reddit_helpers import ( + RedditResponses, + find_tickers, + get_praw_api, + reddit_requirements, +) from openbb_terminal.core.session.current_user import get_current_user from openbb_terminal.decorators import check_api_key, log_start_end from openbb_terminal.rich_config import console @@ -37,15 +40,7 @@ l_sub_reddits = [ @log_start_end(log=logger) -@check_api_key( - [ - "API_REDDIT_CLIENT_ID", - "API_REDDIT_CLIENT_SECRET", - "API_REDDIT_USERNAME", - "API_REDDIT_USER_AGENT", - "API_REDDIT_PASSWORD", - ] -) +@check_api_key(reddit_requirements) def get_popular_tickers( limit: int = 10, post_limit: int = 50, subreddits: str = "" ) -> pd.DataFrame: @@ -71,28 +66,8 @@ def get_popular_tickers( if subreddits else l_sub_reddits ) - d_watchlist_tickers: dict = {} - l_watchlist_author = [] - praw_api = praw.Reddit( - client_id=current_user.credentials.API_REDDIT_CLIENT_ID, - client_secret=current_user.credentials.API_REDDIT_CLIENT_SECRET, - username=current_user.credentials.API_REDDIT_USERNAME, - user_agent=current_user.credentials.API_REDDIT_USER_AGENT, - password=current_user.credentials.API_REDDIT_PASSWORD, - check_for_updates=False, - comment_kind="t1", - message_kind="t4", - redditor_kind="t2", - submission_kind="t3", - subreddit_kind="t5", - trophy_kind="t6", - oauth_url="https://oauth.reddit.com", - reddit_url="https://www.reddit.com", - short_url="https://redd.it", - ratelimit_seconds=5, - timeout=16, - ) + praw_api = get_praw_api(current_user) try: praw_api.user.me() except (Exception, ResponseException): @@ -101,6 +76,7 @@ def get_popular_tickers( pmaw_api = PushshiftAPI() + responses = RedditResponses() for s_sub_reddit in sub_reddit_list: console.print( f"Searching for tickers in latest {post_limit} '{s_sub_reddit}' posts" @@ -114,128 +90,15 @@ def get_popular_tickers( filter=["id"], ) - n_tickers = 0 - for submission in submissions.responses: - try: - # Get more information about post using PRAW api - submission_ = praw_api.submission(id=submission["id"]) + count = responses.gather(praw_api, submissions.responses) + responses.validate() + console.print(f" {count} potential tickers found.") - # Ensure that the post hasn't been removed by moderator in the meanwhile, - # that there is a description and it's not just an image, that the flair is - # meaningful, and that we aren't re-considering same author's content - def has_author(submission_) -> bool: - """Check if submission has author.""" - return ( - hasattr(submission_, "author") - and hasattr(submission_.author, "name") - and submission_.author.name not in l_watchlist_author - ) - - def has_content(submission_) -> bool: - """Check if submission has text or title.""" - return hasattr(submission_, "selftext") or hasattr( - submission_, "title" - ) - - if ( - submission_ is not None - and not submission_.removed_by_category - and has_content(submission_) - and has_author(submission_) - ): - l_tickers_found = find_tickers(submission_) - - if l_tickers_found: - n_tickers += len(l_tickers_found) - - # Add another author's name to the parsed watchlists - l_watchlist_author.append(submission_.author.name) - - # Lookup stock tickers within a watchlist - for key in l_tickers_found: - if key in d_watchlist_tickers: - # Increment stock ticker found - d_watchlist_tickers[key] += 1 - else: - # Initialize stock ticker found - d_watchlist_tickers[key] = 1 - - except ResponseException as e: - logger.exception("Invalid response: %s", str(e)) - - if "received 401 HTTP response" in str(e): - console.print("[red]Invalid API Key[/red]\n") - else: - console.print(f"[red]Invalid response: {str(e)}[/red]\n") - - return pd.DataFrame() - - console.print(f" {n_tickers} potential tickers found.") - lt_watchlist_sorted = sorted( - d_watchlist_tickers.items(), key=lambda item: item[1], reverse=True - ) - - if lt_watchlist_sorted: - n_top_stocks = 0 - # pylint: disable=redefined-outer-name - popular_tickers = [] - for t_ticker in lt_watchlist_sorted: - if n_top_stocks > limit: - break - try: - # If try doesn't trigger exception, it means that this stock exists on finviz - # thus we can print it. - stock_info = finviz.get_stock(t_ticker[0]) - popular_tickers.append( - ( - t_ticker[1], - t_ticker[0], - stock_info["Company"], - stock_info["Sector"], - stock_info["Price"], - stock_info["Change"], - stock_info["Perf Month"], - f"https://finviz.com/quote.ashx?t={t_ticker[0]}", - ) - ) - n_top_stocks += 1 - except HTTPError as e: - if e.response.status_code != 404: - logger.exception("Unexpected exception from Finviz: %s", str(e)) - console.print(f"Unexpected exception from Finviz: {e}") - except Exception as e: - logger.exception(str(e)) - console.print(e, "\n") - return pd.DataFrame() - - popular_tickers_df = pd.DataFrame( - popular_tickers, - columns=[ - "Mentions", - "Ticker", - "Company", - "Sector", - "Price", - "Change", - "Perf Month", - "URL", - ], - ) - else: - popular_tickers_df = pd.DataFrame() - return popular_tickers_df + return responses.to_df().head(limit) @log_start_end(log=logger) -@check_api_key( - [ - "API_REDDIT_CLIENT_ID", - "API_REDDIT_CLIENT_SECRET", - "API_REDDIT_USERNAME", - "API_REDDIT_USER_AGENT", - "API_REDDIT_PASSWORD", - ] -) +@check_api_key(reddit_requirements) def get_spac_community( limit: int = 10, popular: bool = False ) -> Tuple[pd.DataFrame, dict]: @@ -256,25 +119,7 @@ def get_spac_community( """ current_user = get_current_user() - praw_api = praw.Reddit( - client_id=current_user.credentials.API_REDDIT_CLIENT_ID, - client_secret=current_user.credentials.API_REDDIT_CLIENT_SECRET, - username=current_user.credentials.API_REDDIT_USERNAME, - user_agent=current_user.credentials.API_REDDIT_USER_AGENT, - password=current_user.credentials.API_REDDIT_PASSWORD, - check_for_updates=False, - comment_kind="t1", - message_kind="t4", - redditor_kind="t2", - submission_kind="t3", - subreddit_kind="t5", - trophy_kind="t6", - oauth_url="https://oauth.reddit.com", - reddit_url="https://www.reddit.com", - short_url="https://redd.it", - ratelimit_seconds=5, - timeout=16, - ) + praw_api = get_praw_api(current_user) try: praw_api.user.me() except (Exception, ResponseException): @@ -368,15 +213,7 @@ def get_spac_community( @log_start_end(log=logger) -@check_api_key( - [ - "API_REDDIT_CLIENT_ID", - "API_REDDIT_CLIENT_SECRET", - "API_REDDIT_USERNAME", - "API_REDDIT_USER_AGENT", - "API_REDDIT_PASSWORD", - ] -) +@check_api_key(reddit_requirements) def get_wsb_community(limit: int = 10, new: bool = False) -> pd.DataFrame: """Get wsb posts [Source: reddit]. @@ -394,26 +231,7 @@ def get_wsb_community(limit: int = 10, new: bool = False) -> pd.DataFrame: """ current_user = get_current_user() - # See https://github.com/praw-dev/praw/issues/1016 regarding praw arguments - praw_api = praw.Reddit( - client_id=current_user.credentials.API_REDDIT_CLIENT_ID, - client_secret=current_user.credentials.API_REDDIT_CLIENT_SECRET, - username=current_user.credentials.API_REDDIT_USERNAME, - user_agent=current_user.credentials.API_REDDIT_USER_AGENT, - password=current_user.credentials.API_REDDIT_PASSWORD, - check_for_updates=False, - comment_kind="t1", - message_kind="t4", - redditor_kind="t2", - submission_kind="t3", - subreddit_kind="t5", - trophy_kind="t6", - oauth_url="https://oauth.reddit.com", - reddit_url="https://www.reddit.com", - short_url="https://redd.it", - ratelimit_seconds=5, - timeout=16, - ) + praw_api = get_praw_api(current_user) try: praw_api.user.me() except (Exception, ResponseException): @@ -479,15 +297,7 @@ def get_wsb_community(limit: int = 10, new: bool = False) -> pd.DataFrame: @log_start_end(log=logger) -@check_api_key( - [ - "API_REDDIT_CLIENT_ID", - "API_REDDIT_CLIENT_SECRET", - "API_REDDIT_USERNAME", - "API_REDDIT_USER_AGENT", - "API_REDDIT_PASSWORD", - ] -) +@check_api_key(reddit_requirements) def get_due_dilligence( limit: int = 5, n_days: int = 3, show_all_flairs: bool = False ) -> pd.DataFrame: @@ -509,25 +319,7 @@ def get_due_dilligence( """ current_user = get_current_user() - praw_api = praw.Reddit( - client_id=current_user.credentials.API_REDDIT_CLIENT_ID, - client_secret=current_user.credentials.API_REDDIT_CLIENT_SECRET, - username=current_user.credentials.API_REDDIT_USERNAME, - user_agent=current_user.credentials.API_REDDIT_USER_AGENT, - password=current_user.credentials.API_REDDIT_PASSWORD, - check_for_updates=False, - comment_kind="t1", - message_kind="t4", - redditor_kind="t2", - submission_kind="t3", - subreddit_kind="t5", - trophy_kind="t6", - oauth_url="https://oauth.reddit.com", - reddit_url="https://www.reddit.com", - short_url="https://redd.it", - ratelimit_seconds=5, - timeout=16, - ) + praw_api = get_praw_api(current_user) try: praw_api.user.me() except (Exception, ResponseException): @@ -632,15 +424,7 @@ def get_due_dilligence( @log_start_end(log=logger) -@check_api_key( - [ - "API_REDDIT_CLIENT_ID", - "API_REDDIT_CLIENT_SECRET", - "API_REDDIT_USERNAME", - "API_REDDIT_USER_AGENT", - "API_REDDIT_PASSWORD", - ] -) +@check_api_key(reddit_requirements) def get_posts_about( symbol: str, limit: int = 100, @@ -675,25 +459,7 @@ def get_posts_about( """ current_user = get_current_user() - praw_api = praw.Reddit( - client_id=current_user.credentials.API_REDDIT_CLIENT_ID, - client_secret=current_user.credentials.API_REDDIT_CLIENT_SECRET, - username=current_user.credentials.API_REDDIT_USERNAME, - user_agent=current_user.credentials.API_REDDIT_USER_AGENT, - password=current_user.credentials.API_REDDIT_PASSWORD, - check_for_updates=False, - comment_kind="t1", - message_kind="t4", - redditor_kind="t2", - submission_kind="t3", - subreddit_kind="t5", - trophy_kind="t6", - oauth_url="https://oauth.reddit.com", - reddit_url="https://www.reddit.com", - short_url="https://redd.it", - ratelimit_seconds=5, - timeout=16, - ) + praw_api = get_praw_api(current_user) try: praw_api.user.me() except (Exception, ResponseException): @@ -748,15 +514,7 @@ def get_posts_about( @log_start_end(log=logger) -@check_api_key( - [ - "API_REDDIT_CLIENT_ID", - "API_REDDIT_CLIENT_SECRET", - "API_REDDIT_USERNAME", - "API_REDDIT_USER_AGENT", - "API_REDDIT_PASSWORD", - ] -) +@check_api_key(reddit_requirements) def get_comments( post: praw.models.reddit.submission.Submission, ) -> List[praw.models.reddit.comment.Comment]: @@ -819,15 +577,7 @@ def clean_reddit_text(docs: List[str]) -> List[str]: @log_start_end(log=logger) -@check_api_key( - [ - "API_REDDIT_CLIENT_ID", - "API_REDDIT_CLIENT_SECRET", - "API_REDDIT_USERNAME", - "API_REDDIT_USER_AGENT", - "API_REDDIT_PASSWORD", - ] -) +@check_api_key(reddit_requirements) def get_sentiment(post_data: List[str]) -> float: """Find the sentiment of a post and related comments. diff --git a/openbb_terminal/common/behavioural_analysis/reddit_view.py b/openbb_terminal/common/behavioural_analysis/reddit_view.py index f0dc65f637b..7af722d49ea 100644 --- a/openbb_terminal/common/behavioural_analysis/reddit_view.py +++ b/openbb_terminal/common/behavioural_analysis/reddit_view.py @@ -12,6 +12,9 @@ import praw from openbb_terminal import OpenBBFigure, rich_config from openbb_terminal.common.behavioural_analysis import reddit_model +from openbb_terminal.common.behavioural_analysis.reddit_helpers import ( + reddit_requirements, +) from openbb_terminal.core.session.current_user import get_current_user from openbb_terminal.decorators import check_api_key, log_start_end from openbb_terminal.helper_funcs import export_data, print_rich_table @@ -25,15 +28,7 @@ logger = logging.getLogger(__name__) @log_start_end(log=logger) -@check_api_key( - [ - "API_REDDIT_CLIENT_ID", - "API_REDDIT_CLIENT_SECRET", - "API_REDDIT_USERNAME", - "API_REDDIT_USER_AGENT", - "API_REDDIT_PASSWORD", - ] -) +@check_api_key(reddit_requirements) def print_and_record_reddit_post( submissions_dict: Dict, submission: praw.models.reddit.submission.Submission ): @@ -87,15 +82,7 @@ def print_and_record_reddit_post( @log_start_end(log=logger) -@check_api_key( - [ - "API_REDDIT_CLIENT_ID", - "API_REDDIT_CLIENT_SECRET", - "API_REDDIT_USERNAME", - "API_REDDIT_USER_AGENT", - "API_REDDIT_PASSWORD", - ] -) +@check_api_key(reddit_requirements) def print_reddit_post(sub: tuple): """Print reddit submission. @@ -131,15 +118,7 @@ def print_reddit_post(sub: tuple): @log_start_end(log=logger) -@check_api_key( - [ - "API_REDDIT_CLIENT_ID", - "API_REDDIT_CLIENT_SECRET", - "API_REDDIT_USERNAME", - "API_REDDIT_USER_AGENT", - "API_REDDIT_PASSWORD", - ] -) +@check_api_key(reddit_requirements) def display_popular_tickers( limit: int = 10, post_limit: int = 50, @@ -161,16 +140,16 @@ def display_popular_tickers( Format to export dataframe """ popular_tickers_df = reddit_model.get_popular_tickers(limit, post_limit, subreddits) - if not popular_tickers_df.empty: - print_rich_table( - popular_tickers_df, - headers=list(popular_tickers_df.columns), - show_index=False, - title=f"The following TOP {limit} tickers have been mentioned", - export=bool(export), - ) - else: + if popular_tickers_df.empty: console.print("No tickers found") + return + print_rich_table( + popular_tickers_df, + headers=list(popular_tickers_df.columns), + show_index=True, + title=f"The following TOP {limit} tickers have been mentioned", + export=bool(export), + ) export_data( export, @@ -182,15 +161,7 @@ def display_popular_tickers( @log_start_end(log=logger) -@check_api_key( - [ - "API_REDDIT_CLIENT_ID", - "API_REDDIT_CLIENT_SECRET", - "API_REDDIT_USERNAME", - "API_REDDIT_USER_AGENT", - "API_REDDIT_PASSWORD", - ] -) +@check_api_key(reddit_requirements) def display_spac_community(limit: int = 10, popular: bool = False): """Print tickers mentioned in r/SPACs [Source: Reddit]. @@ -233,15 +204,7 @@ def display_spac_community(limit: int = 10, popular: bool = False): @log_start_end(log=logger) -@check_api_key( - [ - "API_REDDIT_CLIENT_ID", - "API_REDDIT_CLIENT_SECRET", - "API_REDDIT_USERNAME", - "API_REDDIT_USER_AGENT", - "API_REDDIT_PASSWORD", - ] -) +@check_api_key(reddit_requirements) def display_wsb_community(limit: int = 10, new: bool = False): """Print WSB posts. @@ -260,15 +223,7 @@ def display_wsb_community(limit: int = 10, new: bool = False): @log_start_end(log=logger) -@check_api_key( - [ - "API_REDDIT_CLIENT_ID", - "API_REDDIT_CLIENT_SECRET", - "API_REDDIT_USERNAME", - "API_REDDIT_USER_AGENT", - "API_REDDIT_PASSWORD", - ] -) +@check_api_key(reddit_requirements) def display_due_diligence( limit: int = 10, n_days: int = 3, show_all_flairs: bool = False ): @@ -293,15 +248,7 @@ def display_due_diligence( @log_start_end(log=logger) -@check_api_key( - [ - "API_REDDIT_CLIENT_ID", - "API_REDDIT_CLIENT_SECRET", - "API_REDDIT_USERNAME", - "API_REDDIT_USER_AGENT", - "API_REDDIT_PASSWORD", - ] -) +@check_api_key(reddit_requirements) def display_redditsent( symbol: str, sortby: str = "relevance",