From bcb2262a9e05a729313664da506f3975be684f96 Mon Sep 17 00:00:00 2001 From: didier Date: Tue, 23 Feb 2021 21:45:36 +0000 Subject: [PATCH] Improve reddit API file --- sentiment/reddit_api.py | 317 +++++++++++++++++++++------------------- sentiment/sen_menu.py | 4 +- 2 files changed, 166 insertions(+), 155 deletions(-) diff --git a/sentiment/reddit_api.py b/sentiment/reddit_api.py index 33e8c19c7b9..d370b906b08 100644 --- a/sentiment/reddit_api.py +++ b/sentiment/reddit_api.py @@ -162,12 +162,12 @@ def watchlist(l_args): # ------------------------------------------------------------------------------------------------------------------- def popular_tickers(l_args): parser = argparse.ArgumentParser(prog='popular', - description=""" Latest popular tickers [Reddit] """) - parser.add_argument('-l', "--limit", action="store", dest="n_limit", type=check_positive, default=30, + description="""Print latest popular tickers. [Source: Reddit] """) + parser.add_argument('-l', "--limit", action="store", dest="n_limit", type=check_positive, default=50, help='limit of posts retrieved per sub reddit.') parser.add_argument('-s', "--sub", action="store", dest="s_subreddit", type=str, - help="""subreddit to look for tickers. Default: pennystocks, RobinHoodPennyStocks, Daytrading, - StockMarket, stocks, investing, wallstreetbets""") + help="""subreddits to look for tickers, e.g. pennystocks,stocks. + Default: pennystocks, RobinHoodPennyStocks, Daytrading, StockMarket, stocks, investing, wallstreetbets""") parser.add_argument('-d', "--days", action="store", dest="n_days", type=check_positive, default=1, help="look for the tickers from those n past days.") @@ -181,7 +181,10 @@ def popular_tickers(l_args): n_ts_after = int((datetime.today() - timedelta(days=ns_parser.n_days)).timestamp()) if ns_parser.s_subreddit: - l_sub_reddits = [ns_parser.s_subreddit] + if ',' in ns_parser.s_subreddit: + l_sub_reddits = ns_parser.s_subreddit.split(',') + else: + l_sub_reddits = [ns_parser.s_subreddit] else: l_sub_reddits = ['pennystocks', 'RobinHoodPennyStocks', 'Daytrading', 'StockMarket', 'stocks', 'investing', 'wallstreetbets'] @@ -189,6 +192,7 @@ def popular_tickers(l_args): d_watchlist_tickers = {} l_watchlist_links = list() l_watchlist_author = list() + ls_text = list() praw_api = praw.Reddit(client_id=cfg.API_REDDIT_CLIENT_ID, client_secret=cfg.API_REDDIT_CLIENT_SECRET, @@ -216,16 +220,20 @@ def popular_tickers(l_args): #that there is a description and it's not just an image, that the flair is #meaningful, and that we aren't re-considering same author's content if not submission.removed_by_category and (submission.selftext or submission.title) \ - and submission.link_flair_text not in ['Yolo', 'Meme'] \ and submission.author.name not in l_watchlist_author: + ls_text.append(submission.selftext) + ls_text.append(submission.title) - # Make sure there is at least 1 ticker found on the post text - l_tickers_found = set(re.findall("[A-Z]{3,5}", submission.selftext)) - if not l_tickers_found: - l_tickers_found = set(re.findall("[A-Z]{3,5}", submission.title)) + submission.comments.replace_more(limit=0) + for comment in submission.comments.list(): + ls_text.append(comment.body) + + l_tickers_found = list() + for s_text in ls_text: + for s_ticker in set(re.findall(r'\$([A-Z]{3,5} )', s_text)): + l_tickers_found.append(s_ticker.strip()) if l_tickers_found: - n_tickers += len(l_tickers_found) # Add another author's name to the parsed watchlists @@ -246,156 +254,23 @@ def popular_tickers(l_args): print(f" {n_tickers} tickers found.") - - lt_watchlist_sorted = sorted(d_watchlist_tickers.items(), key=lambda item: item[1], reverse=True) - print("\nThe following TOP10 tickers have been mentioned in the last 24 hours:") - n_top_stocks = 0 - for t_ticker in lt_watchlist_sorted: - if n_top_stocks > 9: - break - try: - # If try doesn't trigger exception, it means that this stock exists on finviz - #thus we can print it. - finviz.get_stock(t_ticker[0]) - print(f"{t_ticker[1]} {t_ticker[0]}") - n_top_stocks += 1 - except: - pass - print("") - - except: - print("") - - -# ------------------------------------------------------------------------------------------------------------------- -def spac(l_args): - parser = argparse.ArgumentParser(prog='spac', - description=""" Show other users SPACs announcement [Reddit] """) - parser.add_argument('-l', "--limit", action="store", dest="n_limit", type=check_positive, default=5, - help='limit of posts with SPACs retrieved') - parser.add_argument('-d', "--days", action="store", dest="n_days", type=check_positive, default=5, - help="look for the tickers from those n past days") - - try: - (ns_parser, l_unknown_args) = parser.parse_known_args(l_args) - - if l_unknown_args: - print(f"The following args couldn't be interpreted: {l_unknown_args}\n") - return - - praw_api = praw.Reddit(client_id=cfg.API_REDDIT_CLIENT_ID, - client_secret=cfg.API_REDDIT_CLIENT_SECRET, - username=cfg.API_REDDIT_USERNAME, - user_agent=cfg.API_REDDIT_USER_AGENT, - password=cfg.API_REDDIT_PASSWORD) - - d_submission = {} - d_watchlist_tickers = {} - l_watchlist_links = list() - l_watchlist_author = list() - - n_ts_after = int((datetime.today() - timedelta(days=ns_parser.n_days)).timestamp()) - l_sub_reddits = ['pennystocks', 'RobinHoodPennyStocks', 'Daytrading', 'StockMarket', 'stocks', 'investing', 'wallstreetbets'] - - warnings.filterwarnings("ignore") # To avoid printing the warning - psaw_api = PushshiftAPI() - submissions = psaw_api.search_submissions(after=n_ts_after, - subreddit=l_sub_reddits, - q='SPAC|Spac|spac|Spacs|spacs', - filter=['id']) - n_flair_posts_found = 0 - while True: - submission = next(submissions, None) - if submission: - # Get more information about post using PRAW api - submission = praw_api.submission(id=submission.id) - - # Ensure that the post hasn't been removed by moderator in the meanwhile, - #that there is a description and it's not just an image, that the flair is - #meaningful, and that we aren't re-considering same author's watchlist - if not submission.removed_by_category and submission.selftext \ - and submission.link_flair_text not in ['Yolo', 'Meme'] \ - and submission.author.name not in l_watchlist_author: - - # Make sure there is at least 1 ticker found on the post text - l_tickers_found = set(re.findall("[A-Z]{3,5}", submission.selftext)) - if not l_tickers_found: - l_tickers_found = set(re.findall("[A-Z]{3,5}", submission.title)) - if l_tickers_found: - - # Add another author's name to the parsed watchlists - l_watchlist_author.append(submission.author.name) - - # Lookup stock tickers within a watchlist - for key in l_tickers_found: - if key in d_watchlist_tickers: - # Increment stock ticker found - d_watchlist_tickers[key] += 1 - else: - # Initialize stock ticker found - d_watchlist_tickers[key] = 1 - - l_watchlist_links.append(f"https://www.reddit.com{submission.permalink}") - # delte below, not necessary I reckon. Probably just link? - - # Refactor data - s_datetime = datetime.utcfromtimestamp(submission.created_utc).strftime("%d/%m/%Y %H:%M:%S") - s_link = f"https://www.reddit.com{submission.permalink}" - s_all_awards = "" - for award in submission.all_awardings: - s_all_awards += f"{award['count']} {award['name']}\n" - s_all_awards = s_all_awards[:-2] - - # Create dictionary with data to construct dataframe allows to save data - d_submission[submission.id] = { - 'created_utc': s_datetime, - 'subreddit': submission.subreddit, - 'link_flair_text': submission.link_flair_text, - 'title':submission.title, - 'score': submission.score, - 'link': s_link, - 'num_comments': submission.num_comments, - 'upvote_ratio': submission.upvote_ratio, - 'awards': s_all_awards - } - - # Print post data collected so far - print(f"{s_datetime} - {submission.title}") - print(f"{s_link}") - t_post = PrettyTable(['Subreddit', 'Flair', 'Score', '# Comments', 'Upvote %', "Awards"]) - t_post.add_row([submission.subreddit, submission.link_flair_text, submission.score, - submission.num_comments,f"{round(100*submission.upvote_ratio)}%", s_all_awards]) - print(t_post) - print("\n") - - # Increment count of valid posts found - n_flair_posts_found += 1 - - # Check if number of wanted posts found has been reached - if n_flair_posts_found > ns_parser.n_limit-1: - break - - # Check if search_submissions didn't get anymore posts - else: - break - - if n_flair_posts_found: - lt_watchlist_sorted = sorted(d_watchlist_tickers.items(), key=lambda item: item[1], reverse=True) - s_watchlist_tickers = "" - n_tickers = 0 + if lt_watchlist_sorted: + print(f"\nThe following TOP10 tickers have been mentioned in the last {ns_parser.n_days} days:") + n_top_stocks = 0 for t_ticker in lt_watchlist_sorted: + if n_top_stocks > 9: + break try: # If try doesn't trigger exception, it means that this stock exists on finviz #thus we can print it. finviz.get_stock(t_ticker[0]) - s_watchlist_tickers += f"{t_ticker[1]} {t_ticker[0]}, " - n_tickers += 1 + print(f"{t_ticker[1]} {t_ticker[0]}") + n_top_stocks += 1 except: pass - if n_tickers: - print("The following stock tickers have been mentioned across the previous SPACs:") - print(s_watchlist_tickers[:-2]) + else: + print("No tickers found") print("") except: @@ -525,6 +400,142 @@ def spac_community(l_args): print("") +# ------------------------------------------------------------------------------------------------------------------- +def spac(l_args): + parser = argparse.ArgumentParser(prog='spac', + description=""" Show other users SPACs announcement [Reddit] """) + parser.add_argument('-l', "--limit", action="store", dest="n_limit", type=check_positive, default=5, + help='limit of posts with SPACs retrieved.') + parser.add_argument('-d', "--days", action="store", dest="n_days", type=check_positive, default=5, + help="look for the tickers from those n past days.") + + try: + (ns_parser, l_unknown_args) = parser.parse_known_args(l_args) + + if l_unknown_args: + print(f"The following args couldn't be interpreted: {l_unknown_args}\n") + return + + praw_api = praw.Reddit(client_id=cfg.API_REDDIT_CLIENT_ID, + client_secret=cfg.API_REDDIT_CLIENT_SECRET, + username=cfg.API_REDDIT_USERNAME, + user_agent=cfg.API_REDDIT_USER_AGENT, + password=cfg.API_REDDIT_PASSWORD) + + d_submission = {} + d_watchlist_tickers = {} + l_watchlist_links = list() + l_watchlist_author = list() + + n_ts_after = int((datetime.today() - timedelta(days=ns_parser.n_days)).timestamp()) + l_sub_reddits = ['pennystocks', 'RobinHoodPennyStocks', 'Daytrading', 'StockMarket', 'stocks', 'investing', 'wallstreetbets'] + + warnings.filterwarnings("ignore") # To avoid printing the warning + psaw_api = PushshiftAPI() + submissions = psaw_api.search_submissions(after=n_ts_after, + subreddit=l_sub_reddits, + q='SPAC|Spac|spac|Spacs|spacs', + filter=['id']) + n_flair_posts_found = 0 + while True: + submission = next(submissions, None) + if submission: + # Get more information about post using PRAW api + submission = praw_api.submission(id=submission.id) + + # Ensure that the post hasn't been removed by moderator in the meanwhile, + #that there is a description and it's not just an image, that the flair is + #meaningful, and that we aren't re-considering same author's watchlist + if not submission.removed_by_category and submission.selftext \ + and submission.link_flair_text not in ['Yolo', 'Meme'] \ + and submission.author.name not in l_watchlist_author: + + # Make sure there is at least 1 ticker found on the post text + l_tickers_found = set(re.findall("[A-Z]{3,5}", submission.selftext)) + if not l_tickers_found: + l_tickers_found = set(re.findall("[A-Z]{3,5}", submission.title)) + if l_tickers_found: + + # Add another author's name to the parsed watchlists + l_watchlist_author.append(submission.author.name) + + # Lookup stock tickers within a watchlist + for key in l_tickers_found: + if key in d_watchlist_tickers: + # Increment stock ticker found + d_watchlist_tickers[key] += 1 + else: + # Initialize stock ticker found + d_watchlist_tickers[key] = 1 + + l_watchlist_links.append(f"https://www.reddit.com{submission.permalink}") + # delte below, not necessary I reckon. Probably just link? + + # Refactor data + s_datetime = datetime.utcfromtimestamp(submission.created_utc).strftime("%d/%m/%Y %H:%M:%S") + s_link = f"https://www.reddit.com{submission.permalink}" + s_all_awards = "" + for award in submission.all_awardings: + s_all_awards += f"{award['count']} {award['name']}\n" + s_all_awards = s_all_awards[:-2] + + # Create dictionary with data to construct dataframe allows to save data + d_submission[submission.id] = { + 'created_utc': s_datetime, + 'subreddit': submission.subreddit, + 'link_flair_text': submission.link_flair_text, + 'title':submission.title, + 'score': submission.score, + 'link': s_link, + 'num_comments': submission.num_comments, + 'upvote_ratio': submission.upvote_ratio, + 'awards': s_all_awards + } + + # Print post data collected so far + print(f"{s_datetime} - {submission.title}") + print(f"{s_link}") + t_post = PrettyTable(['Subreddit', 'Flair', 'Score', '# Comments', 'Upvote %', "Awards"]) + t_post.add_row([submission.subreddit, submission.link_flair_text, submission.score, + submission.num_comments,f"{round(100*submission.upvote_ratio)}%", s_all_awards]) + print(t_post) + print("\n") + + # Increment count of valid posts found + n_flair_posts_found += 1 + + # Check if number of wanted posts found has been reached + if n_flair_posts_found > ns_parser.n_limit-1: + break + + # Check if search_submissions didn't get anymore posts + else: + break + + ''' + if n_flair_posts_found: + lt_watchlist_sorted = sorted(d_watchlist_tickers.items(), key=lambda item: item[1], reverse=True) + s_watchlist_tickers = "" + n_tickers = 0 + for t_ticker in lt_watchlist_sorted: + try: + # If try doesn't trigger exception, it means that this stock exists on finviz + #thus we can print it. + finviz.get_stock(t_ticker[0]) + s_watchlist_tickers += f"{t_ticker[1]} {t_ticker[0]}, " + n_tickers += 1 + except: + pass + if n_tickers: + print("The following stock tickers have been mentioned across the previous SPACs:") + print(s_watchlist_tickers[:-2]) + ''' + print("") + + except: + print("") + + # ------------------------------------------------------------------------------------------------------------------- def wsb_community(l_args): parser = argparse.ArgumentParser(prog='wsb', diff --git a/sentiment/sen_menu.py b/sentiment/sen_menu.py index 83c1a5d3a8b..392c924dfc8 100644 --- a/sentiment/sen_menu.py +++ b/sentiment/sen_menu.py @@ -19,8 +19,8 @@ def print_sentiment(): print(" wsb show what WSB gang is up to in subreddit wallstreetbets") print(" watchlist show other users watchlist") print(" popular show popular tickers") - print(" spac show other users spacs announcements") - print(" spac_c show other users spacs announcements from subreddit SPACs") + print(" spac_c show other users spacs announcements from subreddit SPACs community") + print(" spac show other users spacs announcements from other subs") print("") print("Stocktwits:") print(" bullbear estimate quick sentiment from last 30 messages on board")