jerry/OpenBB
1
0
Fork 0
mirror of https://github.com/OpenBB-finance/OpenBB.git synced 2026-11-06 17:31:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve reddit API file

Browse Source
This commit is contained in:
didier
2021-02-23 21:45:36 +00:00
parent 59c8128c73
commit bcb2262a9e
2 changed files with 166 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

317
sentiment/reddit_api.py
View File

@@ -162,12 +162,12 @@ def watchlist(l_args):
# -------------------------------------------------------------------------------------------------------------------
def popular_tickers(l_args):
parser = argparse.ArgumentParser(prog='popular',
description=""" Latest popular tickers [Reddit] """)
parser.add_argument('-l', "--limit", action="store", dest="n_limit", type=check_positive, default=30,
description="""Print latest popular tickers. [Source: Reddit] """)
parser.add_argument('-l', "--limit", action="store", dest="n_limit", type=check_positive, default=50,
help='limit of posts retrieved per sub reddit.')
parser.add_argument('-s', "--sub", action="store", dest="s_subreddit", type=str,
help="""subreddit to look for tickers. Default: pennystocks, RobinHoodPennyStocks, Daytrading,
StockMarket, stocks, investing, wallstreetbets""")
help="""subreddits to look for tickers, e.g. pennystocks,stocks.
Default: pennystocks, RobinHoodPennyStocks, Daytrading, StockMarket, stocks, investing, wallstreetbets""")
parser.add_argument('-d', "--days", action="store", dest="n_days", type=check_positive, default=1,
help="look for the tickers from those n past days.")
@@ -181,7 +181,10 @@ def popular_tickers(l_args):
n_ts_after = int((datetime.today() - timedelta(days=ns_parser.n_days)).timestamp())
if ns_parser.s_subreddit:
l_sub_reddits = [ns_parser.s_subreddit]
if ',' in ns_parser.s_subreddit:
l_sub_reddits = ns_parser.s_subreddit.split(',')
else:
l_sub_reddits = [ns_parser.s_subreddit]
else:
l_sub_reddits = ['pennystocks', 'RobinHoodPennyStocks', 'Daytrading', 'StockMarket', 'stocks', 'investing', 'wallstreetbets']
@@ -189,6 +192,7 @@ def popular_tickers(l_args):
d_watchlist_tickers = {}
l_watchlist_links = list()
l_watchlist_author = list()
ls_text = list()
praw_api = praw.Reddit(client_id=cfg.API_REDDIT_CLIENT_ID,
client_secret=cfg.API_REDDIT_CLIENT_SECRET,
@@ -216,16 +220,20 @@ def popular_tickers(l_args):
#that there is a description and it's not just an image, that the flair is
#meaningful, and that we aren't re-considering same author's content
if not submission.removed_by_category and (submission.selftext or submission.title) \
and submission.link_flair_text not in ['Yolo', 'Meme'] \
and submission.author.name not in l_watchlist_author:
ls_text.append(submission.selftext)
ls_text.append(submission.title)
# Make sure there is at least 1 ticker found on the post text
l_tickers_found = set(re.findall("[A-Z]{3,5}", submission.selftext))
if not l_tickers_found:
l_tickers_found = set(re.findall("[A-Z]{3,5}", submission.title))
submission.comments.replace_more(limit=0)
for comment in submission.comments.list():
ls_text.append(comment.body)
l_tickers_found = list()
for s_text in ls_text:
for s_ticker in set(re.findall(r'\$([A-Z]{3,5} )', s_text)):
l_tickers_found.append(s_ticker.strip())
if l_tickers_found:
n_tickers += len(l_tickers_found)
# Add another author's name to the parsed watchlists
@@ -246,156 +254,23 @@ def popular_tickers(l_args):
print(f" {n_tickers} tickers found.")
lt_watchlist_sorted = sorted(d_watchlist_tickers.items(), key=lambda item: item[1], reverse=True)
print("\nThe following TOP10 tickers have been mentioned in the last 24 hours:")
n_top_stocks = 0
for t_ticker in lt_watchlist_sorted:
if n_top_stocks > 9:
break
try:
# If try doesn't trigger exception, it means that this stock exists on finviz
#thus we can print it.
finviz.get_stock(t_ticker[0])
print(f"{t_ticker[1]} {t_ticker[0]}")
n_top_stocks += 1
except:
pass
print("")
except:
print("")
# -------------------------------------------------------------------------------------------------------------------
def spac(l_args):
parser = argparse.ArgumentParser(prog='spac',
description=""" Show other users SPACs announcement [Reddit] """)
parser.add_argument('-l', "--limit", action="store", dest="n_limit", type=check_positive, default=5,
help='limit of posts with SPACs retrieved')
parser.add_argument('-d', "--days", action="store", dest="n_days", type=check_positive, default=5,
help="look for the tickers from those n past days")
try:
(ns_parser, l_unknown_args) = parser.parse_known_args(l_args)
if l_unknown_args:
print(f"The following args couldn't be interpreted: {l_unknown_args}\n")
return
praw_api = praw.Reddit(client_id=cfg.API_REDDIT_CLIENT_ID,
client_secret=cfg.API_REDDIT_CLIENT_SECRET,
username=cfg.API_REDDIT_USERNAME,
user_agent=cfg.API_REDDIT_USER_AGENT,
password=cfg.API_REDDIT_PASSWORD)
d_submission = {}
d_watchlist_tickers = {}
l_watchlist_links = list()
l_watchlist_author = list()
n_ts_after = int((datetime.today() - timedelta(days=ns_parser.n_days)).timestamp())
l_sub_reddits = ['pennystocks', 'RobinHoodPennyStocks', 'Daytrading', 'StockMarket', 'stocks', 'investing', 'wallstreetbets']
warnings.filterwarnings("ignore") # To avoid printing the warning
psaw_api = PushshiftAPI()
submissions = psaw_api.search_submissions(after=n_ts_after,
subreddit=l_sub_reddits,
q='SPAC|Spac|spac|Spacs|spacs',
filter=['id'])
n_flair_posts_found = 0
while True:
submission = next(submissions, None)
if submission:
# Get more information about post using PRAW api
submission = praw_api.submission(id=submission.id)
# Ensure that the post hasn't been removed by moderator in the meanwhile,
#that there is a description and it's not just an image, that the flair is
#meaningful, and that we aren't re-considering same author's watchlist
if not submission.removed_by_category and submission.selftext \
and submission.link_flair_text not in ['Yolo', 'Meme'] \
and submission.author.name not in l_watchlist_author:
# Make sure there is at least 1 ticker found on the post text
l_tickers_found = set(re.findall("[A-Z]{3,5}", submission.selftext))
if not l_tickers_found:
l_tickers_found = set(re.findall("[A-Z]{3,5}", submission.title))
if l_tickers_found:
# Add another author's name to the parsed watchlists
l_watchlist_author.append(submission.author.name)
# Lookup stock tickers within a watchlist
for key in l_tickers_found:
if key in d_watchlist_tickers:
# Increment stock ticker found
d_watchlist_tickers[key] += 1
else:
# Initialize stock ticker found
d_watchlist_tickers[key] = 1
l_watchlist_links.append(f"https://www.reddit.com{submission.permalink}")
# delte below, not necessary I reckon. Probably just link?
# Refactor data
s_datetime = datetime.utcfromtimestamp(submission.created_utc).strftime("%d/%m/%Y %H:%M:%S")
s_link = f"https://www.reddit.com{submission.permalink}"
s_all_awards = ""
for award in submission.all_awardings:
s_all_awards += f"{award['count']} {award['name']}\n"
s_all_awards = s_all_awards[:-2]
# Create dictionary with data to construct dataframe allows to save data
d_submission[submission.id] = {
'created_utc': s_datetime,
'subreddit': submission.subreddit,
'link_flair_text': submission.link_flair_text,
'title':submission.title,
'score': submission.score,
'link': s_link,
'num_comments': submission.num_comments,
'upvote_ratio': submission.upvote_ratio,
'awards': s_all_awards
}
# Print post data collected so far
print(f"{s_datetime} - {submission.title}")
print(f"{s_link}")
t_post = PrettyTable(['Subreddit', 'Flair', 'Score', '# Comments', 'Upvote %', "Awards"])
t_post.add_row([submission.subreddit, submission.link_flair_text, submission.score,
submission.num_comments,f"{round(100*submission.upvote_ratio)}%", s_all_awards])
print(t_post)
print("\n")
# Increment count of valid posts found
n_flair_posts_found += 1
# Check if number of wanted posts found has been reached
if n_flair_posts_found > ns_parser.n_limit-1:
break
# Check if search_submissions didn't get anymore posts
else:
break
if n_flair_posts_found:
lt_watchlist_sorted = sorted(d_watchlist_tickers.items(), key=lambda item: item[1], reverse=True)
s_watchlist_tickers = ""
n_tickers = 0
if lt_watchlist_sorted:
print(f"\nThe following TOP10 tickers have been mentioned in the last {ns_parser.n_days} days:")
n_top_stocks = 0
for t_ticker in lt_watchlist_sorted:
if n_top_stocks > 9:
break
try:
# If try doesn't trigger exception, it means that this stock exists on finviz
#thus we can print it.
finviz.get_stock(t_ticker[0])
s_watchlist_tickers += f"{t_ticker[1]} {t_ticker[0]}, "
n_tickers += 1
print(f"{t_ticker[1]} {t_ticker[0]}")
n_top_stocks += 1
except:
pass
if n_tickers:
print("The following stock tickers have been mentioned across the previous SPACs:")
print(s_watchlist_tickers[:-2])
else:
print("No tickers found")
print("")
except:
@@ -525,6 +400,142 @@ def spac_community(l_args):
print("")
# -------------------------------------------------------------------------------------------------------------------
def spac(l_args):
parser = argparse.ArgumentParser(prog='spac',
description=""" Show other users SPACs announcement [Reddit] """)
parser.add_argument('-l', "--limit", action="store", dest="n_limit", type=check_positive, default=5,
help='limit of posts with SPACs retrieved.')
parser.add_argument('-d', "--days", action="store", dest="n_days", type=check_positive, default=5,
help="look for the tickers from those n past days.")
try:
(ns_parser, l_unknown_args) = parser.parse_known_args(l_args)
if l_unknown_args:
print(f"The following args couldn't be interpreted: {l_unknown_args}\n")
return
praw_api = praw.Reddit(client_id=cfg.API_REDDIT_CLIENT_ID,
client_secret=cfg.API_REDDIT_CLIENT_SECRET,
username=cfg.API_REDDIT_USERNAME,
user_agent=cfg.API_REDDIT_USER_AGENT,
password=cfg.API_REDDIT_PASSWORD)
d_submission = {}
d_watchlist_tickers = {}
l_watchlist_links = list()
l_watchlist_author = list()
n_ts_after = int((datetime.today() - timedelta(days=ns_parser.n_days)).timestamp())
l_sub_reddits = ['pennystocks', 'RobinHoodPennyStocks', 'Daytrading', 'StockMarket', 'stocks', 'investing', 'wallstreetbets']
warnings.filterwarnings("ignore") # To avoid printing the warning
psaw_api = PushshiftAPI()
submissions = psaw_api.search_submissions(after=n_ts_after,
subreddit=l_sub_reddits,
q='SPAC|Spac|spac|Spacs|spacs',
filter=['id'])
n_flair_posts_found = 0
while True:
submission = next(submissions, None)
if submission:
# Get more information about post using PRAW api
submission = praw_api.submission(id=submission.id)
# Ensure that the post hasn't been removed by moderator in the meanwhile,
#that there is a description and it's not just an image, that the flair is
#meaningful, and that we aren't re-considering same author's watchlist
if not submission.removed_by_category and submission.selftext \
and submission.link_flair_text not in ['Yolo', 'Meme'] \
and submission.author.name not in l_watchlist_author:
# Make sure there is at least 1 ticker found on the post text
l_tickers_found = set(re.findall("[A-Z]{3,5}", submission.selftext))
if not l_tickers_found:
l_tickers_found = set(re.findall("[A-Z]{3,5}", submission.title))
if l_tickers_found:
# Add another author's name to the parsed watchlists
l_watchlist_author.append(submission.author.name)
# Lookup stock tickers within a watchlist
for key in l_tickers_found:
if key in d_watchlist_tickers:
# Increment stock ticker found
d_watchlist_tickers[key] += 1
else:
# Initialize stock ticker found
d_watchlist_tickers[key] = 1
l_watchlist_links.append(f"https://www.reddit.com{submission.permalink}")
# delte below, not necessary I reckon. Probably just link?
# Refactor data
s_datetime = datetime.utcfromtimestamp(submission.created_utc).strftime("%d/%m/%Y %H:%M:%S")
s_link = f"https://www.reddit.com{submission.permalink}"
s_all_awards = ""
for award in submission.all_awardings:
s_all_awards += f"{award['count']} {award['name']}\n"
s_all_awards = s_all_awards[:-2]
# Create dictionary with data to construct dataframe allows to save data
d_submission[submission.id] = {
'created_utc': s_datetime,
'subreddit': submission.subreddit,
'link_flair_text': submission.link_flair_text,
'title':submission.title,
'score': submission.score,
'link': s_link,
'num_comments': submission.num_comments,
'upvote_ratio': submission.upvote_ratio,
'awards': s_all_awards
}
# Print post data collected so far
print(f"{s_datetime} - {submission.title}")
print(f"{s_link}")
t_post = PrettyTable(['Subreddit', 'Flair', 'Score', '# Comments', 'Upvote %', "Awards"])
t_post.add_row([submission.subreddit, submission.link_flair_text, submission.score,
submission.num_comments,f"{round(100*submission.upvote_ratio)}%", s_all_awards])
print(t_post)
print("\n")
# Increment count of valid posts found
n_flair_posts_found += 1
# Check if number of wanted posts found has been reached
if n_flair_posts_found > ns_parser.n_limit-1:
break
# Check if search_submissions didn't get anymore posts
else:
break
'''
if n_flair_posts_found:
lt_watchlist_sorted = sorted(d_watchlist_tickers.items(), key=lambda item: item[1], reverse=True)
s_watchlist_tickers = ""
n_tickers = 0
for t_ticker in lt_watchlist_sorted:
try:
# If try doesn't trigger exception, it means that this stock exists on finviz
#thus we can print it.
finviz.get_stock(t_ticker[0])
s_watchlist_tickers += f"{t_ticker[1]} {t_ticker[0]}, "
n_tickers += 1
except:
pass
if n_tickers:
print("The following stock tickers have been mentioned across the previous SPACs:")
print(s_watchlist_tickers[:-2])
'''
print("")
except:
print("")
# -------------------------------------------------------------------------------------------------------------------
def wsb_community(l_args):
parser = argparse.ArgumentParser(prog='wsb',

4
sentiment/sen_menu.py
View File

@@ -19,8 +19,8 @@ def print_sentiment():
print(" wsb show what WSB gang is up to in subreddit wallstreetbets")
print(" watchlist show other users watchlist")
print(" popular show popular tickers")
print(" spac show other users spacs announcements")
print(" spac_c show other users spacs announcements from subreddit SPACs")
print(" spac_c show other users spacs announcements from subreddit SPACs community")
print(" spac show other users spacs announcements from other subs")
print("")
print("Stocktwits:")
print(" bullbear estimate quick sentiment from last 30 messages on board")