jerry/OpenBB
1
0
Fork 0
mirror of https://github.com/OpenBB-finance/OpenBB.git synced 2026-10-27 12:40:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

remove twitter functions

Browse Source
This commit is contained in:
James Maslek
2023-08-11 11:26:35 -04:00
parent 673afcf405
commit 6502a358db
7 changed files with 0 additions and 542 deletions
Download Patch File Download Diff File Expand all files Collapse all files

212
openbb_terminal/common/behavioural_analysis/twitter_model.py
View File

@@ -1,212 +0,0 @@
"""Twitter Model"""
__docformat__ = "numpy"
import logging
from datetime import datetime, timedelta
from typing import Optional
import pandas as pd
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
from openbb_terminal.core.session.current_user import get_current_user
from openbb_terminal.decorators import check_api_key, log_start_end
from openbb_terminal.helper_funcs import clean_tweet, get_data, request
from openbb_terminal.rich_config import console
logger = logging.getLogger(__name__)
analyzer = SentimentIntensityAnalyzer()
@log_start_end(log=logger)
@check_api_key(["API_TWITTER_BEARER_TOKEN"])
def load_analyze_tweets(
symbol: str,
limit: int = 100,
start_date: Optional[str] = "",
end_date: Optional[str] = "",
) -> pd.DataFrame:
"""Load tweets from twitter API and analyzes using VADER.
Parameters
----------
symbol: str
Ticker symbol to search twitter for
limit: int
Number of tweets to analyze
start_date: Optional[str]
If given, the start time to get tweets from
end_date: Optional[str]
If given, the end time to get tweets from
Returns
-------
df_tweet: pd.DataFrame
Dataframe of tweets and sentiment
"""
params = {
"query": rf"(\${symbol}) (lang:en)",
"max_results": str(limit),
"tweet.fields": "created_at,lang",
}
if start_date:
# Assign from and to datetime parameters for the API
params["start_time"] = start_date
if end_date:
params["end_time"] = end_date
# Request Twitter API
response = request(
"https://api.twitter.com/2/tweets/search/recent",
params=params, # type: ignore
headers={
"authorization": "Bearer "
+ get_current_user().credentials.API_TWITTER_BEARER_TOKEN
},
)
# Create dataframe
df_tweets = pd.DataFrame()
# Check that the API response was successful
if response.status_code == 200:
tweets = []
for tweet in response.json()["data"]:
row = get_data(tweet)
tweets.append(row)
df_tweets = pd.DataFrame(tweets)
elif response.status_code == 401:
console.print("Twitter API Key provided is incorrect\n")
return pd.DataFrame()
elif response.status_code == 400:
console.print(
"""
Status Code 400.
This means you are requesting data from beyond the API's 7 day limit"""
)
return pd.DataFrame()
elif response.status_code == 403:
console.print(
f"""
Status code 403.
It seems you're twitter credentials are invalid - {response.text}
"""
)
return pd.DataFrame()
else:
console.print(
f"""
Status code {response.status_code}.
Something went wrong - {response.text}
"""
)
return pd.DataFrame()
sentiments = []
pos = []
neg = []
neu = []
for s_tweet in df_tweets["text"].to_list():
tweet = clean_tweet(s_tweet, symbol)
sentiments.append(analyzer.polarity_scores(tweet)["compound"])
pos.append(analyzer.polarity_scores(tweet)["pos"])
neg.append(analyzer.polarity_scores(tweet)["neg"])
neu.append(analyzer.polarity_scores(tweet)["neu"])
# Add sentiments to tweets dataframe
df_tweets["sentiment"] = sentiments
df_tweets["positive"] = pos
df_tweets["negative"] = neg
df_tweets["neutral"] = neu
return df_tweets
@log_start_end(log=logger)
def get_sentiment(
symbol: str,
n_tweets: int = 15,
n_days_past: int = 2,
) -> pd.DataFrame:
"""Get sentiments from symbol.
Parameters
----------
symbol: str
Stock ticker symbol to get sentiment for
n_tweets: int
Number of tweets to get per hour
n_days_past: int
Number of days to extract tweets for
Returns
-------
df_sentiment: pd.DataFrame
Dataframe of sentiment
"""
# Date format string required by twitter
dt_format = "%Y-%m-%dT%H:%M:%SZ"
# Algorithm to extract
dt_recent = datetime.utcnow() - timedelta(seconds=20)
dt_old = dt_recent - timedelta(days=n_days_past)
console.print(
f"From {dt_recent.date()} retrieving {n_tweets*24} tweets ({n_tweets} tweets/hour)"
)
df_tweets = pd.DataFrame(
columns=[
"created_at",
"text",
"sentiment",
"positive",
"negative",
"neutral",
]
)
while True:
# Iterate until we haven't passed the old number of days
if dt_recent < dt_old:
break
# Update past datetime
dt_past = dt_recent - timedelta(minutes=60)
temp = load_analyze_tweets(
symbol,
n_tweets,
start_date=dt_past.strftime(dt_format),
end_date=dt_recent.strftime(dt_format),
)
if (isinstance(temp, pd.DataFrame) and temp.empty) or (
not isinstance(temp, pd.DataFrame) and not temp
):
return pd.DataFrame()
df_tweets = pd.concat([df_tweets, temp])
if dt_past.day < dt_recent.day:
console.print(
f"From {dt_past.date()} retrieving {n_tweets*24} tweets ({n_tweets} tweets/hour)"
)
# Update recent datetime
dt_recent = dt_past
# Sort tweets per date
df_tweets.sort_index(ascending=False, inplace=True)
df_tweets["cumulative_compound"] = df_tweets["sentiment"].cumsum()
df_tweets["prob_sen"] = 1
# df_tweets.to_csv(r'notebooks/tweets.csv', index=False)
df_tweets.reset_index(inplace=True)
df_tweets["Month"] = pd.to_datetime(df_tweets["created_at"]).apply(
lambda x: x.month
)
df_tweets["Day"] = pd.to_datetime(df_tweets["created_at"]).apply(lambda x: x.day)
df_tweets["date"] = pd.to_datetime(df_tweets["created_at"])
df_tweets = df_tweets.sort_values(by="date")
df_tweets["cumulative_compound"] = df_tweets["sentiment"].cumsum()
return df_tweets

197
openbb_terminal/common/behavioural_analysis/twitter_view.py
View File

@@ -1,197 +0,0 @@
"""Twitter view."""
__docformat__ = "numpy"
import logging
import os
from typing import Optional, Union
import numpy as np
import pandas as pd
from dateutil import parser as dparse
from openbb_terminal import OpenBBFigure, theme
from openbb_terminal.common.behavioural_analysis import twitter_model
from openbb_terminal.decorators import log_start_end
from openbb_terminal.helper_funcs import export_data, get_closing_price
from openbb_terminal.rich_config import console
logger = logging.getLogger(__name__)
@log_start_end(log=logger)
def display_inference(
symbol: str, limit: int = 100, export: str = "", sheet_name: Optional[str] = None
):
"""Prints Inference sentiment from past n tweets.
Parameters
----------
symbol: str
Stock ticker symbol
limit: int
Number of tweets to analyze
sheet_name: str
Optionally specify the name of the sheet the data is exported to.
export: str
Format to export tweet dataframe
"""
df_tweets = twitter_model.load_analyze_tweets(symbol, limit)
if (isinstance(df_tweets, pd.DataFrame) and df_tweets.empty) or (
not isinstance(df_tweets, pd.DataFrame) and not df_tweets
):
return
# Parse tweets
dt_from = dparse.parse(df_tweets["created_at"].values[-1])
dt_to = dparse.parse(df_tweets["created_at"].values[0])
console.print(f"From: {dt_from.strftime('%Y-%m-%d %H:%M:%S')}")
console.print(f"To: {dt_to.strftime('%Y-%m-%d %H:%M:%S')}")
console.print(f"{len(df_tweets)} tweets were analyzed.")
dt_delta = dt_to - dt_from
n_freq = dt_delta.total_seconds() / len(df_tweets)
console.print(f"Frequency of approx 1 tweet every {round(n_freq)} seconds.")
pos = df_tweets["positive"]
neg = df_tweets["negative"]
percent_pos = len(np.where(pos > neg)[0]) / len(df_tweets)
percent_neg = len(np.where(pos < neg)[0]) / len(df_tweets)
total_sent = np.round(np.sum(df_tweets["sentiment"]), 2)
mean_sent = np.round(np.mean(df_tweets["sentiment"]), 2)
console.print(f"The summed compound sentiment of {symbol} is: {total_sent}")
console.print(f"The average compound sentiment of {symbol} is: {mean_sent}")
console.print(
f"Of the last {len(df_tweets)} tweets, {100*percent_pos:.2f} % had a higher positive sentiment"
)
console.print(
f"Of the last {len(df_tweets)} tweets, {100*percent_neg:.2f} % had a higher negative sentiment"
)
export_data(
export,
os.path.dirname(os.path.abspath(__file__)),
"infer",
df_tweets,
sheet_name,
)
@log_start_end(log=logger)
def display_sentiment(
symbol: str,
n_tweets: int = 15,
n_days_past: int = 2,
compare: bool = False,
export: str = "",
sheet_name: Optional[str] = None,
external_axes: bool = False,
) -> Union[OpenBBFigure, None]:
"""Plots sentiments from symbol
Parameters
----------
symbol: str
Stock ticker symbol to get sentiment for
n_tweets: int
Number of tweets to get per hour
n_days_past: int
Number of days to extract tweets for
compare: bool
Show corresponding change in stock price
sheet_name: str
Optionally specify the name of the sheet the data is exported to.
export: str
Format to export tweet dataframe
external_axes: bool, optional
Whether to return the figure object or not, by default False
"""
df_tweets = twitter_model.get_sentiment(symbol, n_tweets, n_days_past)
if df_tweets.empty:
return None
if compare:
plots_kwargs = dict(
rows=3,
cols=1,
shared_xaxes=True,
vertical_spacing=0.05,
row_heights=[0.2, 0.6, 0.2],
)
else:
plots_kwargs = dict(
rows=2,
cols=1,
shared_xaxes=True,
vertical_spacing=0.05,
row_heights=[0.6, 0.4],
)
fig = OpenBBFigure.create_subplots(**plots_kwargs) # type: ignore
fig.add_scatter(
x=pd.to_datetime(df_tweets["created_at"]),
y=df_tweets["cumulative_compound"].values,
row=1,
col=1,
)
fig.set_yaxis_title("<br>Cumulative<br>VADER Sentiment", row=1, col=1)
for _, day_df in df_tweets.groupby(by="Day"):
day_df["time"] = pd.to_datetime(day_df["created_at"])
day_df = day_df.sort_values(by="time")
fig.add_scatter(
x=day_df["time"],
y=day_df["sentiment"].cumsum(),
row=1,
col=1,
)
fig.add_bar(
x=df_tweets["date"],
y=df_tweets["positive"],
row=2,
col=1,
marker_color=theme.up_color,
)
fig.add_bar(
x=df_tweets["date"],
y=-1 * df_tweets["negative"],
row=2,
col=1,
marker_color=theme.down_color,
)
fig.set_yaxis_title("VADER Polarity Scores", row=2, col=1)
if compare:
# get stock end price for each corresponding day if compare == True
closing_price_df = get_closing_price(symbol, n_days_past)
fig.add_scatter(
x=closing_price_df["Date"],
y=closing_price_df["Close"],
name=pd.to_datetime(closing_price_df["Date"]).iloc[0].strftime("%Y-%m-%d"),
row=3,
col=1,
)
fig.set_yaxis_title("Stock Price", row=3, col=1)
fig.update_layout(
title=f"Twitter's {symbol} total compound sentiment over time is {round(np.sum(df_tweets['sentiment']), 2)}",
xaxis=dict(type="date"),
showlegend=False,
)
export_data(
export,
os.path.dirname(os.path.abspath(__file__)),
"sentiment",
df_tweets,
sheet_name,
fig,
)
return fig.show(external=external_axes)

106
openbb_terminal/stocks/behavioural_analysis/ba_controller.py
View File

@@ -13,7 +13,6 @@ from openbb_terminal.common.behavioural_analysis import (
google_view,
reddit_view,
stocktwits_view,
twitter_view,
)
from openbb_terminal.core.session.current_user import get_current_user
from openbb_terminal.custom_prompt_toolkit import NestedCompleter
@@ -21,7 +20,6 @@ from openbb_terminal.decorators import log_start_end
from openbb_terminal.helper_funcs import (
EXPORT_BOTH_RAW_DATA_AND_FIGURES,
EXPORT_ONLY_RAW_DATA_ALLOWED,
check_int_range,
check_non_negative,
check_positive,
valid_date,
@@ -53,8 +51,6 @@ class BehaviouralAnalysisController(StockBaseController):
"messages",
"trending",
"stalker",
"infer",
"sentiment",
"mentions",
"regions",
"queries",
@@ -102,8 +98,6 @@ class BehaviouralAnalysisController(StockBaseController):
mt.add_cmd("stalker")
mt.add_cmd("bullbear", self.ticker)
mt.add_cmd("messages", self.ticker)
mt.add_cmd("infer", self.ticker)
mt.add_cmd("sentiment", self.ticker)
mt.add_cmd("mentions", self.ticker)
mt.add_cmd("regions", self.ticker)
mt.add_cmd("interest", self.ticker)
@@ -673,106 +667,6 @@ class BehaviouralAnalysisController(StockBaseController):
else:
console.print("No ticker loaded. Please load using 'load <ticker>'\n")
@log_start_end(log=logger)
def call_infer(self, other_args: List[str]):
"""Process infer command."""
parser = argparse.ArgumentParser(
add_help=False,
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
prog="infer",
description="""
Print quick sentiment inference from last tweets that contain the ticker.
This model splits the text into character-level tokens and uses vader sentiment analysis.
[Source: Twitter]
""",
)
parser.add_argument(
"-l",
"--limit",
action="store",
dest="limit",
type=check_int_range(10, 100),
default=100,
help="limit of latest tweets to infer from.",
)
if other_args and "-" not in other_args[0][0]:
other_args.insert(0, "-l")
ns_parser = self.parse_known_args_and_warn(
parser, other_args, EXPORT_ONLY_RAW_DATA_ALLOWED
)
if ns_parser:
if self.ticker:
twitter_view.display_inference(
symbol=self.ticker,
limit=ns_parser.limit,
export=ns_parser.export,
sheet_name=" ".join(ns_parser.sheet_name)
if ns_parser.sheet_name
else None,
)
else:
console.print("No ticker loaded. Please load using 'load <ticker>'\n")
@log_start_end(log=logger)
def call_sentiment(self, other_args: List[str]):
"""Process sentiment command."""
parser = argparse.ArgumentParser(
add_help=False,
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
prog="sentiment",
description="""
Plot in-depth sentiment predicted from tweets from last days
that contain pre-defined ticker. [Source: Twitter]
""",
)
# in reality this argument could be 100, but after testing it takes too long
# to compute which may not be acceptable
# TODO: use https://github.com/twintproject/twint instead of twitter API
parser.add_argument(
"-l",
"--limit",
action="store",
dest="limit",
type=check_int_range(10, 62),
default=15,
help="limit of tweets to extract per hour.",
)
parser.add_argument(
"-d",
"--days",
action="store",
dest="n_days_past",
type=check_int_range(1, 6),
default=6,
help="number of days in the past to extract tweets.",
)
parser.add_argument(
"-c",
"--compare",
action="store_true",
dest="compare",
help="show corresponding change in stock price",
)
if other_args and "-" not in other_args[0][0]:
other_args.insert(0, "-l")
ns_parser = self.parse_known_args_and_warn(
parser, other_args, EXPORT_BOTH_RAW_DATA_AND_FIGURES
)
if ns_parser:
if self.ticker:
twitter_view.display_sentiment(
symbol=self.ticker,
n_tweets=ns_parser.limit,
n_days_past=ns_parser.n_days_past,
compare=ns_parser.compare,
export=ns_parser.export,
sheet_name=" ".join(ns_parser.sheet_name)
if ns_parser.sheet_name
else None,
)
else:
console.print("No ticker loaded. Please load using 'load <ticker>'\n")
@log_start_end(log=logger)
def call_headlines(self, other_args: List[str]):
"""Process finbrain command."""

0
tests/openbb_terminal/common/behavioural_analysis/test_twitter_model.py
View File

0
tests/openbb_terminal/common/behavioural_analysis/test_twitter_view.py
View File

25
tests/openbb_terminal/stocks/behavioural_analysis/test_ba_controller.py
View File

@@ -297,27 +297,6 @@ def test_call_func_expect_queue(expected_queue, queue, func):
limit=5,
),
),
(
"call_infer",
["--limit=20", "--export=csv"],
"twitter_view.display_inference",
[],
dict(symbol="MOCK_TICKER", limit=20, export="csv", sheet_name=None),
),
(
"call_sentiment",
["--limit=20", "--days=2", "--compare", "--export=csv"],
"twitter_view.display_sentiment",
[],
dict(
symbol="MOCK_TICKER",
n_tweets=20,
n_days_past=2,
compare=True,
export="csv",
sheet_name=None,
),
),
(
"call_mentions",
["--start=2020-12-01", "--export=csv"],
@@ -422,8 +401,6 @@ def test_call_func(
"call_messages",
"call_trending",
"call_stalker",
"call_infer",
"call_sentiment",
"call_mentions",
"call_regions",
"call_queries",
@@ -456,8 +433,6 @@ def test_call_func_no_parser(func, mocker):
"func",
[
"call_headlines",
"call_sentiment",
"call_infer",
"call_rise",
"call_queries",
"call_regions",

2
tests/openbb_terminal/stocks/behavioural_analysis/txt/test_ba_controller/test_print_help.txt vendored
View File

@@ -12,8 +12,6 @@ Ticker: TSLA
stalker stalk stocktwits user's last messages [Stocktwits]
bullbear estimate quick sentiment from last 30 messages on board [Stocktwits]
messages output up to the 30 last messages on the board [Stocktwits]
infer infer about stock's sentiment from latest tweets [Twitter]
sentiment in-depth sentiment prediction from tweets over time [Twitter]
mentions interest over time based on stock's mentions [Google]
regions regions that show highest interest in stock [Google]
interest interest over time of sentences versus stock price [Google]