From 8926a199c04d7d81f5f79da0a69eaaa48323d979 Mon Sep 17 00:00:00 2001 From: Artem Veremey Date: Sat, 27 Feb 2021 13:46:19 -0800 Subject: [PATCH] Adding randomized User Agent to BeautifulSoup calls. Adding type hints --- gamestonk_terminal/discovery/fidelity_api.py | 8 +++- .../discovery/seeking_alpha_api.py | 8 ++-- .../discovery/short_interest_api.py | 7 +++- .../due_diligence/business_insider_api.py | 39 ++++++++++++------- .../business_insider_api.py | 16 ++++---- .../fundamental_analysis/market_watch_api.py | 31 ++++++++++++--- gamestonk_terminal/helper_funcs.py | 33 +++++++++++----- 7 files changed, 100 insertions(+), 42 deletions(-) diff --git a/gamestonk_terminal/discovery/fidelity_api.py b/gamestonk_terminal/discovery/fidelity_api.py index e1d8a9bd78c..d6dad0805a0 100644 --- a/gamestonk_terminal/discovery/fidelity_api.py +++ b/gamestonk_terminal/discovery/fidelity_api.py @@ -2,7 +2,7 @@ import argparse from bs4 import BeautifulSoup import requests import pandas as pd -from gamestonk_terminal.helper_funcs import check_positive +from gamestonk_terminal.helper_funcs import check_positive, get_user_agent # ---------------------------------------------------- ORDERS ---------------------------------------------------- def orders(l_args): @@ -35,7 +35,11 @@ def orders(l_args): url_orders = ( "https://eresearch.fidelity.com/eresearch/gotoBL/fidelityTopOrders.jhtml" ) - text_soup_url_orders = BeautifulSoup(requests.get(url_orders).text, "lxml") + + text_soup_url_orders = BeautifulSoup( + requests.get(url_orders, headers={"User-Agent": get_user_agent()}).text, + "lxml", + ) l_orders = list() l_orders_vals = list() diff --git a/gamestonk_terminal/discovery/seeking_alpha_api.py b/gamestonk_terminal/discovery/seeking_alpha_api.py index b627d40e2a7..cd08be749f6 100644 --- a/gamestonk_terminal/discovery/seeking_alpha_api.py +++ b/gamestonk_terminal/discovery/seeking_alpha_api.py @@ -2,7 +2,7 @@ import argparse from bs4 import BeautifulSoup import requests import pandas as pd -from gamestonk_terminal.helper_funcs import check_positive +from gamestonk_terminal.helper_funcs import check_positive, get_user_agent # ------------------------------------------------ EARNINGS_RELEASE_DATES ------------------------------------------------- @@ -48,7 +48,10 @@ def earnings_release_dates(l_args): f"https://seekingalpha.com/earnings/earnings-calendar/{idx+1}" ) text_soup_earnings = BeautifulSoup( - requests.get(url_next_earnings).text, "lxml" + requests.get( + url_next_earnings, headers={"User-Agent": get_user_agent()} + ).text, + "lxml", ) for bs_stock in text_soup_earnings.findAll( @@ -76,7 +79,6 @@ def earnings_release_dates(l_args): ].to_string(index=False, header=False) ) print("") - except SystemExit: print("") return diff --git a/gamestonk_terminal/discovery/short_interest_api.py b/gamestonk_terminal/discovery/short_interest_api.py index 998d97807e5..ca500cfeee2 100644 --- a/gamestonk_terminal/discovery/short_interest_api.py +++ b/gamestonk_terminal/discovery/short_interest_api.py @@ -3,7 +3,7 @@ import re import requests from bs4 import BeautifulSoup import pandas as pd -from gamestonk_terminal.helper_funcs import check_positive +from gamestonk_terminal.helper_funcs import check_positive, get_user_agent # ------------------------------------------------ HIGH_SHORT_INTEREST ------------------------------------------------- @@ -111,7 +111,10 @@ def low_float(l_args): url_high_short_interested_stocks = "https://www.lowfloat.com" text_soup_low_float_stocks = BeautifulSoup( - requests.get(url_high_short_interested_stocks).text, "lxml" + requests.get( + url_high_short_interested_stocks, headers={"User-Agent": get_user_agent()} + ).text, + "lxml", ) a_low_float_header = list() diff --git a/gamestonk_terminal/due_diligence/business_insider_api.py b/gamestonk_terminal/due_diligence/business_insider_api.py index 71f5578d466..fc83a4ebf4a 100644 --- a/gamestonk_terminal/due_diligence/business_insider_api.py +++ b/gamestonk_terminal/due_diligence/business_insider_api.py @@ -1,16 +1,20 @@ +import argparse +from datetime import datetime +import json +import re +import requests +import matplotlib.pyplot as plt +import pandas as pd +from pandas.plotting import register_matplotlib_converters from bs4 import BeautifulSoup from alpha_vantage.timeseries import TimeSeries -from gamestonk_terminal.helper_funcs import * -import requests -import pandas as pd -import re -import json -from gamestonk_terminal import config_terminal as cfg -from datetime import datetime -import argparse from fuzzywuzzy import fuzz -import matplotlib.pyplot as plt -from pandas.plotting import register_matplotlib_converters +from gamestonk_terminal.helper_funcs import ( + check_positive, + get_next_stock_market_days, + get_user_agent, +) +from gamestonk_terminal import config_terminal as cfg register_matplotlib_converters() @@ -43,7 +47,10 @@ def price_target_from_analysts(l_args, df_stock, s_ticker, s_start, s_interval): f"https://markets.businessinsider.com/stocks/{s_ticker.lower()}-stock" ) text_soup_market_business_insider = BeautifulSoup( - requests.get(url_market_business_insider).text, "lxml" + requests.get( + url_market_business_insider, headers={"User-Agent": get_user_agent()} + ).text, + "lxml", ) for script in text_soup_market_business_insider.find_all("script"): @@ -129,7 +136,10 @@ def estimates(l_args, s_ticker): f"https://markets.businessinsider.com/stocks/{s_ticker.lower()}-stock" ) text_soup_market_business_insider = BeautifulSoup( - requests.get(url_market_business_insider).text, "lxml" + requests.get( + url_market_business_insider, headers={"User-Agent": get_user_agent()} + ).text, + "lxml", ) l_estimates_year_header = list() @@ -295,7 +305,10 @@ def insider_activity(l_args, df_stock, s_ticker, s_start, s_interval): f"https://markets.businessinsider.com/stocks/{s_ticker.lower()}-stock" ) text_soup_market_business_insider = BeautifulSoup( - requests.get(url_market_business_insider).text, "lxml" + requests.get( + url_market_business_insider, headers={"User-Agent": get_user_agent()} + ).text, + "lxml", ) d_insider = dict() diff --git a/gamestonk_terminal/fundamental_analysis/business_insider_api.py b/gamestonk_terminal/fundamental_analysis/business_insider_api.py index 97b26805c6d..f0fb5ac8c7a 100644 --- a/gamestonk_terminal/fundamental_analysis/business_insider_api.py +++ b/gamestonk_terminal/fundamental_analysis/business_insider_api.py @@ -1,20 +1,22 @@ -from bs4 import BeautifulSoup -import requests -import pandas as pd +import argparse import re import json -from gamestonk_terminal import config_terminal as cfg from datetime import datetime -import argparse +import requests +from bs4 import BeautifulSoup +import pandas as pd from fuzzywuzzy import fuzz +from gamestonk_terminal import config_terminal as cfg # ---------------------------------------------------- MANAGEMENT ---------------------------------------------------- def management(l_args, s_ticker): parser = argparse.ArgumentParser( prog="mgmt", - description="""Print management team. Namely: Name, Title, Information from google and - (potentially) Insider Activity page. [Source: Business Insider]""", + description=""" + Print management team. Namely: Name, Title, Information from google and + (potentially) Insider Activity page. [Source: Business Insider] + """, ) try: diff --git a/gamestonk_terminal/fundamental_analysis/market_watch_api.py b/gamestonk_terminal/fundamental_analysis/market_watch_api.py index dd25073cb72..5348bd96113 100644 --- a/gamestonk_terminal/fundamental_analysis/market_watch_api.py +++ b/gamestonk_terminal/fundamental_analysis/market_watch_api.py @@ -2,6 +2,7 @@ import argparse import pandas as pd from bs4 import BeautifulSoup import requests +from gamestonk_terminal.helper_funcs import get_user_agent # ---------------------------------------------------- INCOME ---------------------------------------------------- def income(l_args, s_ticker): @@ -48,7 +49,10 @@ def income(l_args, s_ticker): else: url_financials = f"https://www.marketwatch.com/investing/stock/{s_ticker}/financials/income" - text_soup_financials = BeautifulSoup(requests.get(url_financials).text, "lxml") + text_soup_financials = BeautifulSoup( + requests.get(url_financials, headers={"User-Agent": get_user_agent()}).text, + "lxml", + ) # Define financials columns a_financials_header = list() @@ -119,7 +123,10 @@ def assets(l_args, s_ticker): else: url_financials = f"https://www.marketwatch.com/investing/stock/{s_ticker}/financials/balance-sheet" - text_soup_financials = BeautifulSoup(requests.get(url_financials).text, "lxml") + text_soup_financials = BeautifulSoup( + requests.get(url_financials, headers={"User-Agent": get_user_agent()}).text, + "lxml", + ) # Define financials columns a_financials_header = list() @@ -202,7 +209,10 @@ def liabilities(l_args, s_ticker): else: url_financials = f"https://www.marketwatch.com/investing/stock/{s_ticker}/financials/balance-sheet" - text_soup_financials = BeautifulSoup(requests.get(url_financials).text, "lxml") + text_soup_financials = BeautifulSoup( + requests.get(url_financials, headers={"User-Agent": get_user_agent()}).text, + "lxml", + ) # Define financials columns a_financials_header = list() @@ -277,7 +287,10 @@ def operating(l_args, s_ticker): else: url_financials = f"https://www.marketwatch.com/investing/stock/{s_ticker}/financials/cash-flow" - text_soup_financials = BeautifulSoup(requests.get(url_financials).text, "lxml") + text_soup_financials = BeautifulSoup( + requests.get(url_financials, headers={"User-Agent": get_user_agent()}).text, + "lxml", + ) # Define financials columns a_financials_header = list() @@ -356,7 +369,10 @@ def investing(l_args, s_ticker): else: url_financials = f"https://www.marketwatch.com/investing/stock/{s_ticker}/financials/cash-flow" - text_soup_financials = BeautifulSoup(requests.get(url_financials).text, "lxml") + text_soup_financials = BeautifulSoup( + requests.get(url_financials, headers={"User-Agent": get_user_agent()}).text, + "lxml", + ) # Define financials columns a_financials_header = list() @@ -438,7 +454,10 @@ def financing(l_args, s_ticker): else: url_financials = f"https://www.marketwatch.com/investing/stock/{s_ticker}/financials/cash-flow" - text_soup_financials = BeautifulSoup(requests.get(url_financials).text, "lxml") + text_soup_financials = BeautifulSoup( + requests.get(url_financials, headers={"User-Agent": get_user_agent()}).text, + "lxml", + ) # Define financials columns a_financials_header = list() diff --git a/gamestonk_terminal/helper_funcs.py b/gamestonk_terminal/helper_funcs.py index a178e5d395e..bd674603907 100644 --- a/gamestonk_terminal/helper_funcs.py +++ b/gamestonk_terminal/helper_funcs.py @@ -1,5 +1,6 @@ import argparse from datetime import datetime, timedelta, time as Time +import random import re import sys import iso8601 @@ -8,7 +9,7 @@ from pytz import timezone from holidays import US as holidaysUS # ----------------------------------------------------------------------------------------------------------------------- -def check_non_negative(value): +def check_non_negative(value) -> int: ivalue = int(value) if ivalue < 0: raise argparse.ArgumentTypeError(f"{value} is negative") @@ -16,7 +17,7 @@ def check_non_negative(value): # ----------------------------------------------------------------------------------------------------------------------- -def check_positive(value): +def check_positive(value) -> int: ivalue = int(value) if ivalue <= 0: raise argparse.ArgumentTypeError(f"{value} is an invalid positive int value") @@ -24,7 +25,7 @@ def check_positive(value): # ----------------------------------------------------------------------------------------------------------------------- -def valid_date(s): +def valid_date(s: str) -> datetime: try: return datetime.strptime(s, "%Y-%m-%d") except ValueError: @@ -118,7 +119,7 @@ def plot_ta(s_ticker, df_ta, s_ta): # ----------------------------------------------------------------------------------------------------------------------- -def b_is_stock_market_open(): +def b_is_stock_market_open() -> bool: """ checks if the stock market is open """ # Get current US time now = datetime.now(timezone("US/Eastern")) @@ -139,7 +140,7 @@ def b_is_stock_market_open(): # ----------------------------------------------------------------------------------------------------------------------- -def long_number_format(num): +def long_number_format(num) -> str: if isinstance(num, float): magnitude = 0 while abs(num) >= 1000: @@ -166,7 +167,7 @@ def long_number_format(num): # ----------------------------------------------------------------------------------------------------------------------- -def clean_data_values_to_float(val): +def clean_data_values_to_float(val: str) -> float: # Remove parenthesis if they exist if val.startswith("("): val = val[1:] @@ -210,7 +211,7 @@ def divide_chunks(l, n): # ----------------------------------------------------------------------------------------------------------------------- -def get_next_stock_market_days(last_stock_day, n_next_days): +def get_next_stock_market_days(last_stock_day, n_next_days) -> list: n_days = 0 l_pred_days = list() while n_days < n_next_days: @@ -232,7 +233,7 @@ def get_next_stock_market_days(last_stock_day, n_next_days): # ----------------------------------------------------------------------------------------------------------------------- -def get_data(tweet): +def get_data(tweet) -> hash: if "+" in tweet["created_at"]: s_datetime = tweet["created_at"].split(" +")[0] else: @@ -250,7 +251,7 @@ def get_data(tweet): # ----------------------------------------------------------------------------------------------------------------------- -def clean_tweet(tweet, s_ticker): +def clean_tweet(tweet: str, s_ticker: str) -> str: whitespace = re.compile(r"\s+") web_address = re.compile(r"(?i)http(s):\/\/[a-z0-9.~_\-\/]+") ticker = re.compile(r"(?i)@{}(?=\b)".format(s_ticker)) @@ -262,3 +263,17 @@ def clean_tweet(tweet, s_ticker): tweet = user.sub("", tweet) return tweet + + +def get_user_agent() -> str: + user_agent_strings = [ + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:86.1) Gecko/20100101 Firefox/86.1", + "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:86.1) Gecko/20100101 Firefox/86.1", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:82.1) Gecko/20100101 Firefox/82.1", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:86.0) Gecko/20100101 Firefox/86.0", + "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:86.0) Gecko/20100101 Firefox/86.0", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:83.0) Gecko/20100101 Firefox/83.0", + "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:84.0) Gecko/20100101 Firefox/84.0", + ] + + return random.choice(user_agent_strings)