Adding randomized User Agent to BeautifulSoup calls. Adding type hints

gamestonk_terminal/discovery/fidelity_api.py

@@ -2,7 +2,7 @@ import argparse
 from bs4 import BeautifulSoup
 import requests
 import pandas as pd
-from gamestonk_terminal.helper_funcs import check_positive
+from gamestonk_terminal.helper_funcs import check_positive, get_user_agent
 
 # ---------------------------------------------------- ORDERS ----------------------------------------------------
 def orders(l_args):
@@ -35,7 +35,11 @@ def orders(l_args):
         url_orders = (
             "https://eresearch.fidelity.com/eresearch/gotoBL/fidelityTopOrders.jhtml"
         )
-        text_soup_url_orders = BeautifulSoup(requests.get(url_orders).text, "lxml")
+
+        text_soup_url_orders = BeautifulSoup(
+            requests.get(url_orders, headers={"User-Agent": get_user_agent()}).text,
+            "lxml",
+        )
 
         l_orders = list()
         l_orders_vals = list()

gamestonk_terminal/discovery/seeking_alpha_api.py

@@ -2,7 +2,7 @@ import argparse
 from bs4 import BeautifulSoup
 import requests
 import pandas as pd
-from gamestonk_terminal.helper_funcs import check_positive
+from gamestonk_terminal.helper_funcs import check_positive, get_user_agent
 
 
 # ------------------------------------------------ EARNINGS_RELEASE_DATES -------------------------------------------------
@@ -48,7 +48,10 @@ def earnings_release_dates(l_args):
                     f"https://seekingalpha.com/earnings/earnings-calendar/{idx+1}"
                 )
             text_soup_earnings = BeautifulSoup(
-                requests.get(url_next_earnings).text, "lxml"
+                requests.get(
+                    url_next_earnings, headers={"User-Agent": get_user_agent()}
+                ).text,
+                "lxml",
             )
 
             for bs_stock in text_soup_earnings.findAll(
@@ -76,7 +79,6 @@ def earnings_release_dates(l_args):
                 ].to_string(index=False, header=False)
             )
             print("")
-
     except SystemExit:
         print("")
         return

gamestonk_terminal/discovery/short_interest_api.py

@@ -3,7 +3,7 @@ import re
 import requests
 from bs4 import BeautifulSoup
 import pandas as pd
-from gamestonk_terminal.helper_funcs import check_positive
+from gamestonk_terminal.helper_funcs import check_positive, get_user_agent
 
 
 # ------------------------------------------------ HIGH_SHORT_INTEREST -------------------------------------------------
@@ -111,7 +111,10 @@ def low_float(l_args):
 
     url_high_short_interested_stocks = "https://www.lowfloat.com"
     text_soup_low_float_stocks = BeautifulSoup(
-        requests.get(url_high_short_interested_stocks).text, "lxml"
+        requests.get(
+            url_high_short_interested_stocks, headers={"User-Agent": get_user_agent()}
+        ).text,
+        "lxml",
     )
 
     a_low_float_header = list()

gamestonk_terminal/due_diligence/business_insider_api.py

@@ -1,16 +1,20 @@
+import argparse
+from datetime import datetime
+import json
+import re
+import requests
+import matplotlib.pyplot as plt
+import pandas as pd
+from pandas.plotting import register_matplotlib_converters
 from bs4 import BeautifulSoup
 from alpha_vantage.timeseries import TimeSeries
-from gamestonk_terminal.helper_funcs import *
-import requests
-import pandas as pd
-import re
-import json
-from gamestonk_terminal import config_terminal as cfg
-from datetime import datetime
-import argparse
 from fuzzywuzzy import fuzz
-import matplotlib.pyplot as plt
-from pandas.plotting import register_matplotlib_converters
+from gamestonk_terminal.helper_funcs import (
+    check_positive,
+    get_next_stock_market_days,
+    get_user_agent,
+)
+from gamestonk_terminal import config_terminal as cfg
 
 register_matplotlib_converters()
 
@@ -43,7 +47,10 @@ def price_target_from_analysts(l_args, df_stock, s_ticker, s_start, s_interval):
             f"https://markets.businessinsider.com/stocks/{s_ticker.lower()}-stock"
         )
         text_soup_market_business_insider = BeautifulSoup(
-            requests.get(url_market_business_insider).text, "lxml"
+            requests.get(
+                url_market_business_insider, headers={"User-Agent": get_user_agent()}
+            ).text,
+            "lxml",
         )
 
         for script in text_soup_market_business_insider.find_all("script"):
@@ -129,7 +136,10 @@ def estimates(l_args, s_ticker):
             f"https://markets.businessinsider.com/stocks/{s_ticker.lower()}-stock"
         )
         text_soup_market_business_insider = BeautifulSoup(
-            requests.get(url_market_business_insider).text, "lxml"
+            requests.get(
+                url_market_business_insider, headers={"User-Agent": get_user_agent()}
+            ).text,
+            "lxml",
         )
 
         l_estimates_year_header = list()
@@ -295,7 +305,10 @@ def insider_activity(l_args, df_stock, s_ticker, s_start, s_interval):
             f"https://markets.businessinsider.com/stocks/{s_ticker.lower()}-stock"
         )
         text_soup_market_business_insider = BeautifulSoup(
-            requests.get(url_market_business_insider).text, "lxml"
+            requests.get(
+                url_market_business_insider, headers={"User-Agent": get_user_agent()}
+            ).text,
+            "lxml",
         )
 
         d_insider = dict()

gamestonk_terminal/fundamental_analysis/business_insider_api.py

@@ -1,20 +1,22 @@
-from bs4 import BeautifulSoup
-import requests
-import pandas as pd
+import argparse
 import re
 import json
-from gamestonk_terminal import config_terminal as cfg
 from datetime import datetime
-import argparse
+import requests
+from bs4 import BeautifulSoup
+import pandas as pd
 from fuzzywuzzy import fuzz
+from gamestonk_terminal import config_terminal as cfg
 
 
 # ---------------------------------------------------- MANAGEMENT ----------------------------------------------------
 def management(l_args, s_ticker):
     parser = argparse.ArgumentParser(
         prog="mgmt",
-        description="""Print management team. Namely: Name, Title, Information from google and 
-                                                    (potentially) Insider Activity page. [Source: Business Insider]""",
+        description="""
+            Print management team. Namely: Name, Title, Information from google and
+            (potentially) Insider Activity page. [Source: Business Insider]
+        """,
     )
 
     try:

gamestonk_terminal/fundamental_analysis/market_watch_api.py

@@ -2,6 +2,7 @@ import argparse
 import pandas as pd
 from bs4 import BeautifulSoup
 import requests
+from gamestonk_terminal.helper_funcs import get_user_agent
 
 # ---------------------------------------------------- INCOME ----------------------------------------------------
 def income(l_args, s_ticker):
@@ -48,7 +49,10 @@ def income(l_args, s_ticker):
         else:
             url_financials = f"https://www.marketwatch.com/investing/stock/{s_ticker}/financials/income"
 
-        text_soup_financials = BeautifulSoup(requests.get(url_financials).text, "lxml")
+        text_soup_financials = BeautifulSoup(
+            requests.get(url_financials, headers={"User-Agent": get_user_agent()}).text,
+            "lxml",
+        )
 
         # Define financials columns
         a_financials_header = list()
@@ -119,7 +123,10 @@ def assets(l_args, s_ticker):
         else:
             url_financials = f"https://www.marketwatch.com/investing/stock/{s_ticker}/financials/balance-sheet"
 
-        text_soup_financials = BeautifulSoup(requests.get(url_financials).text, "lxml")
+        text_soup_financials = BeautifulSoup(
+            requests.get(url_financials, headers={"User-Agent": get_user_agent()}).text,
+            "lxml",
+        )
 
         # Define financials columns
         a_financials_header = list()
@@ -202,7 +209,10 @@ def liabilities(l_args, s_ticker):
         else:
             url_financials = f"https://www.marketwatch.com/investing/stock/{s_ticker}/financials/balance-sheet"
 
-        text_soup_financials = BeautifulSoup(requests.get(url_financials).text, "lxml")
+        text_soup_financials = BeautifulSoup(
+            requests.get(url_financials, headers={"User-Agent": get_user_agent()}).text,
+            "lxml",
+        )
 
         # Define financials columns
         a_financials_header = list()
@@ -277,7 +287,10 @@ def operating(l_args, s_ticker):
         else:
             url_financials = f"https://www.marketwatch.com/investing/stock/{s_ticker}/financials/cash-flow"
 
-        text_soup_financials = BeautifulSoup(requests.get(url_financials).text, "lxml")
+        text_soup_financials = BeautifulSoup(
+            requests.get(url_financials, headers={"User-Agent": get_user_agent()}).text,
+            "lxml",
+        )
 
         # Define financials columns
         a_financials_header = list()
@@ -356,7 +369,10 @@ def investing(l_args, s_ticker):
         else:
             url_financials = f"https://www.marketwatch.com/investing/stock/{s_ticker}/financials/cash-flow"
 
-        text_soup_financials = BeautifulSoup(requests.get(url_financials).text, "lxml")
+        text_soup_financials = BeautifulSoup(
+            requests.get(url_financials, headers={"User-Agent": get_user_agent()}).text,
+            "lxml",
+        )
 
         # Define financials columns
         a_financials_header = list()
@@ -438,7 +454,10 @@ def financing(l_args, s_ticker):
         else:
             url_financials = f"https://www.marketwatch.com/investing/stock/{s_ticker}/financials/cash-flow"
 
-        text_soup_financials = BeautifulSoup(requests.get(url_financials).text, "lxml")
+        text_soup_financials = BeautifulSoup(
+            requests.get(url_financials, headers={"User-Agent": get_user_agent()}).text,
+            "lxml",
+        )
 
         # Define financials columns
         a_financials_header = list()

gamestonk_terminal/helper_funcs.py

@@ -1,5 +1,6 @@
 import argparse
 from datetime import datetime, timedelta, time as Time
+import random
 import re
 import sys
 import iso8601
@@ -8,7 +9,7 @@ from pytz import timezone
 from holidays import US as holidaysUS
 
 # -----------------------------------------------------------------------------------------------------------------------
-def check_non_negative(value):
+def check_non_negative(value) -> int:
     ivalue = int(value)
     if ivalue < 0:
         raise argparse.ArgumentTypeError(f"{value} is negative")
@@ -16,7 +17,7 @@ def check_non_negative(value):
 
 
 # -----------------------------------------------------------------------------------------------------------------------
-def check_positive(value):
+def check_positive(value) -> int:
     ivalue = int(value)
     if ivalue <= 0:
         raise argparse.ArgumentTypeError(f"{value} is an invalid positive int value")
@@ -24,7 +25,7 @@ def check_positive(value):
 
 
 # -----------------------------------------------------------------------------------------------------------------------
-def valid_date(s):
+def valid_date(s: str) -> datetime:
     try:
         return datetime.strptime(s, "%Y-%m-%d")
     except ValueError:
@@ -118,7 +119,7 @@ def plot_ta(s_ticker, df_ta, s_ta):
 
 
 # -----------------------------------------------------------------------------------------------------------------------
-def b_is_stock_market_open():
+def b_is_stock_market_open() -> bool:
     """ checks if the stock market is open """
     # Get current US time
     now = datetime.now(timezone("US/Eastern"))
@@ -139,7 +140,7 @@ def b_is_stock_market_open():
 
 
 # -----------------------------------------------------------------------------------------------------------------------
-def long_number_format(num):
+def long_number_format(num) -> str:
     if isinstance(num, float):
         magnitude = 0
         while abs(num) >= 1000:
@@ -166,7 +167,7 @@ def long_number_format(num):
 
 
 # -----------------------------------------------------------------------------------------------------------------------
-def clean_data_values_to_float(val):
+def clean_data_values_to_float(val: str) -> float:
     # Remove parenthesis if they exist
     if val.startswith("("):
         val = val[1:]
@@ -210,7 +211,7 @@ def divide_chunks(l, n):
 
 
 # -----------------------------------------------------------------------------------------------------------------------
-def get_next_stock_market_days(last_stock_day, n_next_days):
+def get_next_stock_market_days(last_stock_day, n_next_days) -> list:
     n_days = 0
     l_pred_days = list()
     while n_days < n_next_days:
@@ -232,7 +233,7 @@ def get_next_stock_market_days(last_stock_day, n_next_days):
 
 
 # -----------------------------------------------------------------------------------------------------------------------
-def get_data(tweet):
+def get_data(tweet) -> hash:
     if "+" in tweet["created_at"]:
         s_datetime = tweet["created_at"].split(" +")[0]
     else:
@@ -250,7 +251,7 @@ def get_data(tweet):
 
 
 # -----------------------------------------------------------------------------------------------------------------------
-def clean_tweet(tweet, s_ticker):
+def clean_tweet(tweet: str, s_ticker: str) -> str:
     whitespace = re.compile(r"\s+")
     web_address = re.compile(r"(?i)http(s):\/\/[a-z0-9.~_\-\/]+")
     ticker = re.compile(r"(?i)@{}(?=\b)".format(s_ticker))
@@ -262,3 +263,17 @@ def clean_tweet(tweet, s_ticker):
     tweet = user.sub("", tweet)
 
     return tweet
+
+
+def get_user_agent() -> str:
+    user_agent_strings = [
+        "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:86.1) Gecko/20100101 Firefox/86.1",
+        "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:86.1) Gecko/20100101 Firefox/86.1",
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:82.1) Gecko/20100101 Firefox/82.1",
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:86.0) Gecko/20100101 Firefox/86.0",
+        "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:86.0) Gecko/20100101 Firefox/86.0",
+        "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:83.0) Gecko/20100101 Firefox/83.0",
+        "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:84.0) Gecko/20100101 Firefox/84.0",
+    ]
+
+    return random.choice(user_agent_strings)