More additions of randomized User Agent to BeautifulSoup calls

gamestonk_terminal/discovery/short_interest_api.py

@@ -1,5 +1,4 @@
 import argparse
-import re
 import requests
 from bs4 import BeautifulSoup
 import pandas as pd
@@ -41,7 +40,10 @@ def high_short_interest(l_args):
 
     url_high_short_interested_stocks = "https://www.highshortinterest.com"
     text_soup_high_short_interested_stocks = BeautifulSoup(
-        requests.get(url_high_short_interested_stocks).text, "lxml"
+        requests.get(
+            url_high_short_interested_stocks, headers={"User-Agent": get_user_agent()}
+        ).text,
+        "lxml",
     )
 
     a_high_short_interest_header = list()

gamestonk_terminal/due_diligence/market_watch_api.py

@@ -36,7 +36,10 @@ def sec_fillings(l_args, s_ticker):
 
         url_financials = f"https://www.marketwatch.com/investing/stock/{s_ticker}/financials/secfilings"
 
-        text_soup_financials = BeautifulSoup(requests.get(url_financials).text, "lxml")
+        text_soup_financials = BeautifulSoup(
+            requests.get(url_financials, headers={"User-Agent": get_user_agent()}).text,
+            "lxml",
+        )
 
         a_financials_header = list()
         b_ready_to_process_info = False
@@ -120,7 +123,10 @@ def sean_seah_warnings(l_args, s_ticker):
         url_financials = (
             f"https://www.marketwatch.com/investing/stock/{s_ticker}/financials/income"
         )
-        text_soup_financials = BeautifulSoup(requests.get(url_financials).text, "lxml")
+        text_soup_financials = BeautifulSoup(
+            requests.get(url_financials, headers={"User-Agent": get_user_agent()}).text,
+            "lxml",
+        )
 
         # Define financials columns
         a_financials_header = list()
@@ -153,7 +159,10 @@ def sean_seah_warnings(l_args, s_ticker):
 
         # From BALANCE SHEET, get: 'Liabilities & Shareholders\' Equity', 'Long-Term Debt'
         url_financials = f"https://www.marketwatch.com/investing/stock/{s_ticker}/financials/balance-sheet"
-        text_soup_financials = BeautifulSoup(requests.get(url_financials).text, "lxml")
+        text_soup_financials = BeautifulSoup(
+            requests.get(url_financials, headers={"User-Agent": get_user_agent()}).text,
+            "lxml",
+        )
 
         # Define financials columns
         a_financials_header = list()

gamestonk_terminal/fundamental_analysis/business_insider_api.py

@@ -6,6 +6,7 @@ import requests
 from bs4 import BeautifulSoup
 import pandas as pd
 from fuzzywuzzy import fuzz
+from gamestonk_terminal.helper_funcs import get_user_agent
 from gamestonk_terminal import config_terminal as cfg
 
 
@@ -30,7 +31,10 @@ def management(l_args, s_ticker):
             f"https://markets.businessinsider.com/stocks/{s_ticker.lower()}-stock"
         )
         text_soup_market_business_insider = BeautifulSoup(
-            requests.get(url_market_business_insider).text, "lxml"
+            requests.get(
+                url_market_business_insider, headers={"User-Agent": get_user_agent()}
+            ).text,
+            "lxml",
         )
 
         l_titles = list()