From 222aa5bac0a60121e77bc44ddc8f62ac013d77df Mon Sep 17 00:00:00 2001 From: Artem Veremey Date: Sat, 27 Feb 2021 14:06:40 -0800 Subject: [PATCH] More additions of randomized User Agent to BeautifulSoup calls --- .../discovery/short_interest_api.py | 6 ++++-- .../due_diligence/market_watch_api.py | 15 ++++++++++++--- .../fundamental_analysis/business_insider_api.py | 6 +++++- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/gamestonk_terminal/discovery/short_interest_api.py b/gamestonk_terminal/discovery/short_interest_api.py index ca500cfeee2..f41ae09f2c5 100644 --- a/gamestonk_terminal/discovery/short_interest_api.py +++ b/gamestonk_terminal/discovery/short_interest_api.py @@ -1,5 +1,4 @@ import argparse -import re import requests from bs4 import BeautifulSoup import pandas as pd @@ -41,7 +40,10 @@ def high_short_interest(l_args): url_high_short_interested_stocks = "https://www.highshortinterest.com" text_soup_high_short_interested_stocks = BeautifulSoup( - requests.get(url_high_short_interested_stocks).text, "lxml" + requests.get( + url_high_short_interested_stocks, headers={"User-Agent": get_user_agent()} + ).text, + "lxml", ) a_high_short_interest_header = list() diff --git a/gamestonk_terminal/due_diligence/market_watch_api.py b/gamestonk_terminal/due_diligence/market_watch_api.py index ac119847521..38bb9fe1747 100644 --- a/gamestonk_terminal/due_diligence/market_watch_api.py +++ b/gamestonk_terminal/due_diligence/market_watch_api.py @@ -36,7 +36,10 @@ def sec_fillings(l_args, s_ticker): url_financials = f"https://www.marketwatch.com/investing/stock/{s_ticker}/financials/secfilings" - text_soup_financials = BeautifulSoup(requests.get(url_financials).text, "lxml") + text_soup_financials = BeautifulSoup( + requests.get(url_financials, headers={"User-Agent": get_user_agent()}).text, + "lxml", + ) a_financials_header = list() b_ready_to_process_info = False @@ -120,7 +123,10 @@ def sean_seah_warnings(l_args, s_ticker): url_financials = ( f"https://www.marketwatch.com/investing/stock/{s_ticker}/financials/income" ) - text_soup_financials = BeautifulSoup(requests.get(url_financials).text, "lxml") + text_soup_financials = BeautifulSoup( + requests.get(url_financials, headers={"User-Agent": get_user_agent()}).text, + "lxml", + ) # Define financials columns a_financials_header = list() @@ -153,7 +159,10 @@ def sean_seah_warnings(l_args, s_ticker): # From BALANCE SHEET, get: 'Liabilities & Shareholders\' Equity', 'Long-Term Debt' url_financials = f"https://www.marketwatch.com/investing/stock/{s_ticker}/financials/balance-sheet" - text_soup_financials = BeautifulSoup(requests.get(url_financials).text, "lxml") + text_soup_financials = BeautifulSoup( + requests.get(url_financials, headers={"User-Agent": get_user_agent()}).text, + "lxml", + ) # Define financials columns a_financials_header = list() diff --git a/gamestonk_terminal/fundamental_analysis/business_insider_api.py b/gamestonk_terminal/fundamental_analysis/business_insider_api.py index f0fb5ac8c7a..49281a1cb7e 100644 --- a/gamestonk_terminal/fundamental_analysis/business_insider_api.py +++ b/gamestonk_terminal/fundamental_analysis/business_insider_api.py @@ -6,6 +6,7 @@ import requests from bs4 import BeautifulSoup import pandas as pd from fuzzywuzzy import fuzz +from gamestonk_terminal.helper_funcs import get_user_agent from gamestonk_terminal import config_terminal as cfg @@ -30,7 +31,10 @@ def management(l_args, s_ticker): f"https://markets.businessinsider.com/stocks/{s_ticker.lower()}-stock" ) text_soup_market_business_insider = BeautifulSoup( - requests.get(url_market_business_insider).text, "lxml" + requests.get( + url_market_business_insider, headers={"User-Agent": get_user_agent()} + ).text, + "lxml", ) l_titles = list()