From 93379e7d6b36a422da4e018bd3229054a2496efe Mon Sep 17 00:00:00 2001 From: DidierRLopes Date: Fri, 5 Nov 2021 01:17:20 +0000 Subject: [PATCH] Use get_user_agent for all requests (#914) --- .../cryptocurrency/pycoingecko_helpers.py | 3 ++- gamestonk_terminal/economy/wsj_model.py | 13 +++++++------ gamestonk_terminal/etf/stockanalysis_model.py | 7 ++++--- gamestonk_terminal/etf/wsj_model.py | 3 ++- gamestonk_terminal/options/barchart_model.py | 3 ++- gamestonk_terminal/options/chartexchange_model.py | 3 ++- gamestonk_terminal/options/fdscanner_model.py | 3 ++- .../portfolio/brokers/degiro/degiro_model.py | 2 +- .../portfolio/brokers/degiro/degiro_view.py | 2 +- .../stocks/discovery/shortinterest_model.py | 4 +--- .../stocks/fundamental_analysis/dcf_model.py | 8 ++------ 11 files changed, 26 insertions(+), 25 deletions(-) diff --git a/gamestonk_terminal/cryptocurrency/pycoingecko_helpers.py b/gamestonk_terminal/cryptocurrency/pycoingecko_helpers.py index 301552ff66b..27790e25136 100644 --- a/gamestonk_terminal/cryptocurrency/pycoingecko_helpers.py +++ b/gamestonk_terminal/cryptocurrency/pycoingecko_helpers.py @@ -12,6 +12,7 @@ import pandas as pd from dateutil import parser from requests.adapters import HTTPAdapter, RetryError from urllib3.util.retry import Retry +from gamestonk_terminal.helper_funcs import get_user_agent GECKO_BASE_URL = "https://www.coingecko.com" @@ -84,7 +85,7 @@ def scrape_gecko_data(url: str) -> BeautifulSoup: ------- BeautifulSoup object """ - headers = {"User-Agent": "Mozilla/5.0"} + headers = {"User-Agent": get_user_agent()} session = _retry_session("https://www.coingecko.com") try: req = session.get(url, headers=headers, timeout=5) diff --git a/gamestonk_terminal/economy/wsj_model.py b/gamestonk_terminal/economy/wsj_model.py index aceaab65bfd..19de63e0f68 100644 --- a/gamestonk_terminal/economy/wsj_model.py +++ b/gamestonk_terminal/economy/wsj_model.py @@ -2,6 +2,7 @@ __docformat__ = "numpy" import pandas as pd import requests +from gamestonk_terminal.helper_funcs import get_user_agent def us_indices() -> pd.DataFrame: @@ -23,7 +24,7 @@ def us_indices() -> pd.DataFrame: "name%22%3A%22CBOE%20Volatility%22%7D%2C%7B%22symbol%22%3A%22FUTURE%2FUS%2F%2FDJIA%20FUTURES%22%2C%22name%22%3A%" "22DJIA%20Futures%22%7D%2C%7B%22symbol%22%3A%22FUTURE%2FUS%2F%2FS%26P%20500%20FUTURES%22%2C%22name%22%3A%22S%26P" "%20500%20Futures%22%7D%5D%7D&type=mdc_quotes", - headers={"User-Agent": "Mozilla/5.0"}, + headers={"User-Agent": get_user_agent()}, ).json() name, last_price, net_change, percent_change = [], [], [], [] @@ -59,7 +60,7 @@ def market_overview() -> pd.DataFrame: "%22%3A%22FUTURE%2FUS%2F%2FGOLD%22%2C%22name%22%3A%22Gold%20Futures%22%7D%2C%7B%22symbol%22%3A%22CURRENCY%2FUS%2" "F%2FUSDJPY%22%2C%22name%22%3A%22Yen%22%7D%2C%7B%22symbol%22%3A%22CURRENCY%2FUS%2F%2FEURUSD%22%2C%22name%22%3A%" "22Euro%22%7D%5D%7D&type=mdc_quotes", - headers={"User-Agent": "Mozilla/5.0"}, + headers={"User-Agent": get_user_agent()}, ).json() name, last_price, net_change, percent_change = [], [], [], [] @@ -95,7 +96,7 @@ def top_commodities() -> pd.DataFrame: "2FUS%2F%2FCOPPER%22%2C%22name%22%3A%22Copper%20Futures%22%7D%2C%7B%22symbol%22%3A%22FUTURE%2FUS%2F%2FCORN%22%2" "C%22name%22%3A%22Corn%20Futures%22%7D%2C%7B%22symbol%22%3A%22FUTURE%2FUS%2F%2FWHEAT%22%2C%22name%22%3A%22Wheat" "%20Futures%22%7D%2C%7B%22symbol%22%3A%22INDEX%2FXX%2F%2FBCOM%22%7D%5D%7D&type=mdc_quotes", - headers={"User-Agent": "Mozilla/5.0"}, + headers={"User-Agent": get_user_agent()}, ).json() name, last_price, net_change, percent_change = [], [], [], [] @@ -133,7 +134,7 @@ def us_bonds() -> pd.DataFrame: "BOND%2FBX%2F%2FTMUBMUSD06M%22%2C%22name%22%3A%226-Month%20Bill%22%7D%2C%7B%22symbol%22%3A%22BON" "D%2FBX%2F%2FTMUBMUSD03M%22%2C%22name%22%3A%223-Month%20Bill%22%7D%2C%7B%22symbol%22%3A%22BOND%" "2FBX%2F%2FTMUBMUSD01M%22%2C%22name%22%3A%221-Month%20Bill%22%7D%5D%7D&type=mdc_quotes", - headers={"User-Agent": "Mozilla/5.0"}, + headers={"User-Agent": get_user_agent()}, ).json() name, yield_pct, rate, yld_chng = [], [], [], [] @@ -165,7 +166,7 @@ def global_bonds() -> pd.DataFrame: "22name%22%3A%22Japan%2010%20Year%22%7D%2C%7B%22symbol%22%3A%22TMBMKAU-10Y%22%2C%22name%22%3A%2" "2Australia%2010%20Year%22%7D%2C%7B%22symbol%22%3A%22AMBMKRM-10Y%22%2C%22name%22%3A%22China%2010" "%20Year%22%7D%5D%7D&type=mdc_governmentbonds", - headers={"User-Agent": "Mozilla/5.0"}, + headers={"User-Agent": get_user_agent()}, ).json() name, yield_pct, rate, yld_chng = [], [], [], [] @@ -202,7 +203,7 @@ def global_currencies() -> pd.DataFrame: "CURRENCY%2FUS%2F%2FBTCUSD%22%2C%22name%22%3A%22Bitcoin%20(BTC%2FUSD)%22%7D%2C%7B%22symbol%22%3A" "%22INDEX%2FXX%2F%2FBUXX%22%2C%22name%22%3A%22WSJ%20Dollar%20Index%22%7D%2C%7B%22symbol%22%3A%2" "2INDEX%2FUS%2F%2FDXY%22%2C%22name%22%3A%22U.S.%20Dollar%20Index%22%7D%5D%7D&type=mdc_quotes", - headers={"User-Agent": "Mozilla/5.0"}, + headers={"User-Agent": get_user_agent()}, ).json() name, last_price, price_change, pct_change = [], [], [], [] diff --git a/gamestonk_terminal/etf/stockanalysis_model.py b/gamestonk_terminal/etf/stockanalysis_model.py index d55c6540b85..8472048eab0 100644 --- a/gamestonk_terminal/etf/stockanalysis_model.py +++ b/gamestonk_terminal/etf/stockanalysis_model.py @@ -6,6 +6,7 @@ from typing import List, Tuple import pandas as pd import requests from bs4 import BeautifulSoup as bs +from gamestonk_terminal.helper_funcs import get_user_agent def get_all_names_symbols() -> Tuple[List[str], List[str]]: @@ -19,7 +20,7 @@ def get_all_names_symbols() -> Tuple[List[str], List[str]]: List of all available etf names """ r = requests.get( - "https://api.stockanalysis.com/etf/", headers={"User-Agent": "Mozilla/5.0"} + "https://api.stockanalysis.com/etf/", headers={"User-Agent": get_user_agent()} ) soup = bs(r.text, "html.parser").findAll("ul", {"class": "no-spacing"}) all_links = soup[0].findAll("li") @@ -47,7 +48,7 @@ def get_etf_overview(etf_symbol: str) -> pd.DataFrame: """ r = requests.get( f"https://stockanalysis.com/etf/{etf_symbol}", - headers={"User-Agent": "Mozilla/5.0"}, + headers={"User-Agent": get_user_agent()}, ) soup = bs(r.text, "html.parser") # %% tables = soup.findAll("table") @@ -80,7 +81,7 @@ def get_etf_holdings(symbol: str) -> pd.DataFrame: """ link = f"https://api.stockanalysis.com/etf/{symbol}/holdings/" - r = requests.get(link, headers={"User-Agent": "Mozilla/5.0"}) + r = requests.get(link, headers={"User-Agent": get_user_agent()}) soup = bs(r.text, "html.parser") soup = soup.find("table") tds = soup.findAll("td") diff --git a/gamestonk_terminal/etf/wsj_model.py b/gamestonk_terminal/etf/wsj_model.py index 733adc3f277..dd4d927b531 100644 --- a/gamestonk_terminal/etf/wsj_model.py +++ b/gamestonk_terminal/etf/wsj_model.py @@ -2,6 +2,7 @@ __docformat__ = "numpy" import pandas as pd import requests +from gamestonk_terminal.helper_funcs import get_user_agent def etf_movers(sort_type: str = "gainers") -> pd.DataFrame: @@ -34,7 +35,7 @@ def etf_movers(sort_type: str = "gainers") -> pd.DataFrame: "%22%3A%22WSJ%22%2C%22etfMover%22%3A%22most_active%22%2C%22count%22%3A25%7D&type=mdc_etfmovers" ) - data = requests.get(url, headers={"User-Agent": "Mozilla/5.0"}).json() + data = requests.get(url, headers={"User-Agent": get_user_agent()}).json() name, last_price, net_change, percent_change, volume = [], [], [], [], [] for entry in data["data"]["instruments"]: diff --git a/gamestonk_terminal/options/barchart_model.py b/gamestonk_terminal/options/barchart_model.py index 080ebec5b96..1b00dccc13f 100644 --- a/gamestonk_terminal/options/barchart_model.py +++ b/gamestonk_terminal/options/barchart_model.py @@ -4,6 +4,7 @@ __docformat__ = "numpy" import pandas as pd import requests from bs4 import BeautifulSoup +from gamestonk_terminal.helper_funcs import get_user_agent def get_options_info(ticker: str) -> pd.DataFrame: @@ -21,7 +22,7 @@ def get_options_info(ticker: str) -> pd.DataFrame: """ page = f"https://www.barchart.com/stocks/quotes/{ticker}/overview" - r = requests.get(page, headers={"User-Agent": "Mozilla/5.0"}) + r = requests.get(page, headers={"User-Agent": get_user_agent()}) soup = BeautifulSoup(r.text, "html.parser") tags = soup.find( "div", diff --git a/gamestonk_terminal/options/chartexchange_model.py b/gamestonk_terminal/options/chartexchange_model.py index 53bdb4c3e23..174a0422e4f 100644 --- a/gamestonk_terminal/options/chartexchange_model.py +++ b/gamestonk_terminal/options/chartexchange_model.py @@ -6,6 +6,7 @@ import pandas as pd from bs4 import BeautifulSoup from gamestonk_terminal.options.op_helpers import convert +from gamestonk_terminal.helper_funcs import get_user_agent def get_option_history(ticker: str, date: str, call: bool, price: str) -> pd.DataFrame: @@ -32,7 +33,7 @@ def get_option_history(ticker: str, date: str, call: bool, price: str) -> pd.Dat ) url += f"{'c' if call else 'p'}{float(price):g}/historical/" - data = requests.get(url, headers={"User-Agent": "test"}).content + data = requests.get(url, headers={"User-Agent": get_user_agent()}).content soup = BeautifulSoup(data, "html.parser") table = soup.find("div", attrs={"style": "display: table; font-size: 0.9em; "}) rows = table.find_all("div", attrs={"style": "display: table-row;"}) diff --git a/gamestonk_terminal/options/fdscanner_model.py b/gamestonk_terminal/options/fdscanner_model.py index 15d342b1597..374a91c2741 100644 --- a/gamestonk_terminal/options/fdscanner_model.py +++ b/gamestonk_terminal/options/fdscanner_model.py @@ -4,6 +4,7 @@ __docformat__ = "numpy" import numpy as np import pandas as pd import requests +from gamestonk_terminal.helper_funcs import get_user_agent def unusual_options(num: int, sort_column: str, ascending: bool): @@ -31,7 +32,7 @@ def unusual_options(num: int, sort_column: str, ascending: bool): r = requests.get( f"https://app.fdscanner.com/api2/unusualvolume?p=0&page_size=20&page={int(page_num)}", - headers={"User-Agent": "Mozilla/5.0"}, + headers={"User-Agent": get_user_agent()}, ) if r.status_code != 200: diff --git a/gamestonk_terminal/portfolio/brokers/degiro/degiro_model.py b/gamestonk_terminal/portfolio/brokers/degiro/degiro_model.py index fda2e7dad4a..93f060f9d3c 100644 --- a/gamestonk_terminal/portfolio/brokers/degiro/degiro_model.py +++ b/gamestonk_terminal/portfolio/brokers/degiro/degiro_model.py @@ -5,7 +5,7 @@ from typing import Union # IMPORTATION THIRDPARTY import pandas as pd -import degiro_connector.core.helpers.pb_handler as pb_handler +from degiro_connector.core.helpers import pb_handler from degiro_connector.trading.api import API as TradingAPI from degiro_connector.trading.models.trading_pb2 import ( diff --git a/gamestonk_terminal/portfolio/brokers/degiro/degiro_view.py b/gamestonk_terminal/portfolio/brokers/degiro/degiro_view.py index 6c493b948aa..2a41bbf91c5 100644 --- a/gamestonk_terminal/portfolio/brokers/degiro/degiro_view.py +++ b/gamestonk_terminal/portfolio/brokers/degiro/degiro_view.py @@ -1,7 +1,7 @@ # IMPORTATION THIRDPARTY from argparse import Namespace import pandas as pd -import degiro_connector.core.helpers.pb_handler as pb_handler +from degiro_connector.core.helpers import pb_handler from degiro_connector.trading.models.trading_pb2 import ( Credentials, diff --git a/gamestonk_terminal/stocks/discovery/shortinterest_model.py b/gamestonk_terminal/stocks/discovery/shortinterest_model.py index 05b3607947b..1a2b146fb67 100644 --- a/gamestonk_terminal/stocks/discovery/shortinterest_model.py +++ b/gamestonk_terminal/stocks/discovery/shortinterest_model.py @@ -68,9 +68,7 @@ def get_today_hot_penny_stocks() -> DataFrame: text_soup_penny_stock_stocks = BeautifulSoup( requests.get( url_penny_stock_stocks, - headers={ - "User-Agent": "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:86.1) Gecko/20100101 Firefox/86.1" - }, + headers={"User-Agent": get_user_agent()}, ).text, "lxml", ) diff --git a/gamestonk_terminal/stocks/fundamental_analysis/dcf_model.py b/gamestonk_terminal/stocks/fundamental_analysis/dcf_model.py index a58a7f802bf..2973f1d125d 100644 --- a/gamestonk_terminal/stocks/fundamental_analysis/dcf_model.py +++ b/gamestonk_terminal/stocks/fundamental_analysis/dcf_model.py @@ -13,6 +13,7 @@ import financedatabase as fd import yfinance as yf import pandas as pd +from gamestonk_terminal.helper_funcs import get_user_agent opts = Union[int, str, float] @@ -440,12 +441,7 @@ red = Font(color="FF0000") fmt_acct = "_($* #,##0.00_);[Red]_($* (#,##0.00);_($* -_0_0_);_(@" -headers = { - "User-Agent": ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " - "(KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36" - ) -} +headers = {"User-Agent": get_user_agent()} tickers = [ "AEIS",