From ec9a88e8fa637f99bb1cac60cb71ebd7820dbd32 Mon Sep 17 00:00:00 2001 From: montezdesousa <79287829+montezdesousa@users.noreply.github.com> Date: Mon, 18 Jul 2022 18:47:45 +0100 Subject: [PATCH] Fill missing static data (sector, industry, country and region) on portfolio creation (#2072) * pull sector industry country data from yfinance * fix imports with missing info * run black on helper * fix tests * fix tests * fix linting * autofill stock data and display progress bar * redirect stock data file * substitute only stocks not provided by user * reset benchmark when new portfolio is loaded (bug fix) * rewrite help, will be fixed in previous pr * fix mypy * fix linting * fix linting with open * format progress bar * run black * avoid clearing of all fields * reformat error message * improve returns calculation performance * general linting Co-authored-by: Jeroen Bouma Co-authored-by: Theodore Aptekarev --- .../portfolio/portfolio_controller.py | 1 + openbb_terminal/portfolio/portfolio_helper.py | 183 ++++++++++++++++++ openbb_terminal/portfolio/portfolio_model.py | 124 +++++++++--- 3 files changed, 281 insertions(+), 27 deletions(-) diff --git a/openbb_terminal/portfolio/portfolio_controller.py b/openbb_terminal/portfolio/portfolio_controller.py index 410e8d09bd5..6592c23a9b9 100644 --- a/openbb_terminal/portfolio/portfolio_controller.py +++ b/openbb_terminal/portfolio/portfolio_controller.py @@ -319,6 +319,7 @@ class PortfolioController(BaseController): str(file_location) ) self.portfolio = portfolio_model.PortfolioModel(orderbook) + self.benchmark_name = "" if ns_parser.name: self.portfolio_name = ns_parser.name diff --git a/openbb_terminal/portfolio/portfolio_helper.py b/openbb_terminal/portfolio/portfolio_helper.py index 2d701838fb7..5dcd897e10b 100644 --- a/openbb_terminal/portfolio/portfolio_helper.py +++ b/openbb_terminal/portfolio/portfolio_helper.py @@ -2,10 +2,15 @@ __docformat__ = "numpy" from datetime import datetime +import os +from pathlib import Path +import csv from dateutil.relativedelta import relativedelta import yfinance as yf import pandas as pd +from openbb_terminal.rich_config import console + # pylint: disable=too-many-return-statements BENCHMARK_LIST = { @@ -113,6 +118,118 @@ BENCHMARK_LIST = { PERIODS = ["mtd", "qtd", "ytd", "3m", "6m", "1y", "3y", "5y", "10y", "all"] +REGIONS = { + "Afghanistan": "Middle East", + "Anguilla": "North America", + "Argentina": "Latin America", + "Australia": "Asia", + "Austria": "Europe", + "Azerbaijan": "Europe", + "Bahamas": "North America", + "Bangladesh": "Asia", + "Barbados": "North America", + "Belgium": "Europe", + "Belize": "North America", + "Bermuda": "North America", + "Botswana": "Africa", + "Brazil": "Latin America", + "British Virgin Islands": "North America", + "Cambodia": "Asia", + "Canada": "North America", + "Cayman Islands": "North America", + "Chile": "Latin America", + "China": "Asia", + "Colombia": "Latin America", + "Costa Rica": "North America", + "Cyprus": "Europe", + "Czech Republic": "Europe", + "Denmark": "Europe", + "Dominican Republic": "North America", + "Egypt": "Middle East", + "Estonia": "Europe", + "Falkland Islands": "Latin America", + "Finland": "Europe", + "France": "Europe", + "French Guiana": "Europe", + "Gabon": "Africa", + "Georgia": "Europe", + "Germany": "Europe", + "Ghana": "Africa", + "Gibraltar": "Europe", + "Greece": "Europe", + "Greenland": "North America", + "Guernsey": "Europe", + "Hong Kong": "Asia", + "Hungary": "Europe", + "Iceland": "Europe", + "India": "Asia", + "Indonesia": "Asia", + "Ireland": "Europe", + "Isle of Man": "Europe", + "Israel": "Middle East", + "Italy": "Europe", + "Ivory Coast": "Africa", + "Japan": "Asia", + "Jersey": "Europe", + "Jordan": "Middle East", + "Kazakhstan": "Asia", + "Kyrgyzstan": "Asia", + "Latvia": "Europe", + "Liechtenstein": "Europe", + "Lithuania": "Europe", + "Luxembourg": "Europe", + "Macau": "Asia", + "Macedonia": "Europe", + "Malaysia": "Asia", + "Malta": "Europe", + "Mauritius": "Africa", + "Mexico": "Latin America", + "Monaco": "Europe", + "Mongolia": "Asia", + "Montenegro": "Europe", + "Morocco": "Africa", + "Mozambique": "Africa", + "Myanmar": "Asia", + "Namibia": "Africa", + "Netherlands": "Europe", + "Netherlands Antilles": "Europe", + "New Zealand": "Asia", + "Nigeria": "Africa", + "Norway": "Europe", + "Panama": "North America", + "Papua New Guinea": "Asia", + "Peru": "Latin America", + "Philippines": "Asia", + "Poland": "Europe", + "Portugal": "Europe", + "Qatar": "Middle East", + "Reunion": "Africa", + "Romania": "Europe", + "Russia": "Asia", + "Saudi Arabia": "Middle East", + "Senegal": "Africa", + "Singapore": "Asia", + "Slovakia": "Europe", + "Slovenia": "Europe", + "South Africa": "Africa", + "South Korea": "Asia", + "Spain": "Europe", + "Suriname": "Latin America", + "Sweden": "Europe", + "Switzerland": "Europe", + "Taiwan": "Asia", + "Tanzania": "Africa", + "Thailand": "Asia", + "Turkey": "Middle East", + "Ukraine": "Europe", + "United Arab Emirates": "Middle East", + "United Kingdom": "Europe", + "United States": "North America", + "Uruguay": "Latin America", + "Vietnam": "Asia", + "Zambia": "Africa", +} + now = datetime.now() PERIODS_DAYS = { "mtd": (now - datetime(now.year, now.month, 1)).days, @@ -267,3 +384,69 @@ def make_equal_length(df1: pd.DataFrame, df2: pd.DataFrame): df2 = df2.loc[df1.index] return df1, df2 + + +def get_region_from_country(country: str) -> str: + return REGIONS[country] + + +def get_info_update_file(ticker: str, file_path: Path, writemode: str) -> list: + + # Pull ticker info from yf + yf_ticker_info = yf.Ticker(ticker).info + + if "sector" in yf_ticker_info.keys(): + # Ticker has valid sector + # Replace the dash to UTF-8 readable + ticker_info_list = [ + yf_ticker_info["sector"], + yf_ticker_info["industry"].replace("—", "-"), + yf_ticker_info["country"], + get_region_from_country(yf_ticker_info["country"]), + ] + + with open(file_path, writemode, newline="") as f: + writer = csv.writer(f) + + if writemode == "a": + # file already has data, so just append + writer.writerow([ticker] + ticker_info_list) + else: + # file did not exist or as empty, so write headers first + writer.writerow(["Ticker", "Sector", "Industry", "Country", "Region"]) + writer.writerow([ticker] + ticker_info_list) + f.close() + return ticker_info_list + # Ticker does not have a valid sector + console.print(f"F:{ticker}", end="") + return ["", "", "", ""] + + +def get_info_from_ticker(ticker: str) -> list: + + filename = "tickers_info.csv" + + # Direct file to openbb_terminal\portfolio + DEFAULT_HOLDINGS_PATH = os.path.abspath( + os.path.join( + os.path.dirname(__file__), "..", "..", "openbb_terminal", "portfolio" + ) + ) + + file_path = Path(str(DEFAULT_HOLDINGS_PATH), filename) + + if file_path.is_file() and os.stat(file_path).st_size > 0: + # file exists and is not empty, so append if necessary + ticker_info_df = pd.read_csv(file_path) + df_row = ticker_info_df.loc[ticker_info_df["Ticker"] == ticker] + + if len(df_row) > 0: + # ticker is in file, just return it + ticker_info_list = list(df_row.iloc[0].drop("Ticker")) + return ticker_info_list + # ticker is not in file, go get it + ticker_info_list = get_info_update_file(ticker, file_path, "a") + return ticker_info_list + # file does not exist or is empty, so write it + ticker_info_list = get_info_update_file(ticker, file_path, "w") + return ticker_info_list diff --git a/openbb_terminal/portfolio/portfolio_model.py b/openbb_terminal/portfolio/portfolio_model.py index be3a70f7f8b..0a799bf7b9a 100644 --- a/openbb_terminal/portfolio/portfolio_model.py +++ b/openbb_terminal/portfolio/portfolio_model.py @@ -859,17 +859,21 @@ class PortfolioModel: # descrbibe outputs try: + console.print(" Preprocessing orderbook: ", end="") # Convert Date to datetime self.__orderbook["Date"] = pd.to_datetime(self.__orderbook["Date"]) + console.print(".", end="") # Sort orderbook by date self.__orderbook = self.__orderbook.sort_values(by="Date") + console.print(".", end="") # Capitalize Ticker and Type [of instrument...] self.__orderbook["Ticker"] = self.__orderbook["Ticker"].map( lambda x: x.upper() ) self.__orderbook["Type"] = self.__orderbook["Type"].map(lambda x: x.upper()) + console.print(".", end="") # Translate side: ["deposit", "buy"] -> 1 and ["withdrawal", "sell"] -> -1 self.__orderbook["Side"] = self.__orderbook["Side"].map( @@ -877,17 +881,20 @@ class PortfolioModel: if x.lower() in ["deposit", "buy"] else (-1 if x.lower() in ["withdrawal", "sell"] else 0) ) + console.print(".", end="") # Convert quantity to signed integer self.__orderbook["Quantity"] = ( - self.__orderbook["Quantity"] * self.__orderbook["Side"] + abs(self.__orderbook["Quantity"]) * self.__orderbook["Side"] ) + console.print(".", end="") # Determining the investment/divestment value self.__orderbook["Investment"] = ( self.__orderbook["Quantity"] * self.__orderbook["Price"] - self.__orderbook["Fees"] ) + console.print(".", end="") # Reformat crypto tickers to yfinance format (e.g. BTC -> BTC-USD) crypto_trades = self.__orderbook[self.__orderbook.Type == "CRYPTO"] @@ -897,6 +904,7 @@ class PortfolioModel: crypto_trades.Ticker, crypto_trades.Currency ) ] + console.print(".", end="") # Create tickers dictionary with structure {'Type': [Ticker]} for ticker_type in set(self.__orderbook["Type"]): @@ -907,22 +915,77 @@ class PortfolioModel: ] ) ) + console.print(".", end="") # Create list with tickers except cash self.tickers_list = list(set(self.__orderbook["Ticker"])) + console.print(".", end="") # Save orderbook inception date self.inception_date = self.__orderbook["Date"][0] + console.print(".", end="") - # Save trades static data - self.static_data = self.__orderbook.pivot( - index="Ticker", - columns=[], - values=["Type", "Sector", "Industry", "Country"], - ) + # Populate fields Sector, Industry and Country + if not ( + {"Sector", "Industry", "Country", "Region"}.issubset( + set(self.__orderbook.columns) + ) + ): + # if fields not in the orderbook add missing + if "Sector" not in self.__orderbook.columns: + self.__orderbook["Sector"] = np.nan + if "Industry" not in self.__orderbook.columns: + self.__orderbook["Industry"] = np.nan + if "Country" not in self.__orderbook.columns: + self.__orderbook["Country"] = np.nan + if "Region" not in self.__orderbook.columns: + self.__orderbook["Region"] = np.nan + + self.load_company_data() + elif ( + self.__orderbook.loc[ + self.__orderbook["Type"] == "STOCK", + ["Sector", "Industry", "Country", "Region"], + ] + .isnull() + .values.any() + ): + # if any fields is empty for Stocks (overwrites any info there) + self.load_company_data() except Exception: - console.print("Could not preprocess orderbook.") + console.print("\nCould not preprocess orderbook.") + + def load_company_data(self): + + console.print("\n Loading company data: ", end="") + + for ticker_type, ticker_list in self.tickers.items(): + + # yfinance only has sector, industry and country for stocks + if ticker_type == "STOCK": + for ticker in ticker_list: + + # Only gets fields for tickers with missing data + # TODO: Should only get field missing for tickers with missing data + # now it's taking the 4 of them + if ( + self.__orderbook.loc[ + self.__orderbook["Ticker"] == ticker, + ["Sector", "Industry", "Country", "Region"], + ] + .isnull() + .values.any() + ): + ticker_info_list = portfolio_helper.get_info_from_ticker(ticker) + + # replace fields in orderbook + self.__orderbook.loc[ + self.__orderbook.Ticker == ticker, + ["Sector", "Industry", "Country", "Region"], + ] = ticker_info_list + # Display progress + console.print(".", end="") def load_benchmark(self, ticker: str = "SPY", full_shares: bool = False): """Adds benchmark dataframe @@ -1004,6 +1067,7 @@ class PortfolioModel: def generate_portfolio_data(self): """Generates portfolio data from orderbook""" + self.load_portfolio_historical_prices() self.populate_historical_trade_data() self.calculate_value() @@ -1062,6 +1126,8 @@ class PortfolioModel: def load_portfolio_historical_prices(self, use_close: bool = False): """Loads historical adj close prices for tickers in list of trades""" + console.print("\n Loading price data: ", end="") + for ticker_type, data in self.tickers.items(): if ticker_type in ["STOCK", "ETF", "CRYPTO"]: # Download yfinance data @@ -1079,13 +1145,16 @@ class PortfolioModel: [self.portfolio_historical_prices, price_data], axis=1 ) else: - console.print("Type not supported in the orderbook.") + console.print(f"Type {ticker_type} not supported.") + + console.print(".", end="") # Fill missing values with last known price self.portfolio_historical_prices.fillna(method="ffill", inplace=True) def populate_historical_trade_data(self): """Create a new dataframe to store historical prices by ticker""" + trade_data = self.__orderbook.pivot( index="Date", columns="Ticker", @@ -1131,6 +1200,9 @@ class PortfolioModel: def calculate_value(self): """Calculate value from historical data""" + + console.print("\n Calculating returns: ", end="") + trade_data = self.historical_trade_data # For each type [STOCK, ETF, etc] calculate value value by trade date @@ -1151,31 +1223,29 @@ class PortfolioModel: for ticker_type, data in self.tickers.items(): self.itemized_value[ticker_type] = trade_data["End Value"][data].sum(axis=1) - # Initial Value = Cumulative Investment - (Previous End Value - Previous Initial Value) trade_data[ - pd.MultiIndex.from_product([["Initial Value"], self.tickers_list]) + pd.MultiIndex.from_product( + [["Initial Value"], self.tickers_list + ["Total"]] + ) ] = 0 - for i, date in enumerate(trade_data.index): - if i == 0: - for t in self.tickers_list: - trade_data.at[date, ("Initial Value", t)] = trade_data.iloc[i][ - "Investment" - ][t] - else: - for t in self.tickers_list: - trade_data.at[date, ("Initial Value", t)] = ( - +trade_data.iloc[i - 1]["End Value"][t] - + trade_data.iloc[i]["Investment"][t] - - trade_data.iloc[i - 1]["Investment"][t] - ) + # Initial Value = Previous End Value + Investment changes + trade_data["Initial Value"] = trade_data["End Value"].shift(1) + trade_data[ + "Investment" + ].diff(periods=1) - trade_data.loc[:, ("Initial Value", "Total")] = trade_data["Initial Value"][ - self.tickers_list - ].sum(axis=1) + # Set first day Initial Value as the Investment (NaNs break first period) + for t in self.tickers_list + ["Total"]: + trade_data.at[trade_data.index[0], ("Initial Value", t)] = trade_data.iloc[ + 0 + ]["Investment"][t] + + console.print(".", end="") self.historical_trade_data = trade_data + console.print("\n") + def calculate_reserves(self): """_summary_""" # TODO: Add back cash dividends and deduct exchange costs