jerry/OpenBB
1
0
Fork 0
mirror of https://github.com/OpenBB-finance/OpenBB.git synced 2026-10-27 12:40:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add ETF support for portfolio allocation command (#2143)

Browse Source 
* fill etf and prevent different length errors

* auto fill etf sectors and allow 0 value portfolios

* allow 0 value portfolios

* remove unused folder

* allow 1 asset portfolios

* split allocation calls by category  in controller

* comment sector allocation model

* get country and region allocations for etf

* add comments and black

* improve comments

* improve comments

* allow for 1 category in sector, country and region

* add progress bars

* linting fix

* fix mypy

* set default date

* fix pylint

* add isin on paexport

* merge main

* auto pre load benchmark

* fix rich table for np.float64

* refactor portfolio allocs

* refactor alloc command

* add isins column

* format output alloc

* rename variable

* fix nan bug

* black

* add ticker conversion to yf format by isin

* warn and removed unsupported ISINs

* solve same day trades bug

* display bench loading progress

* portfolio show

* check if valid isins on preprocessing

* black

* fix bug when region empty

* warn when category is empty

* reformat preprocessing

* codespell

* check if ticker is valid

* flake8

* fix test

* fix bug with trades on holidays

Co-authored-by: Jeroen Bouma <jer.bouma@gmail.com>
This commit is contained in:
montezdesousa
2022-09-09 15:12:59 +01:00
committed by GitHub
parent 10fce6d1d7
commit 291e7d6991
7 changed files with 641 additions and 214 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
openbb_terminal/helper_funcs.py
View File

@@ -271,7 +271,7 @@ def print_rich_table(
row = [str(idx)] if show_index else []
row += [
str(x)
if not isinstance(x, float) or not isinstance(x, np.float64)
if not isinstance(x, float) and not isinstance(x, np.float64)
else (
f"{x:{floatfmt[idx]}}"
if isinstance(floatfmt, list)

295
openbb_terminal/portfolio/allocation_model.py
View File

@@ -3,13 +3,15 @@ from typing import Dict
import pandas as pd
import requests
import yfinance as yf
from openbb_terminal.decorators import log_start_end
from openbb_terminal.rich_config import console
logger = logging.getLogger(__name__)
@log_start_end(log=logger)
def obtain_assets_allocation(benchmark_info: Dict, portfolio_trades: pd.DataFrame):
def get_assets_allocation(benchmark_info: Dict, portfolio_trades: pd.DataFrame):
"""Obtain the assets allocation of the benchmark and portfolio [Source: Yahoo Finance]
Parameters
@@ -27,22 +29,28 @@ def obtain_assets_allocation(benchmark_info: Dict, portfolio_trades: pd.DataFram
Dictionary with the portfolio's asset allocations
"""
benchmark_assets_allocation = pd.DataFrame(benchmark_info["holdings"])
portfolio_assets_allocation = (
(
portfolio_trades[portfolio_trades["Type"] != "CASH"]
.groupby(by="Ticker")
.agg({"Portfolio Value": "sum"})
.div(portfolio_trades["Portfolio Value"].sum())
)
.squeeze()
.sort_values(ascending=False)
benchmark_assets_allocation.rename(
columns={"symbol": "Symbol", "holdingPercent": "Benchmark"}, inplace=True
)
benchmark_assets_allocation.drop(columns=["holdingName"], inplace=True)
portfolio_assets_allocation = (
portfolio_trades[portfolio_trades["Type"] != "CASH"]
.groupby(by="Ticker")
.agg({"Portfolio Value": "sum"})
.div(portfolio_trades["Portfolio Value"].sum())
).sort_values(by="Portfolio Value", ascending=False)
portfolio_assets_allocation.reset_index(inplace=True)
portfolio_assets_allocation.rename(
columns={"Ticker": "Symbol", "Portfolio Value": "Portfolio"}, inplace=True
)
portfolio_assets_allocation.fillna(0, inplace=True)
return benchmark_assets_allocation, portfolio_assets_allocation
@log_start_end(log=logger)
def obtain_sector_allocation(benchmark_info: Dict, portfolio_trades: pd.DataFrame):
def get_sector_allocation(benchmark_info: Dict, portfolio_trades: pd.DataFrame):
"""Obtain the sector allocation of the benchmark and portfolio [Source: Yahoo Finance]
Parameters
@@ -59,6 +67,9 @@ def obtain_sector_allocation(benchmark_info: Dict, portfolio_trades: pd.DataFram
country_allocation: dict
Dictionary with country allocations
"""
console.print(" Loading sector data: ", end="")
benchmark_sectors_allocation = (
pd.DataFrame.from_dict(
data={
@@ -80,32 +91,110 @@ def obtain_sector_allocation(benchmark_info: Dict, portfolio_trades: pd.DataFram
benchmark_sectors_allocation.index = prettified
# Define portfolio sector allocation
# Aggregate sector value for stocks and crypto
portfolio_sectors_allocation = (
(
portfolio_trades[portfolio_trades["Type"] != "CASH"]
.groupby(by="Sector")
.agg({"Portfolio Value": "sum"})
portfolio_trades[portfolio_trades["Type"].isin(["STOCK", "CRYPTO"])]
.groupby(by="Sector")
.agg({"Portfolio Value": "sum"})
)
# Aggregate sector value for ETFs
# Start by getting value by isin/ticker
etf_ticker_value = (
portfolio_trades[portfolio_trades["Type"].isin(["ETF"])]
.groupby(by="Ticker")
.agg({"Portfolio Value": "sum"})
)
etf_global_sector_alloc = pd.DataFrame()
# Loop through each etf and multiply sector weights by current value
for item in etf_ticker_value.index.values:
# TODO: This can be improved by caching this info similar to what is done in stocks
etf_info = yf.Ticker(item).info
try:
etf_sector_weight = pd.DataFrame.from_dict(
data={
sector_name: allocation
for sector in etf_info["sectorWeightings"]
for sector_name, allocation in sector.items()
},
orient="index",
columns=["Portfolio Value"],
)
except Exception:
# If ETF has no sectors like VIX for example or it was not found, add to Other
etf_sector_weight = pd.DataFrame.from_dict(
data={"Other": 1}, orient="index", columns=["Portfolio Value"]
)
etf_value = etf_ticker_value["Portfolio Value"][item]
etf_ticker_sector_alloc = etf_sector_weight * etf_value
# Aggregate etf sector allocation by value
etf_global_sector_alloc = pd.concat(
[etf_global_sector_alloc, etf_ticker_sector_alloc], axis=1
)
.div(portfolio_trades["Portfolio Value"].sum())
.squeeze()
etf_global_sector_alloc.fillna(0, inplace=True)
etf_global_sector_alloc = etf_global_sector_alloc.sum(axis=1)
console.print(".", end="")
console.print("\n")
etf_global_sector_alloc = pd.DataFrame(
etf_global_sector_alloc, columns=["Portfolio Value"]
)
# Rename columns to match stock and crypto classification
etf_global_sector_alloc.index.name = "Sector"
prettified = []
for sector in etf_global_sector_alloc.index:
prettified.append(sector.replace("_", " ").title())
etf_global_sector_alloc.index = prettified
# Aggregate sector allocation for stocks and crypto with ETFs
portfolio_sectors_allocation = pd.merge(
portfolio_sectors_allocation,
etf_global_sector_alloc,
how="outer",
left_index=True,
right_index=True,
).sum(axis=1)
portfolio_sectors_allocation = pd.DataFrame(
portfolio_sectors_allocation, columns=["Portfolio Value"]
)
portfolio_sectors_allocation = (
portfolio_sectors_allocation.div(portfolio_trades["Portfolio Value"].sum())
.squeeze(axis=1)
.sort_values(ascending=False)
)
portfolio_sectors_allocation.fillna(0, inplace=True)
return benchmark_sectors_allocation, portfolio_sectors_allocation
@log_start_end(log=logger)
def obtain_benchmark_regional_and_country_allocation(
benchmark_ticker: str, regional_test: list = None, country_test: list = None
def get_region_country_allocation(
ticker: str, region_test: list = None, country_test: list = None
):
"""Obtain the regional and country allocation of the benchmark ticker. [Source: Fidelity.com]
"""Obtain the region and country allocation for ETF ticker. [Source: Fidelity.com]
Parameters
----------
benchmark_ticker: str
The benchmark ticker, e.g. "SPY"
regional_test: list
This includes a list of regional names that should be used to discover
The ticker, e.g. "SPY"
region_test: list
This includes a list of region names that should be used to discover
which list on the Fidelity page is the correct one
country_test: list
This includes a list of country names that should be used to discover
@@ -113,14 +202,15 @@ def obtain_benchmark_regional_and_country_allocation(
Returns
------
benchmark_regional_allocation: dict
region_allocation: dict
Dictionary with regional allocations.
benchmark_country_allocation: dict
country_allocation: dict
Dictionary with country allocations
"""
# Initialize variables
if not regional_test:
regional_test = [
if not region_test:
region_test = [
"North America",
"Europe",
"Asia",
@@ -137,52 +227,52 @@ def obtain_benchmark_regional_and_country_allocation(
"China",
]
regional_list = 0
region_list = 0
country_list = 0
# Collect data from Fidelity about the portfolio composition of the benchmark
URL = f"https://screener.fidelity.com/ftgw/etf/goto/snapshot/portfolioComposition.jhtml?symbols={benchmark_ticker}"
URL = f"https://screener.fidelity.com/ftgw/etf/goto/snapshot/portfolioComposition.jhtml?symbols={ticker}"
html = requests.get(URL).content
df_list = pd.read_html(html)
# Find the ones that contain regions and countries
for index, item in enumerate(df_list):
for region in regional_test:
for region in region_test:
if region in item.values:
regional_list = index
region_list = index
break
for country in country_test:
if country in item.values:
country_list = index
break
if regional_list:
benchmark_regional_allocation = {
if region_list:
region_allocation = {
row[1]: float(row[2].strip("%")) / 100
for _, row in df_list[regional_list].dropna(axis="columns").iterrows()
for _, row in df_list[region_list].dropna(axis="columns").iterrows()
}
benchmark_regional_allocation = pd.DataFrame.from_dict(
benchmark_regional_allocation, orient="index"
region_allocation = pd.DataFrame.from_dict(
region_allocation, orient="index"
).squeeze()
else:
benchmark_regional_allocation = pd.DataFrame()
region_allocation = pd.DataFrame()
if country_list:
benchmark_country_allocation = {
country_allocation = {
row[1]: float(row[2].strip("%")) / 100
for _, row in df_list[country_list].dropna(axis="columns").iterrows()
}
benchmark_country_allocation = pd.DataFrame.from_dict(
benchmark_country_allocation, orient="index"
country_allocation = pd.DataFrame.from_dict(
country_allocation, orient="index"
).squeeze()
else:
benchmark_country_allocation = pd.DataFrame()
country_allocation = pd.DataFrame()
return benchmark_regional_allocation, benchmark_country_allocation
return region_allocation, country_allocation
@log_start_end(log=logger)
def obtain_portfolio_regional_and_country_allocation(portfolio_trades: pd.DataFrame):
def get_portfolio_region_country_allocation(portfolio_trades: pd.DataFrame):
"""Obtain the regional and country allocation of the portfolio.
Parameters
@@ -197,28 +287,121 @@ def obtain_portfolio_regional_and_country_allocation(portfolio_trades: pd.DataFr
portfolio_country_allocation: pd.DataFrame
Dictionary with country allocations
"""
console.print(" Loading country/region data: ", end="")
# Define portfolio regional allocation
portfolio_regional_allocation = (
(
portfolio_trades[portfolio_trades["Type"] != "CASH"]
if not portfolio_trades["Region"].isnull().any():
portfolio_region_allocation = (
portfolio_trades[portfolio_trades["Type"].isin(["STOCK", "CRYPTO"])]
.groupby(by="Region")
.agg({"Portfolio Value": "sum"})
)
.div(portfolio_trades["Portfolio Value"].sum())
.squeeze()
.sort_values(ascending=False)
)
else:
portfolio_region_allocation = pd.DataFrame()
# Define portfolio country allocation
portfolio_country_allocation = (
(
portfolio_trades[portfolio_trades["Type"] != "CASH"]
if not portfolio_trades["Country"].isnull().any():
portfolio_country_allocation = (
portfolio_trades[portfolio_trades["Type"].isin(["STOCK", "CRYPTO"])]
.groupby(by="Country")
.agg({"Portfolio Value": "sum"})
)
.div(portfolio_trades["Portfolio Value"].sum())
.squeeze()
else:
portfolio_country_allocation = pd.DataFrame()
# Aggregate sector value for ETFs
# Start by getting value by ticker
etf_ticker_value = (
portfolio_trades[portfolio_trades["Type"].isin(["ETF"])]
.groupby(by="Ticker")
.agg({"Portfolio Value": "sum"})
)
etf_global_region_alloc = pd.DataFrame()
etf_global_country_alloc = pd.DataFrame()
# Loop through each etf and multiply sector weights by current value
for item in etf_ticker_value.index.values:
etf_region_weight, etf_country_weight = get_region_country_allocation(item)
if etf_region_weight.empty:
# If ETF has no sectors like VIX for example or it was not found, add to Other
etf_region_weight = pd.DataFrame.from_dict(
data={"Other": 1}, orient="index", columns=["Portfolio Value"]
)
if etf_country_weight.empty:
etf_country_weight = pd.DataFrame.from_dict(
data={"Other": 1}, orient="index", columns=["Portfolio Value"]
)
etf_value = etf_ticker_value["Portfolio Value"][item]
# Aggregate etf region allocation by value
etf_ticker_region_alloc = etf_region_weight * etf_value
etf_global_region_alloc = pd.concat(
[etf_global_region_alloc, etf_ticker_region_alloc], axis=1
)
etf_global_region_alloc.fillna(0, inplace=True)
etf_global_region_alloc = etf_global_region_alloc.sum(axis=1)
# Aggregate etf country allocation by value
etf_ticker_country_alloc = etf_country_weight * etf_value
etf_global_country_alloc = pd.concat(
[etf_global_country_alloc, etf_ticker_country_alloc], axis=1
)
etf_global_country_alloc.fillna(0, inplace=True)
etf_global_country_alloc = etf_global_country_alloc.sum(axis=1)
console.print(".", end="")
console.print("\n")
etf_global_region_alloc = pd.DataFrame(
etf_global_region_alloc, columns=["Portfolio Value"]
)
etf_global_country_alloc = pd.DataFrame(
etf_global_country_alloc, columns=["Portfolio Value"]
)
# Aggregate region allocation for stocks and crypto with ETFs
portfolio_region_allocation = pd.merge(
portfolio_region_allocation,
etf_global_region_alloc,
how="outer",
left_index=True,
right_index=True,
).sum(axis=1)
# Aggregate country allocation for stocks and crypto with ETFs
portfolio_country_allocation = pd.merge(
portfolio_country_allocation,
etf_global_country_alloc,
how="outer",
left_index=True,
right_index=True,
).sum(axis=1)
portfolio_region_allocation = pd.DataFrame(
portfolio_region_allocation, columns=["Portfolio Value"]
)
portfolio_country_allocation = pd.DataFrame(
portfolio_country_allocation, columns=["Portfolio Value"]
)
portfolio_region_allocation = (
portfolio_region_allocation.div(portfolio_trades["Portfolio Value"].sum())
.squeeze(axis=1)
.sort_values(ascending=False)
)
return portfolio_regional_allocation, portfolio_country_allocation
portfolio_country_allocation = (
portfolio_country_allocation.div(portfolio_trades["Portfolio Value"].sum())
.squeeze(axis=1)
.sort_values(ascending=False)
)
portfolio_region_allocation.fillna(0, inplace=True)
portfolio_country_allocation.fillna(0, inplace=True)
return portfolio_region_allocation, portfolio_country_allocation

4
openbb_terminal/portfolio/brokers/degiro/degiro_model.py
View File

@@ -404,13 +404,14 @@ class DegiroModel:
products_df["productId"] = products_df["productId"].astype("int")
transactions_full_df = pd.merge(
transactions_df,
products_df[{"productId", "symbol", "productType"}],
products_df[{"productId", "symbol", "productType", "isin"}],
on="productId",
)
portfolio_df = transactions_full_df.rename(
columns={
"date": "Date",
"isin": "ISIN",
"symbol": "Ticker",
"productType": "Type", # STOCK or ETF
"price": "Price",
@@ -428,6 +429,7 @@ class DegiroModel:
).dt.date
columns = [
"Date",
"ISIN",
"Ticker",
"Type",
"Price",

53
openbb_terminal/portfolio/portfolio_controller.py
View File

@@ -342,16 +342,40 @@ class PortfolioController(BaseController):
# Add in the Risk-free rate
self.portfolio.set_risk_free_rate(ns_parser.risk_free_rate)
console.print(f"\n[bold]Portfolio:[/bold] {self.portfolio_name}")
# Load benchmark
self.call_bench(["-b", "SPDR S&P 500 ETF Trust (SPY)"])
console.print(
f"[bold]Risk Free Rate:[/bold] {self.portfolio.risk_free_rate}"
f"\n[bold][param]Portfolio:[/param][/bold] {self.portfolio_name}"
)
console.print(
f"[bold][param]Risk Free Rate:[/param][/bold] {self.portfolio.risk_free_rate}"
)
console.print(
f"[bold][param]Benchmark:[/param][/bold] {self.benchmark_name}\n"
)
console.print()
@log_start_end(log=logger)
def call_show(self, _):
def call_show(self, other_args: List[str]):
"""Process show command"""
portfolio_view.display_orderbook(self.portfolio, show_index=False)
parser = argparse.ArgumentParser(
add_help=False,
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
prog="show",
description="Show transactions table",
)
ns_parser = self.parse_known_args_and_warn(
parser,
other_args,
export_allowed=EXPORT_BOTH_RAW_DATA_AND_FIGURES,
limit=10,
)
portfolio_view.display_orderbook(
self.portfolio,
show_index=False,
limit=ns_parser.limit,
export=ns_parser.export,
)
@log_start_end(log=logger)
def call_bench(self, other_args: List[str]):
@@ -382,7 +406,6 @@ class PortfolioController(BaseController):
)
if other_args and "-" not in other_args[0][0]:
other_args.insert(0, "-b")
ns_parser = self.parse_known_args_and_warn(parser, other_args)
if ns_parser and self.portfolio is not None:
@@ -409,9 +432,6 @@ class PortfolioController(BaseController):
self.portfolio.returns, self.portfolio.benchmark_returns
)
console.print(
f"[bold]\nBenchmark:[/bold] {self.benchmark_name} ({benchmark_ticker})"
)
else:
console.print("[red]Please first load orderbook using 'load'[/red]\n")
console.print()
@@ -454,10 +474,9 @@ class PortfolioController(BaseController):
if check_portfolio_benchmark_defined(
self.portfolio_name, self.benchmark_name
):
if self.portfolio.portfolio_assets_allocation.empty:
self.portfolio.calculate_allocations()
if ns_parser.agg == "assets":
if self.portfolio.portfolio_assets_allocation.empty:
self.portfolio.calculate_allocations("asset")
portfolio_view.display_assets_allocation(
self.portfolio.portfolio_assets_allocation,
self.portfolio.benchmark_assets_allocation,
@@ -465,6 +484,8 @@ class PortfolioController(BaseController):
ns_parser.tables,
)
elif ns_parser.agg == "sectors":
if self.portfolio.portfolio_sectors_allocation.empty:
self.portfolio.calculate_allocations("sector")
portfolio_view.display_category_allocation(
ns_parser.agg,
self.portfolio.portfolio_sectors_allocation,
@@ -473,6 +494,8 @@ class PortfolioController(BaseController):
ns_parser.tables,
)
elif ns_parser.agg == "countries":
if self.portfolio.portfolio_country_allocation.empty:
self.portfolio.calculate_allocations("country")
portfolio_view.display_category_allocation(
ns_parser.agg,
self.portfolio.portfolio_country_allocation,
@@ -481,10 +504,12 @@ class PortfolioController(BaseController):
ns_parser.tables,
)
elif ns_parser.agg == "regions":
if self.portfolio.portfolio_region_allocation.empty:
self.portfolio.calculate_allocations("region")
portfolio_view.display_category_allocation(
ns_parser.agg,
self.portfolio.portfolio_regional_allocation,
self.portfolio.benchmark_regional_allocation,
self.portfolio.portfolio_region_allocation,
self.portfolio.benchmark_region_allocation,
ns_parser.limit,
ns_parser.tables,
)

421
openbb_terminal/portfolio/portfolio_model.py
View File

@@ -1,8 +1,10 @@
"""Portfolio Model"""
__docformat__ = "numpy"
import contextlib
import logging
from typing import Dict, Any, Tuple
import datetime
import numpy as np
import scipy
@@ -921,10 +923,6 @@ class PortfolioModel:
Class for portfolio analysis in OpenBB
Implements a Portfolio and related methods.
Attributes
-------
Methods
-------
read_orderbook: Class method to read orderbook from file
@@ -938,36 +936,31 @@ class PortfolioModel:
generate_portfolio_data: Generates portfolio data from orderbook
load_portfolio_historical_prices: Loads historical adj close prices for tickers in list of trades
populate_historical_trade_data: Create a new dataframe to store historical prices by ticker
calculate_value: Calculate value from historical data
calculate_value: Calculate value end of day from historical data
calculate_reserves:
calculate_reserves: Takes dividends into account for returns calculation
calculate_allocations:
calculate_allocations: Determine allocations based on assets, sectors, countries and region
set_risk_free_rate: Sets risk free rate
calculate_metrics:
"""
@log_start_end(log=logger)
def __init__(self, orderbook: pd.DataFrame = pd.DataFrame()):
"""Initialize Portfolio class"""
"""Initialize PortfolioModel class"""
# Portfolio
self.tickers_list = None
self.tickers: Dict[Any, Any] = {}
self.inception_date = None
self.static_data = pd.DataFrame()
self.inception_date = datetime.date(1970, 1, 1)
self.historical_trade_data = pd.DataFrame()
self.returns = pd.DataFrame()
self.itemized_value = pd.DataFrame()
self.portfolio_trades = pd.DataFrame()
self.portfolio_value = None
self.portfolio_assets_allocation = pd.DataFrame()
self.portfolio_regional_allocation = pd.DataFrame()
self.portfolio_country_allocation = pd.DataFrame()
self.portfolio_historical_prices = pd.DataFrame()
self.empty = True
self.risk_free_rate = float(0)
# Benchmark
@@ -976,8 +969,16 @@ class PortfolioModel:
self.benchmark_historical_prices = pd.DataFrame()
self.benchmark_returns = pd.DataFrame()
self.benchmark_trades = pd.DataFrame()
# Allocations
self.portfolio_assets_allocation = pd.DataFrame()
self.portfolio_sectors_allocation = pd.DataFrame()
self.portfolio_region_allocation = pd.DataFrame()
self.portfolio_country_allocation = pd.DataFrame()
self.benchmark_assets_allocation = pd.DataFrame()
self.benchmark_regional_allocation = pd.DataFrame()
self.benchmark_sectors_allocation = pd.DataFrame()
self.benchmark_region_allocation = pd.DataFrame()
self.benchmark_country_allocation = pd.DataFrame()
# Set and preprocess orderbook
@@ -990,14 +991,43 @@ class PortfolioModel:
self.empty = False
def get_orderbook(self):
return self.__orderbook
"""Get formatted transactions
Returns:
pd.DataFrame: formatted transactions
"""
df = self.__orderbook[
[
"Date",
"Type",
"Ticker",
"Side",
"Price",
"Quantity",
"Fees",
"Investment",
"Currency",
"Sector",
"Industry",
"Country",
"Region",
]
]
df = df.replace(np.nan, "-")
df["Date"] = df["Date"].dt.strftime("%Y-%m-%d")
df.sort_values(by="Date", ascending=False, inplace=True)
return df
@staticmethod
@log_start_end(log=logger)
def read_orderbook(path: str) -> pd.DataFrame:
"""Class method to read orderbook from file
"""Static method to read orderbook from file
Parameters
----------
path: str
path to orderbook file
Args:
path (str): path to orderbook file
"""
# Load orderbook from file
if path.endswith(".xlsx"):
@@ -1007,15 +1037,30 @@ class PortfolioModel:
return orderbook
@log_start_end(log=logger)
def preprocess_orderbook(self):
"""Method to preprocess, format and compute auxiliary fields"""
# descrbibe outputs
# Preprocessing steps:
# 0. If optional fields not in the orderbook add missing
# 1. Convert Date to datetime
# 2. Sort orderbook by date
# 3. Capitalize Ticker and Type [of instrument...]
# 4. Translate side: ["deposit", "buy"] -> 1 and ["withdrawal", "sell"] -> -1
# 5. Convert quantity to signed integer
# 6. Determining the investment/divestment value
# 7. Reformat crypto tickers to yfinance format (e.g. BTC -> BTC-USD)
# 8. Reformat STOCK/ETF tickers to yfinance format if ISIN provided
# 9. Remove unsupported ISINs that came out empty
# 10. Create tickers dictionary with structure {'Type': [Ticker]}
# 11. Create list with tickers except cash
# 12. Save orderbook inception date
# 13. Populate fields Sector, Industry and Country
try:
console.print(" Preprocessing orderbook: ", end="")
# if optional fields not in the orderbook add missing
# 0. If optional fields not in the orderbook add missing
optional_fields = [
"Sector",
"Industry",
@@ -1023,61 +1068,50 @@ class PortfolioModel:
"Region",
"Fees",
"Premium",
"ISIN",
]
if not set(optional_fields).issubset(set(self.__orderbook.columns)):
for field in optional_fields:
if field not in self.__orderbook.columns:
self.__orderbook[field] = np.nan
# if optional fields for stock data has missing datapoints, fill them
if (
self.__orderbook.loc[
self.__orderbook["Type"] == "STOCK",
optional_fields,
]
.isnull()
.values.any()
):
# if any fields is empty for Stocks (overwrites any info there)
self.load_company_data()
# Convert Date to datetime
# 1. Convert Date to datetime
self.__orderbook["Date"] = pd.to_datetime(self.__orderbook["Date"])
console.print(".", end="")
# Sort orderbook by date
# 3. Sort orderbook by date
self.__orderbook = self.__orderbook.sort_values(by="Date")
console.print(".", end="")
# Capitalize Ticker and Type [of instrument...]
# 3. Capitalize Ticker and Type [of instrument...]
self.__orderbook["Ticker"] = self.__orderbook["Ticker"].map(
lambda x: x.upper()
)
self.__orderbook["Type"] = self.__orderbook["Type"].map(lambda x: x.upper())
console.print(".", end="")
# Translate side: ["deposit", "buy"] -> 1 and ["withdrawal", "sell"] -> -1
self.__orderbook["Side"] = self.__orderbook["Side"].map(
# 4. Translate side: ["deposit", "buy"] -> 1 and ["withdrawal", "sell"] -> -1
self.__orderbook["Signal"] = self.__orderbook["Side"].map(
lambda x: 1
if x.lower() in ["deposit", "buy"]
else (-1 if x.lower() in ["withdrawal", "sell"] else 0)
)
console.print(".", end="")
# Convert quantity to signed integer
# 5. Convert quantity to signed integer
self.__orderbook["Quantity"] = (
abs(self.__orderbook["Quantity"]) * self.__orderbook["Side"]
abs(self.__orderbook["Quantity"]) * self.__orderbook["Signal"]
)
console.print(".", end="")
# Determining the investment/divestment value
# 6. Determining the investment/divestment value
self.__orderbook["Investment"] = (
self.__orderbook["Quantity"] * self.__orderbook["Price"]
- self.__orderbook["Fees"]
+ self.__orderbook["Fees"]
)
console.print(".", end="")
# Reformat crypto tickers to yfinance format (e.g. BTC -> BTC-USD)
# 7. Reformat crypto tickers to yfinance format (e.g. BTC -> BTC-USD)
crypto_trades = self.__orderbook[self.__orderbook.Type == "CRYPTO"]
self.__orderbook.loc[(self.__orderbook.Type == "CRYPTO"), "Ticker"] = [
f"{crypto}-{currency}"
@@ -1087,7 +1121,57 @@ class PortfolioModel:
]
console.print(".", end="")
# Create tickers dictionary with structure {'Type': [Ticker]}
# 8. Reformat STOCK/ETF tickers to yfinance format if ISIN provided.
# If isin not valid ticker is empty
self.__orderbook["yf_Ticker"] = self.__orderbook["ISIN"].apply(
lambda x: yf.utils.get_ticker_by_isin(x) if not pd.isna(x) else np.nan
)
empty_tickers = list(
self.__orderbook[
(self.__orderbook["yf_Ticker"] == "")
| (self.__orderbook["yf_Ticker"].isna())
]["Ticker"].unique()
)
# If ticker from isin is empty it is not valid in yfinance, so check if user provided ticker is supported
removed_tickers = []
for item in empty_tickers:
with contextlib.redirect_stdout(None):
# Suppress yfinance failed download message if occurs
valid_ticker = not (
yf.download(
item,
start=datetime.datetime.now() + datetime.timedelta(days=-5),
progress=False,
).empty
)
if valid_ticker:
# Invalid ISIN but valid ticker
self.__orderbook.loc[
self.__orderbook["Ticker"] == item, "yf_Ticker"
] = np.nan
else:
self.__orderbook.loc[
self.__orderbook["Ticker"] == item, "yf_Ticker"
] = ""
removed_tickers.append(item)
# Merge reformated tickers into Ticker
self.__orderbook["Ticker"] = self.__orderbook["yf_Ticker"].fillna(
self.__orderbook["Ticker"]
)
console.print(".", end="")
# 9. Remove unsupported ISINs that came out empty
self.__orderbook.drop(
self.__orderbook[self.__orderbook["Ticker"] == ""].index, inplace=True
)
console.print(".", end="")
# 10. Create tickers dictionary with structure {'Type': [Ticker]}
for ticker_type in set(self.__orderbook["Type"]):
self.tickers[ticker_type] = list(
set(
@@ -1098,27 +1182,49 @@ class PortfolioModel:
)
console.print(".", end="")
# Create list with tickers except cash
# 11. Create list with tickers except cash
self.tickers_list = list(set(self.__orderbook["Ticker"]))
console.print(".", end="")
# Save orderbook inception date
# 12. Save orderbook inception date
self.inception_date = self.__orderbook["Date"][0]
console.print(".", end="")
# 13. Populate fields Sector, Industry and Country
if (
self.__orderbook.loc[
self.__orderbook["Type"] == "STOCK",
optional_fields,
]
.isnull()
.values.any()
):
# if any fields is empty for Stocks (overwrites any info there)
self.load_company_data()
console.print(".", end="")
# Warn user of removed ISINs
if removed_tickers:
console.print(
f"\n\n[red]The following tickers are not supported and were removed: {removed_tickers}."
f"\nManually edit the 'Ticker' field with the proper Yahoo Finance suffix or provide a valid ISIN."
f"\nSuffix info on 'Yahoo Finance market coverage':"
" https://help.yahoo.com/kb/exchanges-data-providers-yahoo-finance-sln2310.html"
f"\nE.g. IWDA -> IWDA.AS[/red]"
)
except Exception:
console.print("\nCould not preprocess orderbook.")
@log_start_end(log=logger)
def load_company_data(self):
"""Method to populate company data for stocks such as sector, industry and country"""
console.print("\n Loading company data: ", end="")
for ticker_type, ticker_list in self.tickers.items():
# yfinance only has sector, industry and country for stocks
if ticker_type == "STOCK":
for ticker in ticker_list:
# Only gets fields for tickers with missing data
# TODO: Should only get field missing for tickers with missing data
# now it's taking the 4 of them
@@ -1130,34 +1236,117 @@ class PortfolioModel:
.isnull()
.values.any()
):
ticker_info_list = portfolio_helper.get_info_from_ticker(ticker)
# Get ticker info in list ["Sector", "Industry", "Country", "Region"] from isin/ticker
info_list = portfolio_helper.get_info_from_ticker(ticker)
# replace fields in orderbook
# Replace fields in orderbook
self.__orderbook.loc[
self.__orderbook.Ticker == ticker,
["Sector", "Industry", "Country", "Region"],
] = ticker_info_list
] = info_list
# Display progress
console.print(".", end="")
elif ticker_type == "CRYPTO":
for ticker in ticker_list:
if (
self.__orderbook.loc[
self.__orderbook["Ticker"] == ticker,
["Sector", "Industry", "Country", "Region"],
]
.isnull()
.values.any()
):
# Get ticker info in list ["Sector", "Industry", "Country", "Region"]
info_list = ["Crypto", "Crypto", "Crypto", "Crypto"]
# Replace fields in orderbook
self.__orderbook.loc[
self.__orderbook.Ticker == ticker,
["Sector", "Industry", "Country", "Region"],
] = info_list
# Display progress
console.print(".", end="")
else:
for ticker in ticker_list:
if (
self.__orderbook.loc[
self.__orderbook["Ticker"] == ticker,
["Sector", "Industry", "Country", "Region"],
]
.isnull()
.values.any()
):
# Get ticker info in list ["Sector", "Industry", "Country", "Region"]
info_list = ["-", "-", "-", "-"]
# Replace fields in orderbook
self.__orderbook.loc[
self.__orderbook.Ticker == ticker,
["Sector", "Industry", "Country", "Region"],
] = info_list
# Display progress
console.print(".", end="")
@log_start_end(log=logger)
def load_benchmark(self, ticker: str = "SPY", full_shares: bool = False):
"""Adds benchmark dataframe
Args:
ticker (str): benchmark ticker to download data
full_shares (bool): whether to mimic the portfolio trades exactly (partial shares) or round down the
Parameters
----------
ticker: str
benchmark ticker to download data
full_shares: bool
whether to mimic the portfolio trades exactly (partial shares) or round down the
quantity to the nearest number.
"""
console.print("\n Loading benchmark: ", end="")
self.benchmark_ticker = ticker
self.benchmark_historical_prices = yf.download(
ticker, start=self.inception_date, threads=False, progress=False
ticker,
start=self.inception_date - datetime.timedelta(days=1),
threads=False,
progress=False,
)["Adj Close"]
self.benchmark_returns = self.benchmark_historical_prices.pct_change().dropna()
self.benchmark_info = yf.Ticker(ticker).info
self.mimic_trades_for_benchmark(full_shares)
# Merge benchmark and portfolio dates to ensure same length
self.benchmark_historical_prices = pd.merge(
self.portfolio_historical_prices["Close"],
self.benchmark_historical_prices,
how="outer",
left_index=True,
right_index=True,
)["Adj Close"]
self.benchmark_historical_prices.fillna(method="ffill", inplace=True)
self.benchmark_returns = self.benchmark_historical_prices.pct_change().dropna()
self.benchmark_info = yf.Ticker(ticker).info
# Display progress
console.print(".")
@log_start_end(log=logger)
def mimic_trades_for_benchmark(self, full_shares: bool = False):
"""Mimic trades from the orderbook based on chosen benchmark assuming partial shares"""
"""Mimic trades from the orderbook based on chosen benchmark assuming partial shares
Parameters
----------
full_shares: bool
whether to mimic the portfolio trades exactly (partial shares) or round down the
quantity to the nearest number.
"""
# Create dataframe to store benchmark trades
self.benchmark_trades = self.__orderbook[["Date", "Type", "Investment"]].copy()
@@ -1223,6 +1412,7 @@ class PortfolioModel:
f" in the benchmark ({round(sum(self.benchmark_trades['Benchmark Investment']), 2)})."
)
@log_start_end(log=logger)
def generate_portfolio_data(self):
"""Generates portfolio data from orderbook"""
@@ -1255,6 +1445,7 @@ class PortfolioModel:
self.portfolio_returns = pd.DataFrame(self.__orderbook["Date"])
# Save portfolio trades to compute allocations later
self.portfolio_trades = self.__orderbook.copy()
self.portfolio_trades[
[
@@ -1281,8 +1472,16 @@ class PortfolioModel:
- self.portfolio_trades["Portfolio Investment"][index]
)
@log_start_end(log=logger)
def load_portfolio_historical_prices(self, use_close: bool = False):
"""Loads historical adj close prices for tickers in list of trades"""
"""Loads historical adj close/close prices for tickers in list of trades
Parameters
----------
use_close: bool
whether to use close or adjusted close prices
"""
console.print("\n Loading price data: ", end="")
@@ -1310,25 +1509,18 @@ class PortfolioModel:
# Fill missing values with last known price
self.portfolio_historical_prices.fillna(method="ffill", inplace=True)
@log_start_end(log=logger)
def populate_historical_trade_data(self):
"""Create a new dataframe to store historical prices by ticker"""
trade_data = self.__orderbook.pivot(
trade_data = self.__orderbook.pivot_table(
index="Date",
columns="Ticker",
columns=["Ticker"],
values=[
"Type",
"Sector",
"Industry",
"Country",
"Price",
"Quantity",
"Fees",
"Premium",
"Investment",
"Side",
"Currency",
],
aggfunc={"Quantity": np.sum, "Investment": np.sum},
)
# Make historical prices columns a multi-index. This helps the merging.
@@ -1340,10 +1532,13 @@ class PortfolioModel:
trade_data = pd.merge(
trade_data,
self.portfolio_historical_prices,
how="right",
how="outer",
left_index=True,
right_index=True,
).fillna(0)
)
trade_data["Close"] = trade_data["Close"].fillna(method="ffill")
trade_data.fillna(0, inplace=True)
# Accumulate quantity held by trade date
trade_data["Quantity"] = trade_data["Quantity"].cumsum()
@@ -1356,8 +1551,9 @@ class PortfolioModel:
self.historical_trade_data = trade_data
@log_start_end(log=logger)
def calculate_value(self):
"""Calculate value from historical data"""
"""Calculate end of day value from historical data"""
console.print("\n Calculating returns: ", end="")
@@ -1402,56 +1598,65 @@ class PortfolioModel:
self.historical_trade_data = trade_data
console.print("\n")
@log_start_end(log=logger)
def calculate_reserves(self):
"""_summary_"""
"""Takes dividends into account for returns calculation"""
# TODO: Add back cash dividends and deduct exchange costs
console.print("Still has to be build.")
def calculate_allocations(self):
"""Determine allocations based on assets, sectors, countries and regional."""
@log_start_end(log=logger)
def calculate_allocations(self, category: str):
"""Determine allocations based on assets, sectors, countries and region.
# Determine asset allocation
(
self.benchmark_assets_allocation,
self.portfolio_assets_allocation,
) = allocation_model.obtain_assets_allocation(
self.benchmark_info, self.portfolio_trades
)
Parameters
----------
category: str
chosen allocation category from asset, sector, country or region
# Determine sector allocation
(
self.benchmark_sectors_allocation,
self.portfolio_sectors_allocation,
) = allocation_model.obtain_sector_allocation(
self.benchmark_info, self.portfolio_trades
)
"""
# Determine regional and country allocations
(
self.benchmark_regional_allocation,
self.benchmark_country_allocation,
) = allocation_model.obtain_benchmark_regional_and_country_allocation(
self.benchmark_ticker
)
if category == "asset":
# Determine asset allocation
(
self.benchmark_assets_allocation,
self.portfolio_assets_allocation,
) = allocation_model.get_assets_allocation(
self.benchmark_info, self.portfolio_trades
)
elif category == "sector":
# Determine sector allocation
(
self.benchmark_sectors_allocation,
self.portfolio_sectors_allocation,
) = allocation_model.get_sector_allocation(
self.benchmark_info, self.portfolio_trades
)
elif category in ("country", "region"):
# Determine region and country allocations
(
self.benchmark_region_allocation,
self.benchmark_country_allocation,
) = allocation_model.get_region_country_allocation(self.benchmark_ticker)
(
self.portfolio_regional_allocation,
self.portfolio_country_allocation,
) = allocation_model.obtain_portfolio_regional_and_country_allocation(
self.portfolio_trades
)
(
self.portfolio_region_allocation,
self.portfolio_country_allocation,
) = allocation_model.get_portfolio_region_country_allocation(
self.portfolio_trades
)
@log_start_end(log=logger)
def set_risk_free_rate(self, risk_free_rate: float):
"""Sets risk free rate
Parameters
----------
risk_free (float): risk free rate in decimal format
risk_free (float): risk free rate in float format
"""
self.risk_free_rate = risk_free_rate
# Metrics
@log_start_end(log=logger)
def get_r2_score(self) -> pd.DataFrame:
"""Class method that retrieves R2 Score for portfolio and benchmark selected

80
openbb_terminal/portfolio/portfolio_view.py
View File

@@ -65,7 +65,12 @@ In order to load a CSV do the following:
@log_start_end(log=logger)
def display_orderbook(portfolio=None, show_index=False):
def display_orderbook(
portfolio=None,
show_index=False,
limit: int = 10,
export: str = "",
):
"""Display portfolio orderbook
Parameters
@@ -74,13 +79,29 @@ def display_orderbook(portfolio=None, show_index=False):
Instance of Portfolio class
show_index: bool
Defaults to False.
limit: int
Number of rows to display
export : str
Export certain type of data
"""
if portfolio.empty:
logger.warning("No orderbook loaded")
console.print("[red]No orderbook loaded.[/red]\n")
else:
print_rich_table(portfolio.get_orderbook(), show_index)
df = portfolio.get_orderbook()
print_rich_table(
df=df[:limit],
show_index=show_index,
title=f"Last {limit if limit < len(df) else len(df)} transactions",
)
export_data(
export,
os.path.dirname(os.path.abspath(__file__)),
"transactions",
df.set_index("Date"),
)
@log_start_end(log=logger)
@@ -106,54 +127,36 @@ def display_assets_allocation(
benchmark_allocation = benchmark_allocation.iloc[:limit]
portfolio_allocation = portfolio_allocation.iloc[:limit]
combined = pd.DataFrame()
for ticker, allocation in portfolio_allocation.items():
if ticker in benchmark_allocation["symbol"].values:
benchmark_allocation_value = float(
benchmark_allocation[benchmark_allocation["symbol"] == ticker][
"holdingPercent"
]
)
else:
benchmark_allocation_value = 0
combined = combined.append(
[
[
ticker,
allocation,
benchmark_allocation_value,
allocation - benchmark_allocation_value,
]
]
)
combined.columns = ["Symbol", "Portfolio", "Benchmark", "Difference"]
combined = pd.merge(
portfolio_allocation, benchmark_allocation, on="Symbol", how="left"
)
combined["Difference"] = combined["Portfolio"] - combined["Benchmark"]
combined = combined.replace(np.nan, "-")
combined = combined.replace(0, "-")
print_rich_table(
combined.replace(0, "-"),
combined,
headers=list(combined.columns),
title=f"Portfolio vs. Benchmark - Top {len(combined) if len(combined) < limit else limit} Assets Allocation",
floatfmt=[".2f", ".2%", ".2%", ".2%"],
floatfmt=["", ".2%", ".2%", ".2%"],
show_index=False,
)
if include_separate_tables:
print_rich_table(
pd.DataFrame(portfolio_allocation),
headers=list(["Allocation"]),
portfolio_allocation,
headers=list(portfolio_allocation.columns),
title=f"Portfolio - Top {len(portfolio_allocation) if len(benchmark_allocation) < limit else limit} "
f"Assets Allocation",
floatfmt=[".2%"],
show_index=True,
floatfmt=[".2%", ".2%"],
show_index=False,
)
print_rich_table(
benchmark_allocation,
headers=list(["Symbol", "Name", "Allocation"]),
headers=list(benchmark_allocation.columns),
title=f"Benchmark - Top {len(benchmark_allocation) if len(benchmark_allocation) < limit else limit} "
f"Assets Allocation",
floatfmt=[".2f", ".2f", ".2%"],
floatfmt=[".2%", ".2%"],
show_index=False,
)
@@ -181,6 +184,15 @@ def display_category_allocation(
include_separate_tables: bool
Whether to include separate asset allocation tables
"""
if benchmark_allocation.empty:
console.print(f"[red]Benchmark data for {category} is empty.\n[/red]")
return
if portfolio_allocation.empty:
console.print(f"[red]Portfolio data for {category} is empty.\n[/red]")
return
benchmark_allocation = benchmark_allocation.iloc[:limit]
portfolio_allocation = portfolio_allocation.iloc[:limit]

0
portfolio/holdings/paexport_degiro.csv vendored
View File