jerry/OpenBB
1
0
Fork 0
mirror of https://github.com/OpenBB-finance/OpenBB.git synced 2026-10-30 14:02:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactoring crypto load (#2366)

Browse Source 
* refactoring load

* updated to match minh changes

* improved loading

* updated tests

Co-authored-by: minhhoang1023 <40023817+minhhoang1023@users.noreply.github.com>
This commit is contained in:
jose-donato
2022-08-24 10:10:16 +01:00
committed by GitHub
parent 1e88223848
commit 4399d06a78
8 changed files with 321 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
i18n/en.yml vendored
View File

@@ -487,7 +487,9 @@ en:
crypto/load: load a specific cryptocurrency for analysis
crypto/find: find coins
crypto/_symbol: Coin
crypto/_interval: Interval (min)
crypto/_source: Source
crypto/_exchange: Exchange
crypto/headlines: crypto sentiment from 15+ major news headlines
crypto/candle: view a candle chart for a specific cryptocurrency
crypto/prt: potential returns tool - check how much upside if ETH reaches BTC market cap

19
openbb_terminal/cryptocurrency/crypto_controller.py
View File

@@ -84,9 +84,12 @@ class CryptoController(CryptoBaseController):
if session and obbff.USE_PROMPT_TOOLKIT:
choices: dict = {c: {} for c in self.controller_choices}
choices["load"]["-d"] = {
c: {} for c in ["1", "7", "14", "30", "90", "180", "365"]
choices["load"]["-i"] = {
c: {}
for c in ["1", "5", "15", "30", "60", "240", "1440", "10080", "43200"]
}
choices["load"]["--exchange"] = {c: {} for c in self.exchanges}
choices["load"]["--source"] = {c: {} for c in ["ccxt", "yf", "cg"]}
choices["load"]["--vs"] = {c: {} for c in ["usd", "eur"]}
choices["find"]["-k"] = {c: {} for c in FIND_KEYS}
choices["headlines"] = {c: {} for c in finbrain_crypto_view.COINS}
@@ -103,10 +106,15 @@ class CryptoController(CryptoBaseController):
mt.add_cmd("load")
mt.add_cmd("find")
mt.add_raw("\n")
mt.add_param("_symbol", self.symbol.upper())
mt.add_param(
"_source", "CoinGecko (Price), YahooFinance (Volume)" if self.symbol else ""
"_symbol", f"{self.symbol.upper()}/{self.vs.upper()}" if self.symbol else ""
)
if self.source == "ccxt":
mt.add_param(
"_exchange", self.exchange if self.symbol and self.exchange else ""
)
mt.add_param("_source", self.source if self.symbol and self.source else "")
mt.add_param("_interval", self.current_interval)
mt.add_raw("\n")
mt.add_cmd("headlines", "FinBrain")
mt.add_cmd("candle", "", self.symbol)
@@ -211,9 +219,12 @@ class CryptoController(CryptoBaseController):
return
plot_chart(
exchange=self.exchange,
source=self.source,
symbol=self.symbol,
currency=self.current_currency,
prices_df=self.current_df,
interval=self.current_interval,
)
@log_start_end(log=logger)

254
openbb_terminal/cryptocurrency/cryptocurrency_helpers.py
View File

@@ -11,6 +11,7 @@ import logging
import pandas as pd
import numpy as np
import ccxt
from binance.client import Client
import matplotlib.pyplot as plt
import yfinance as yf
@@ -46,6 +47,17 @@ logger = logging.getLogger(__name__)
INTERVALS = ["1H", "3H", "6H", "1D"]
CCXT_INTERVAL_MAP = {
"1": "1m",
"15": "15m",
"30": "30m",
"60": "1h",
"240": "4h",
"1440": "1d",
"10080": "1w",
"43200": "1M",
}
SOURCES_INTERVALS = {
"bin": [
"1day",
@@ -290,8 +302,12 @@ def get_coinpaprika_id(symbol: str):
def load(
symbol: str,
vs: str = "usd",
days: int = 365,
start_date: datetime = (datetime.now() - timedelta(days=1100)),
interval: str = "1440", # 1, 15, 30, 60, 240, 1440, 10080, 43200
exchange: str = "binance",
vs_currency: str = "usdt",
end_date: datetime = datetime.now(),
source: str = "ccxt",
):
"""Load crypto currency to perform analysis on CoinGecko is used as source for price and
YahooFinance for volume.
@@ -310,43 +326,110 @@ def load(
pd.DataFrame
Dataframe consisting of price and volume data
"""
coingecko_id = get_coingecko_id(symbol)
if not coingecko_id:
return pd.DataFrame()
df = pd.DataFrame()
if source == "ccxt":
pair = f"{symbol.upper()}/{vs_currency.upper()}"
try:
df = fetch_ccxt_ohlc(
exchange,
3,
pair,
CCXT_INTERVAL_MAP[interval],
int(datetime.timestamp(start_date)) * 1000,
1000,
)
if df.empty:
console.print(f"\nPair {pair} not found in {exchange}\n")
return pd.DataFrame()
except: # noqa: E722
console.print(f"\nPair {pair} not found on {exchange}\n")
return df
elif source == "cg":
delta = datetime.now() - start_date
days = delta.days
if days > 365:
console.print("Coingecko free tier only allows a max of 365 days\n")
days = 365
coingecko_id = get_coingecko_id(symbol)
if not coingecko_id:
console.print(f"{symbol} not found in Coingecko\n")
return df
df = pycoingecko_model.get_ohlc(coingecko_id, vs_currency, days)
df_coin = yf.download(
f"{symbol}-{vs_currency}",
end=datetime.now(),
start=start_date,
progress=False,
interval="1d",
).sort_index(ascending=False)
df = pycoingecko_model.get_ohlc(coingecko_id, vs, days)
if not df_coin.empty:
df = pd.merge(
df, df_coin[::-1][["Volume"]], left_index=True, right_index=True
)
df.index.name = "date"
start_date = datetime.now() - timedelta(days=days)
df = yf.download(
f"{symbol}-{vs}",
end=datetime.now(),
start=start_date,
progress=False,
interval="1d",
).sort_index(ascending=True)
elif source == "yf":
pair = f"{symbol}-{vs_currency}"
if int(interval) >= 1440:
YF_INTERVAL_MAP = {
"1440": "1d",
"10080": "1wk",
"43200": "1mo",
}
df = yf.download(
pair,
end=end_date,
start=start_date,
progress=False,
interval=YF_INTERVAL_MAP[interval],
).sort_index(ascending=True)
else:
s_int = str(interval) + "m"
d_granularity = {"1m": 6, "5m": 59, "15m": 59, "30m": 59, "60m": 729}
s_start_dt = datetime.utcnow() - timedelta(days=d_granularity[s_int])
s_date_start = s_start_dt.strftime("%Y-%m-%d")
df = yf.download(
pair,
start=s_date_start
if s_start_dt > start_date
else start_date.strftime("%Y-%m-%d"),
progress=False,
interval=s_int,
)
if df.empty:
return pd.DataFrame()
df.index.name = "date"
if not df.empty:
console.print(
f"\nLoading Daily {symbol.upper()} crypto "
f"with starting period {start_date.strftime('%Y-%m-%d')} for analysis.",
)
open_sum = df["Open"].sum()
if open_sum == 0:
console.print(f"\nPair {pair} has invalid data on Yahoo Finance\n")
return pd.DataFrame()
if df.empty:
console.print(f"\nPair {pair} not found in Yahoo Finance\n")
return pd.DataFrame()
df.index.name = "date"
return df
def show_quick_performance(crypto_df: pd.DataFrame, symbol: str, current_currency: str):
def show_quick_performance(
crypto_df: pd.DataFrame,
symbol: str,
current_currency: str,
source: str,
exchange: str,
interval: str,
):
"""Show quick performance stats of crypto prices. Daily prices expected"""
closes = crypto_df["Close"]
volumes = crypto_df["Volume"] if "Volume" in crypto_df else pd.DataFrame()
perfs = {
"1D": 100 * closes.pct_change(2)[-1],
"7D": 100 * closes.pct_change(7)[-1],
"1M": 100 * closes.pct_change(30)[-1],
"1Y": 100 * closes.pct_change(365)[-1],
}
perfs = {}
if interval == "1440":
perfs = {
"1D": 100 * closes.pct_change(2)[-1],
"7D": 100 * closes.pct_change(7)[-1],
"1M": 100 * closes.pct_change(30)[-1],
"1Y": 100 * closes.pct_change(365)[-1],
}
first_day_current_year = str(datetime.now().date().replace(month=1, day=1))
if first_day_current_year in closes.index:
closes_ytd = closes[closes.index > first_day_current_year]
@@ -368,32 +451,39 @@ def show_quick_performance(crypto_df: pd.DataFrame, symbol: str, current_currenc
if len(volumes) > 7:
df["Volume (7D avg)"] = lambda_long_number_format(np.mean(volumes[-9:-2]), 2)
df.insert(0, f"Price ({current_currency.upper()})", closes[-1])
df.insert(0, f"\nPrice ({current_currency.upper()})", closes[-1])
# df.insert(
# len(df.columns),
# f"Market Cap ({current_currency.upper()})",
# lambda_long_number_format(int(crypto_df["Market Cap"][-1])),
# )
coingecko_id = get_coingecko_id(symbol)
try:
coingecko_id = get_coingecko_id(symbol)
coin_data_cg = pycoingecko_model.get_coin_tokenomics(coingecko_id)
if not coin_data_cg.empty:
df.insert(
len(df.columns),
"Circulating Supply",
lambda_long_number_format(
int(
coin_data_cg.loc[coin_data_cg["Metric"] == "Circulating Supply"][
"Value"
]
)
),
)
coin_data_cg = pycoingecko_model.get_coin_tokenomics(coingecko_id)
if not coin_data_cg.empty:
df.insert(
len(df.columns),
"Circulating Supply",
lambda_long_number_format(
int(
coin_data_cg.loc[
coin_data_cg["Metric"] == "Circulating Supply"
]["Value"]
)
),
)
except: # noqa: E722
pass
console.print()
exchange_str = f"in {exchange.capitalize()}" if source == "ccxt" else ""
print_rich_table(
df, show_index=False, headers=df.columns, title=f"{symbol.upper()} Performance"
df,
show_index=False,
headers=df.columns,
title=f"{symbol.upper()}/{current_currency.upper()} Performance {exchange_str}",
)
console.print()
@@ -1177,7 +1267,14 @@ def display_all_coins(
)
def plot_chart(prices_df: pd.DataFrame, symbol: str = "", currency: str = "") -> None:
def plot_chart(
prices_df: pd.DataFrame,
symbol: str = "",
currency: str = "",
source: str = "",
exchange: str = "",
interval: str = "",
) -> None:
"""Load data for Technical Analysis
Parameters
@@ -1194,15 +1291,18 @@ def plot_chart(prices_df: pd.DataFrame, symbol: str = "", currency: str = "") ->
console.print("There is not data to plot chart\n")
return
title = f"{symbol}/{currency} from {prices_df.index[0].strftime('%Y/%m/%d')} to {prices_df.index[-1].strftime('%Y/%m/%d')}" # noqa: E501
exchange_str = f"/{exchange}" if source == "ccxt" else ""
title = f"{source}{exchange_str} - {symbol.upper()}/{currency.upper()} from {prices_df.index[0].strftime('%Y/%m/%d')} to {prices_df.index[-1].strftime('%Y/%m/%d')} - {interval}m" # noqa: E501
prices_df["Volume"] = prices_df["Volume"] / 1_000_000
volume_mean = prices_df["Volume"].mean()
if volume_mean > 1_000_000:
prices_df["Volume"] = prices_df["Volume"] / 1_000_000
plot_candles(
candles_df=prices_df,
title=title,
volume=True,
ylabel="Volume [1M]",
ylabel="Volume [1M]" if volume_mean > 1_000_000 else "Volume",
)
console.print("")
@@ -1330,3 +1430,59 @@ def check_cg_id(symbol: str):
print(f"\n{symbol} not found on CoinGecko")
return ""
return symbol
def fetch_ccxt_ohlc(exchange_id, max_retries, symbol, timeframe, since, limit):
exchange = getattr(ccxt, exchange_id)(
{
"enableRateLimit": True, # required by the Manual
}
)
if isinstance(since, str):
since = exchange.parse8601(since)
ohlcv = get_ohlcv(exchange, max_retries, symbol, timeframe, since, limit)
df = pd.DataFrame(ohlcv, columns=["date", "Open", "High", "Low", "Close", "Volume"])
df["date"] = pd.to_datetime(df.date, unit="ms")
df.set_index("date", inplace=True)
return df
def retry_fetch_ohlcv(exchange, max_retries, symbol, timeframe, since, limit):
num_retries = 0
try:
num_retries += 1
ohlcv = exchange.fetch_ohlcv(symbol, timeframe, since, limit)
return ohlcv
except Exception:
if num_retries > max_retries:
raise
return []
def get_ohlcv(exchange, max_retries, symbol, timeframe, since, limit):
timeframe_duration_in_seconds = exchange.parse_timeframe(timeframe)
timeframe_duration_in_ms = timeframe_duration_in_seconds * 1000
timedelta_ = limit * timeframe_duration_in_ms
now = exchange.milliseconds()
all_ohlcv = []
fetch_since = since
while fetch_since < now:
ohlcv = retry_fetch_ohlcv(
exchange, max_retries, symbol, timeframe, fetch_since, limit
)
fetch_since = (ohlcv[-1][0] + 1) if len(ohlcv) else (fetch_since + timedelta_)
all_ohlcv = all_ohlcv + ohlcv
return exchange.filter_by_since_limit(all_ohlcv, since, None, key=0)
def get_exchanges_ohlc():
arr = []
for exchange in ccxt.exchanges:
exchange_ccxt = getattr(ccxt, exchange)(
{
"enableRateLimit": True,
}
)
if exchange_ccxt.has["fetchOHLCV"]:
arr.append(exchange)
return arr

100
openbb_terminal/parent_classes.py
View File

@@ -1,7 +1,8 @@
"""Parent Classes"""
__docformat__ = "numpy"
# pylint: disable= C0301
# pylint: disable= C0301,C0302,R0902
from abc import ABCMeta, abstractmethod
import argparse
@@ -884,13 +885,17 @@ class CryptoBaseController(BaseController, metaclass=ABCMeta):
super().__init__(queue)
self.symbol = ""
self.vs = ""
self.current_df = pd.DataFrame()
self.current_currency = ""
self.source = ""
self.current_interval = ""
self.exchange = ""
self.price_str = ""
self.interval = ""
self.resolution = "1D"
self.TRY_RELOAD = True
self.exchanges = cryptocurrency_helpers.get_exchanges_ohlc()
def call_load(self, other_args):
"""Process load command"""
@@ -898,8 +903,11 @@ class CryptoBaseController(BaseController, metaclass=ABCMeta):
add_help=False,
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
prog="load",
description="Load crypto currency to perform analysis on."
"CoinGecko is used as source for price and YahooFinance for volume.",
description="""Load crypto currency to perform analysis on.
Yahoo Finance is used as default source.
Other sources can be used such as 'ccxt' or 'cg' with --source.
If you select 'ccxt', you can then select any exchange with --exchange.
You can also select a specific interval with --interval.""",
)
parser.add_argument(
"-c",
@@ -911,20 +919,57 @@ class CryptoBaseController(BaseController, metaclass=ABCMeta):
)
parser.add_argument(
"-d",
"--days",
default="365",
dest="days",
help="Data up to number of days ago",
choices=["1", "7", "14", "30", "90", "180", "365"],
"-s",
"--start",
type=valid_date,
default=(datetime.now() - timedelta(days=1100)).strftime("%Y-%m-%d"),
dest="start",
help="The starting date (format YYYY-MM-DD) of the crypto",
)
parser.add_argument(
"--exchange",
help="Exchange to search",
dest="exchange",
type=str,
default="binance",
choices=self.exchanges,
)
parser.add_argument(
"-e",
"--end",
type=valid_date,
default=datetime.now().strftime("%Y-%m-%d"),
dest="end",
help="The ending date (format YYYY-MM-DD) of the crypto",
)
parser.add_argument(
"--vs",
help="Quote currency (what to view coin vs)",
dest="vs",
default="usd",
"-i",
"--interval",
action="store",
dest="interval",
type=str,
choices=["usd", "eur"],
default="1440",
choices=["1", "5", "15", "30", "60", "240", "1440", "10080", "43200"],
help="The interval of the crypto",
)
parser.add_argument(
"--vs",
help="Quote currency (what to view coin vs). e.g., usdc, usdt, ... if source is ccxt, usd, eur, ... otherwise", # noqa
dest="vs",
default="usdt",
type=str,
)
parser.add_argument(
"--source",
action="store",
dest="source",
choices=["ccxt", "yf", "cg"],
default="yf",
help="Data source to select from",
)
if other_args and "-" not in other_args[0][0]:
@@ -933,20 +978,29 @@ class CryptoBaseController(BaseController, metaclass=ABCMeta):
ns_parser = parse_simple_args(parser, other_args)
if ns_parser:
if ns_parser.source in ("yf", "cg"):
if ns_parser.vs == "usdt":
ns_parser.vs = "usd"
(self.current_df) = cryptocurrency_helpers.load(
symbol=ns_parser.coin.lower(),
days=int(ns_parser.days),
vs=ns_parser.vs,
vs_currency=ns_parser.vs,
end_date=ns_parser.end,
start_date=ns_parser.start,
interval=ns_parser.interval,
source=ns_parser.source,
)
if not self.current_df.empty:
self.current_interval = "1day"
self.vs = ns_parser.vs
self.exchange = ns_parser.exchange
self.source = ns_parser.source
self.current_interval = ns_parser.interval
self.current_currency = ns_parser.vs
self.symbol = ns_parser.coin.lower()
cryptocurrency_helpers.show_quick_performance(
self.current_df, self.symbol, self.current_currency
)
else:
console.print(
f"\n[red]Couldn't find [bold]{ns_parser.coin}[/bold] in [bold]yfinance[/bold]."
f"Search for symbol (e.g., btc) and not full name (e.g., bitcoin)[/red]\n"
self.current_df,
self.symbol,
self.current_currency,
ns_parser.source,
ns_parser.exchange,
self.current_interval,
)

2
tests/openbb_terminal/cryptocurrency/due_diligence/test_pycoingecko_view.py
View File

@@ -38,7 +38,7 @@ def get_bitcoin(mock_load):
) as f:
sample_return = json.load(f)
mock_load.return_value = sample_return
df = load("btc", vs="usd")
df = load("btc", vs_currency="usd", source="yf")
return df

4
tests/openbb_terminal/cryptocurrency/test_cryptocurrency_helpers.py
View File

@@ -59,7 +59,7 @@ def test_create_closet_match_df(recorder):
],
)
def test_load_none(coin, vs):
df = load(symbol=coin, vs=vs)
df = load(symbol=coin, vs_currency=vs)
assert df is not None
@@ -77,7 +77,7 @@ def fixture_get_bitcoin(mocker):
) as f:
sample_return = json.load(f)
mock_load.return_value = sample_return
df = load("BTC", vs="usd")
df = load("BTC", vs_currency="usd", source="yf")
return df

1
tests/openbb_terminal/cryptocurrency/txt/test_crypto_controller/test_print_help.txt vendored
View File

@@ -3,6 +3,7 @@
Coin:
Source:
Interval (min):
headlines crypto sentiment from 15+ major news headlines [FinBrain]
[unvl] candle view a candle chart for a specific cryptocurrency[/unvl]

26
website/content/terminal/crypto/load/_index.md vendored
View File

@@ -1,14 +1,24 @@
```
usage: load [-c COIN] [-d {1,7,14,30,90,180,365}] [--vs {usd,eur}] [-h]
```
`usage: load [-c COIN] [-s START]
[--exchange EXCHANGE]
[-e END] [-i {1,5,15,30,60,240,1440,10080,43200}] [--vs VS] [--source {ccxt,yf,cg}] [-h]`
Load crypto currency to perform analysis on CoinGecko is used as source for price and YahooFinance for volume.
Load crypto currency to perform analysis on. Yahoo Finance is used as default source. Other sources can be used such as 'ccxt'
or 'cg' with --source. If you select 'ccxt', you can then select any exchange with --exchange. You can also select a specific
interval with --interval.
```
optional arguments:
-c COIN, --coin COIN Coin to get. Must be coin symbol (e.g., btc, eth) (default: None)
-d {1,7,14,30,90,180,365}, --days {1,7,14,30,90,180,365}
Data up to number of days ago (default: 365)
--vs {usd,eur} Quote currency (what to view coin vs) (default: usd)
-s START, --start START
The starting date (format YYYY-MM-DD) of the crypto (default: 2019-08-19)
--exchange {aax,ascendex,bequant,bibox,bigone,binance,binancecoinm,binanceus,binanceusdm,bit2c,bitbank,bitbay,bitcoincom,bitfinex,bitfinex2,bitflyer,bitforex,bitget,bithumb,bitmart,bitmex,bitopro,bitpanda,bitrue,bitso,bitstamp,bitstamp1,bittrex,bitvavo,bkex,bl3p,btcalpha,btcbox,btcex,btcmarkets,btctradeua,btcturk,buda,bw,bybit,bytetrade,cdax,cex,coinbaseprime,coinbasepro,coincheck,coinex,coinfalcon,coinmate,coinone,coinspot,crex24,cryptocom,currencycom,delta,deribit,digifinex,eqonex,exmo,flowbtc,fmfwio,ftx,ftxus,gate,gateio,gemini,hitbtc,hitbtc3,hollaex,huobi,huobijp,huobipro,idex,independentreserve,indodax,itbit,kraken,kucoin,kucoinfutures,kuna,latoken,lbank,lbank2,liquid,luno,lykke,mercado,mexc,mexc3,ndax,novadax,oceanex,okcoin,okex,okex5,okx,paymium,phemex,poloniex,probit,qtrade,ripio,stex,therock,tidebit,tidex,timex,tokocrypto,upbit,wavesexchange,whitebit,woo,xena,yobit,zaif,zb,zipmex,zonda}
Exchange to search (default: binance)
-e END, --end END The ending date (format YYYY-MM-DD) of the crypto (default: 2022-08-23)
-i {1,5,15,30,60,240,1440,10080,43200}, --interval {1,5,15,30,60,240,1440,10080,43200}
The interval of the crypto (default: 1440)
--vs VS Quote currency (what to view coin vs). e.g., usdc, usdt, ... if source is ccxt, usd, eur, ... otherwise
(default: usdt)
--source {ccxt,yf,cg}
Data source to select from (default: yf)
-h, --help show this help message (default: False)
```
```