"""BitQuery model""" __docformat__ = "numpy" import datetime import json import os from typing import Optional import numpy as np from requests.exceptions import HTTPError from requests.adapters import HTTPAdapter import requests import pandas as pd from gamestonk_terminal import config_terminal as cfg class BitQueryApiKeyException(Exception): """Bit Query Api Key Exception object""" def __init__(self, message: str): super().__init__(message) self.message = message def __str__(self) -> str: return f"BitQueryApiKeyException: {self.message}" class BitQueryTimeoutException(Exception): """BitQuery Timeout Exception class""" BQ_URL = "https://graphql.bitquery.io" CURRENCIES = ["ETH", "USD", "BTC", "USDT"] LT_FILTERS = ["exchange", "trades", "tradeAmount"] LT_KIND = ["dex", "time"] INTERVALS = ["day", "month", "week"] DVCP_FILTERS = [ "date", "exchange", "base", "quote", "open", "high", "low", "close", "tradeAmount", "trades", ] UEAT_FILTERS = [ "date", "uniqueSenders", "transactions", "averageGasPrice", "mediumGasPrice", "maximumGasPrice", ] TTCP_FILTERS = ["base", "quoted", "trades", "tradeAmount"] BAAS_FILTERS = [ "date", "baseCurrency", "quoteCurrency", "dailySpread", "averageBidPrice", "averageAskPrice", ] DECENTRALIZED_EXCHANGES = [ "1inch", "AfroDex", "AirSwap", "Amplbitcratic", "Balancer", "BestSwap", "Bitox", "CellSwap", "Cellswap", "Cofix", "Coinchangex", "Curve", "DDEX", "DUBIex", "DecentrEx", "DeversiFi", "Dodo", "ETHERCExchange", "EtherBlockchain", "EtherDelta", "Ethernext", "Ethfinex", "FEGex", "FFFSwap", "Fordex", "GUDecks", "GUDeks", "HiSwap", "IDEX", "LedgerDex", "Matcha", "Miniswap", "Mooniswap", "Oasis", "OpenRelay", "S.Finance", "SakeSwap", "SeedDex", "SingularX", "StarBitEx", "SushiSwap", "SwapX", "SwitchDex", "TacoSwap", "TokenJar", "TokenStore", "TokenTrove", "Tokenlon", "TradexOne", "Uniswap", "ZeusSwap", "dYdX", "dex.blue", ] DECENTRALIZED_EXCHANGES_MAP = {e.lower(): e for e in DECENTRALIZED_EXCHANGES} NETWORKS = ["bsc", "ethereum", "matic"] def _extract_dex_trades(data: dict) -> pd.DataFrame: """Helper method that extracts from bitquery api response data from nested dictionary: response = {'ethereum' : {'dexTrades' : }}. If 'dexTrades' is None, raises Exception. Parameters ---------- data: dict response data from bitquery api. Returns ------- pd.DataFrame normalized pandas data frame with data """ dex_trades = data["ethereum"]["dexTrades"] if not dex_trades: raise ValueError("No data was returned in request response\n") return pd.json_normalize(dex_trades) def query_graph(url: str, query: str) -> dict: """Helper methods for querying graphql api. [Source: https://bitquery.io/] Parameters ---------- url: str Endpoint url query: str Graphql query Returns ------- dict: Dictionary with response data """ session = requests.Session() session.mount("https://", HTTPAdapter(max_retries=5)) headers = {"x-api-key": cfg.API_BITQUERY_KEY} timeout = 30 try: response = session.post( url, json={"query": query}, headers=headers, timeout=timeout ) except requests.Timeout as e: raise BitQueryTimeoutException( f"BitQuery API didn't respond within {timeout} seconds.\n" ) from e if response.status_code == 500: raise HTTPError(f"Internal sever error {response.reason}") if not 200 <= response.status_code < 300: raise BitQueryApiKeyException( f"Invalid Authentication: {response.status_code}. " f"Please visit https://bitquery.io/pricing and generate you free api key\n" ) try: data = response.json() if "error" in data: raise ValueError(f"Invalid Response: {data['error']}\n") except Exception as e: raise ValueError(f"Invalid Response: {response.text}\n") from e return data["data"] def get_erc20_tokens() -> pd.DataFrame: """Helper method that loads ~1500 most traded erc20 token. [Source: json file] Returns ------- pd.DataFrame ERC20 tokens with address, symbol and name """ file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "data", "erc20_coins.json" ) with open(file_path) as f: data = json.load(f) df = pd.json_normalize(data) df.columns = ["count", "address", "symbol", "name"] df = df[~df["symbol"].isin(["", None, np.NaN])] return df[["name", "symbol", "address", "count"]] def find_token_address(token: str) -> Optional[str]: """Helper methods that search for ERC20 coin base on provided symbol or token address. If erc20 token address is provided, then function checks if it's proper address and returns it back. In other case mapping data is loaded from file (or cache), and function lookup for belonging token address. Parameters ---------- token: str ERC20 token symbol e.g. UNI, SUSHI, ETH, WBTC or token address e.g. 0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48 Returns ------- str or None ERC20 token address, or None if nothing found. """ if token.startswith("0x") and len(token) >= 38: return token if token == "ETH": return token token = "WBTC" if token == "BTC" else token tokens_map: pd.DataFrame = get_erc20_tokens() found_token = tokens_map.loc[tokens_map["symbol"] == token] if found_token.empty: return None if len(found_token) > 1: return found_token.sort_values(by="count", ascending=False).iloc[0]["address"] return found_token.iloc[0]["address"] def get_dex_trades_by_exchange( trade_amount_currency: str = "USD", limit: int = 90 ) -> pd.DataFrame: """Get trades on Decentralized Exchanges aggregated by DEX [Source: https://graphql.bitquery.io/] Parameters ---------- trade_amount_currency: str Currency of displayed trade amount. Default: USD limit: int Last n days to query data. Maximum 365 (bigger numbers can cause timeouts on server side) Returns ------- pd.DataFrame Trades on Decentralized Exchanges aggregated by DEX """ dt = (datetime.date.today() - datetime.timedelta(min(limit, 365))).strftime( "%Y-%m-%d" ) if trade_amount_currency not in CURRENCIES: trade_amount_currency = "USD" query = f""" {{ ethereum {{ dexTrades(options: {{limit: 40, desc: ["count"]}} date: {{since: "{dt}"}} ) {{ exchange {{ name }} count tradeAmount(in: {trade_amount_currency}) }} }} }} """ data = query_graph(BQ_URL, query) df = _extract_dex_trades(data) df.columns = ["trades", "tradeAmount", "exchange"] df = df[~df["exchange"].isin([None, np.NaN, ""])] return df[["exchange", "trades", "tradeAmount"]].sort_values( by="tradeAmount", ascending=True ) def get_dex_trades_monthly( trade_amount_currency: str = "USD", limit: int = 90 ) -> pd.DataFrame: """Get list of trades on Decentralized Exchanges monthly aggregated. [Source: https://graphql.bitquery.io/] Parameters ---------- trade_amount_currency: str Currency of displayed trade amount. Default: USD limit: int Last n days to query data. Maximum 365 (bigger numbers can cause timeouts on server side) Returns ------- pd.DataFrame Trades on Decentralized Exchanges monthly aggregated """ if trade_amount_currency not in CURRENCIES: trade_amount_currency = "USD" dt = (datetime.date.today() - datetime.timedelta(min(limit, 365))).strftime( "%Y-%m-%d" ) query = f""" {{ ethereum {{ dexTrades( options: {{desc: ["date.year", "date.month", "count"]}} date: {{since: "{dt}"}} ) {{ count date {{ month year }} tradeAmount(in: {trade_amount_currency}) }} }} }} """ data = query_graph(BQ_URL, query) if not data: return pd.DataFrame() df = _extract_dex_trades(data) df["date"] = df.apply( lambda x: datetime.date(int(x["date.year"]), int(x["date.month"]), 1), axis=1 ) df.rename(columns={"count": "trades"}, inplace=True) return df[["date", "trades", "tradeAmount"]] def get_daily_dex_volume_for_given_pair( limit: int = 100, token: str = "UNI", vs: str = "USDT", ) -> pd.DataFrame: """Get daily volume for given pair [Source: https://graphql.bitquery.io/] Parameters ------- limit: int Last n days to query data token: str ERC20 token symbol vs: str Quote currency. Returns ------- pd.DataFrame Daily volume for given pair """ dt = (datetime.date.today() - datetime.timedelta(min(limit, 365))).strftime( "%Y-%m-%d" ) base, quote = find_token_address(token), find_token_address(vs) if not base or not quote: raise ValueError("Provided coin or quote currency doesn't exist\n") query = f""" {{ ethereum(network: ethereum) {{ dexTrades( options: {{desc: ["timeInterval.day", "trades"]}} baseCurrency: {{is: "{base}"}} quoteCurrency: {{is: "{quote}"}} date: {{since: "{dt}" }} ) {{ timeInterval {{ day(count: 1) }} baseCurrency {{ symbol }} quoteCurrency {{ symbol }} exchange {{ fullName }} trades: count tradeAmount(in: USD) quotePrice maximum_price: quotePrice(calculate: maximum) minimum_price: quotePrice(calculate: minimum) open_price: minimum(of: block, get: quote_price) close_price: maximum(of: block, get: quote_price) }} }} }} """ data = query_graph(BQ_URL, query) if not data: return pd.DataFrame() df = _extract_dex_trades(data) df.columns = [ "trades", "tradeAmount", "price", "high", "low", "open", "close", "date", "base", "quote", "exchange", ] return df[ [ "date", "exchange", "base", "quote", "open", "high", "low", "close", "tradeAmount", "trades", ] ].sort_values(by="date", ascending=False) def get_token_volume_on_dexes( token: str = "UNI", trade_amount_currency: str = "USD", ) -> pd.DataFrame: """Get token volume on different Decentralized Exchanges. [Source: https://graphql.bitquery.io/] Parameters ---------- token: str ERC20 token symbol. trade_amount_currency: str Currency to display trade amount in. Returns ------- pd.DataFrame Token volume on Decentralized Exchanges """ if trade_amount_currency not in CURRENCIES: trade_amount_currency = "USD" token_address = find_token_address(token) if token_address is None: raise ValueError(f"Couldn't find token with symbol {token}\n") query = f""" {{ ethereum {{ dexTrades( baseCurrency: {{is:"{token_address}"}} ) {{ baseCurrency{{ symbol }} exchange {{ name fullName }} count tradeAmount(in: {trade_amount_currency}) }} }} }} """ data = query_graph(BQ_URL, query) if not data: return pd.DataFrame() df = _extract_dex_trades(data)[["exchange.fullName", "tradeAmount", "count"]] df.columns = LT_FILTERS return df[~df["exchange"].str.startswith("<")].sort_values( by="tradeAmount", ascending=False ) def get_ethereum_unique_senders(interval: str = "day", limit: int = 90) -> pd.DataFrame: """Get number of unique ethereum addresses which made a transaction in given time interval. Parameters ---------- interval: str Time interval in which count unique ethereum addresses which made transaction. day, month or week. limit: int Number of records for data query. Returns ------- pd.DataFrame Unique ethereum addresses which made a transaction """ intervals = { "day": 1, "month": 30, "week": 7, } if interval not in intervals: interval = "day" days = min(limit * intervals[interval], 90) dt = (datetime.date.today() - datetime.timedelta(days)).strftime("%Y-%m-%d") query = f""" {{ ethereum(network: ethereum) {{ transactions(options: {{desc: "date.date"}}, date: {{since: "{dt}"}}) {{ uniqueSenders: count(uniq: senders) date {{ date:startOfInterval(unit: {interval}) }} averageGasPrice: gasPrice(calculate: average) mediumGasPrice: gasPrice(calculate: median) maximumGasPrice: gasPrice(calculate: maximum) transactions: count }} }} }} """ data = query_graph(BQ_URL, query) if not data: return pd.DataFrame() df = pd.DataFrame(data["ethereum"]["transactions"]) df["date"] = df["date"].apply(lambda x: x["date"]) return df[UEAT_FILTERS] def get_most_traded_pairs( network: str = "ethereum", exchange: str = "Uniswap", limit: int = 90 ) -> pd.DataFrame: """Get most traded crypto pairs on given decentralized exchange in chosen time period. [Source: https://graphql.bitquery.io/] Parameters ---------- network: str EVM network. One from list: bsc (binance smart chain), ethereum or matic exchange: Decentralized exchange name limit: Number of days taken into calculation account. Returns ------- """ dt = (datetime.date.today() - datetime.timedelta(limit)).strftime("%Y-%m-%d") exchange = DECENTRALIZED_EXCHANGES_MAP.get(exchange, "Uniswap") query = f""" {{ ethereum(network: {network}){{ dexTrades(options: {{limit: 100, desc: "tradeAmount"}}, exchangeName: {{is: "{exchange}"}} date: {{since: "{dt}"}}) {{ buyCurrency {{ symbol }} sellCurrency{{ symbol }} trades: count tradeAmount(in: USD) }} }} }} """ data = query_graph(BQ_URL, query) if not data: return pd.DataFrame() df = _extract_dex_trades(data) df.columns = ["trades", "tradeAmount", "base", "quoted"] return df[TTCP_FILTERS] def get_spread_for_crypto_pair( token: str = "WETH", vs: str = "USDT", limit: int = 30 ) -> pd.DataFrame: """Get an average bid and ask prices, average spread for given crypto pair for chosen time period. [Source: https://graphql.bitquery.io/] Parameters ---------- limit: int Last n days to query data token: str ERC20 token symbol vs: str Quoted currency. Returns ------- pd.DataFrame Average bid and ask prices, spread for given crypto pair for chosen time period """ dt = (datetime.date.today() - datetime.timedelta(limit)).strftime("%Y-%m-%d") base, quote = find_token_address(token), find_token_address(vs) if not base or not quote: raise ValueError("Provided coin or quote currency doesn't exist\n") query = f""" {{ ethereum(network: ethereum){{ dexTrades( date: {{since:"{dt}"}} baseCurrency: {{is: "{base}"}}, quoteCurrency: {{is: "{quote}"}}) {{ date {{date}} baseCurrency {{symbol}} baseAmount quoteCurrency {{ symbol }} quoteAmount trades: count quotePrice side }} }} }} """ data = query_graph(BQ_URL, query) if not data: return pd.DataFrame() df = _extract_dex_trades(data) columns = ["quotePrice", "date.date", "baseCurrency.symbol", "quoteCurrency.symbol"] bids = df.query("side == 'SELL'")[columns] asks = df.query("side == 'BUY'")[columns] bids.columns = ["averageBidPrice", "date", "baseCurrency", "quoteCurrency"] asks.columns = ["averageAskPrice", "date", "baseCurrency", "quoteCurrency"] daily_spread = pd.merge(asks, bids, on=["date", "baseCurrency", "quoteCurrency"]) daily_spread["dailySpread"] = abs( daily_spread["averageBidPrice"] - daily_spread["averageAskPrice"] ) return daily_spread[ [ "date", "baseCurrency", "quoteCurrency", "dailySpread", "averageBidPrice", "averageAskPrice", ] ].sort_values(by="date", ascending=True) POSSIBLE_CRYPTOS = list(get_erc20_tokens()["symbol"].unique())