jerry/OpenBB
1
0
Fork 0
mirror of https://github.com/OpenBB-finance/OpenBB.git synced 2026-11-05 17:01:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Log Collection : Change endpoint (#1564)

Browse Source 
* Log Collection : change url

* Log Collection : settings update + retry policy

* Log Collection : change sending order on close
This commit is contained in:
Chavithra
2022-03-28 13:35:00 +02:00
committed by GitHub
parent 684b644ef5
commit b2ac3465e5
7 changed files with 181 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
gamestonk_terminal/config_terminal.py
View File

@@ -1,6 +1,11 @@
# IMPORTATION STANDARD
import os
from distutils.util import strtobool
# IMPORTATION THIRDPARTY
import dotenv
# IMPORTATION INTERNAL
from .helper_classes import TerminalStyle as _TerminalStyle
env_files = [f for f in os.listdir() if f.endswith(".env")]
@@ -26,10 +31,14 @@ PAPERMILL_NOTEBOOK_REPORT_PORT = (
# USE IN LOG LINES + FOR FOLDER NAME INSIDE S3 BUCKET
LOGGING_APP_NAME = os.getenv("GT_LOGGING_APP_NAME") or "gst"
# AWS KEYS
LOGGING_AWS_ACCESS_KEY_ID = os.getenv("GT_LOGGING_AWS_ACCESS_KEY_ID") or "REPLACE_ME"
LOGGING_AWS_SECRET_ACCESS_KEY = os.getenv("GT_LOGGING_AWS_SECRET_ACCESS_KEY") or "REPLACE_ME"
# D | H | M | S
LOGGING_FREQUENCY = os.getenv("GT_LOGGING_FREQUENCY") or "H"
# stdout,stderr,noop,file
LOGGING_HANDLERS = os.getenv("GT_LOGGING_HANDLERS") or "file"
LOGGING_ROLLING_CLOCK = strtobool(os.getenv("GT_LOGGING_ROLLING_CLOCK") or "True")
# CRITICAL = 50
# FATAL = CRITICAL
# ERROR = 40
@@ -144,9 +153,5 @@ API_BITQUERY_KEY = os.getenv("GT_API_BITQUERY_KEY") or "REPLACE_ME"
API_SMARTSTAKE_KEY = os.getenv("GT_API_SMARTSTAKE_KEY") or "REPLACE_ME"
API_SMARTSTAKE_TOKEN = os.getenv("GT_API_SMARTSTAKE_TOKEN") or "REPLACE_ME"
# AWS KEYS
AWS_ACCESS_KEY_ID = os.getenv("GT_AWS_ACCESS_KEY_ID") or "REPLACE_ME"
AWS_SECRET_ACCESS_KEY = os.getenv("GT_AWS_SECRET_ACCESS_KEY") or "REPLACE_ME"
# https://messari.io/
API_MESSARI_KEY = os.getenv("GT_API_MESSARI_KEY") or "REPLACE_ME"

55
gamestonk_terminal/log/collection/log_sender.py
View File

@@ -7,14 +7,14 @@ from pathlib import Path
# IMPORTATION THIRDPARTY
# IMPORTATION INTERNAL
from gamestonk_terminal import feature_flags as gtff
from gamestonk_terminal.feature_flags import LOG_COLLECTION
from gamestonk_terminal.log.constants import (
ARCHIVES_FOLDER_NAME,
S3_FOLDER_SUFFIX,
TMP_FOLDER_NAME,
)
from gamestonk_terminal.log.collection.s3_sender import send_to_s3
from gamestonk_terminal.log.generation.settings import AppSettings
from gamestonk_terminal.log.generation.settings import Settings
# DO NOT USE THE FILE LOGGER IN THIS MODULE
@@ -37,18 +37,32 @@ class QueueItem:
class LogSender(Thread):
MAX_FAILS = 3
@staticmethod
def start_required() -> bool:
"""Check if it makes sense to start a LogsSender instance ."""
return LOG_COLLECTION
@property
def settings(self) -> Settings:
return deepcopy(self.__settings)
@property
def fails(self) -> int:
return self.__fails
@property
def queue(self) -> SimpleQueue:
return self.__queue
@property
def app_settings(self) -> AppSettings:
return deepcopy(self.__app_settings)
# OVERRIDE
def run(self):
settings = self.__settings
app_settings = settings.app_settings
aws_settings = settings.aws_settings
queue = self.__queue
app_settings = self.__app_settings
app_name = app_settings.name
identifier = app_settings.identifier
@@ -58,7 +72,7 @@ class LogSender(Thread):
file = item.path
last = item.last
if gtff.LOG_COLLECTION:
if self.check_sending_conditions():
archives_file = file.parent / ARCHIVES_FOLDER_NAME / f"{file.stem}.log"
object_key = (
f"{app_name}{S3_FOLDER_SUFFIX}/logs/{identifier}/{file.stem}.log"
@@ -68,14 +82,16 @@ class LogSender(Thread):
try:
send_to_s3(
archives_file=archives_file,
aws_settings=aws_settings,
file=file,
object_key=object_key,
tmp_file=tmp_file,
copy=last,
)
except Exception:
pass
self.fails_increment()
finally:
pass
self.fails_reset()
if last:
break
@@ -83,16 +99,31 @@ class LogSender(Thread):
# OVERRIDE
def __init__(
self,
app_settings: AppSettings,
settings: Settings,
*args,
**kwargs,
):
super().__init__(*args, **kwargs)
self.__app_settings = app_settings
self.__settings = settings
self.__fails: int = 0 # type: ignore
self.__queue: SimpleQueue = SimpleQueue()
def check_sending_conditions(self):
"""Check if the condition are met to send data."""
return self.start_required() and not self.max_fails_reached()
def fails_increment(self):
self.__fails += 1
def fails_reset(self):
self.__fails = 0
def max_fails_reached(self) -> bool:
return self.__fails > self.MAX_FAILS
def send_path(self, path: Path, last: bool = False):
"""Only closed files should be sent."""

49
gamestonk_terminal/log/collection/s3_sender.py
View File

@@ -1,5 +1,6 @@
# IMPORTATION STANDARD
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict
# IMPORTATION THIRDPARTY
@@ -13,8 +14,11 @@ finally:
WITH_BOTO3 = True
# IMPORTATION INTERNAL
from gamestonk_terminal import config_terminal as cfg
from gamestonk_terminal.log.constants import DEFAULT_API_URL, DEFAULT_BUCKET
from gamestonk_terminal.log.constants import (
DEFAULT_API_URL,
DEFAULT_BUCKET,
)
from gamestonk_terminal.log.generation.settings import AWSSettings
# DO NOT USE THE FILE LOGGER IN THIS MODULE
@@ -38,7 +42,11 @@ def send_to_s3_directly(
def fetch_presigned_url(api_url: str, object_key: str) -> Dict[str, Any]:
raw_response = requests.put(url=api_url, json={"object_key": object_key})
raw_response = requests.put(
json={"object_key": object_key},
timeout=3,
url=api_url,
)
raw_response.raise_for_status()
response = raw_response.json()
@@ -57,9 +65,10 @@ def send_to_s3_using_presigned_url(
files = {"file": f}
raw_response = requests.post(
presigned_info["url"],
data=presigned_info["fields"],
files=files,
timeout=3,
url=presigned_info["url"],
)
raw_response.raise_for_status()
@@ -68,10 +77,15 @@ def send_to_s3_using_presigned_url(
pass
def send_to_s3(archives_file: Path, file: Path, object_key: str, tmp_file: Path):
def send_to_s3(
archives_file: Path,
aws_settings: AWSSettings,
file: Path,
object_key: str,
tmp_file: Path,
copy: bool = False,
):
api_url = DEFAULT_API_URL
aws_access_key_id = cfg.AWS_ACCESS_KEY_ID
aws_secret_access_key = cfg.AWS_SECRET_ACCESS_KEY
bucket = DEFAULT_BUCKET
if file.stat().st_size <= 0:
@@ -79,22 +93,29 @@ def send_to_s3(archives_file: Path, file: Path, object_key: str, tmp_file: Path)
raise AttributeError(f"File is empty : {file}")
tmp_file.parent.mkdir(exist_ok=True)
file = file.rename(tmp_file)
if aws_access_key_id != "REPLACE_ME" and aws_secret_access_key != "REPLACE_ME":
if copy:
copyfile(file, tmp_file)
else:
file.rename(tmp_file)
if (
aws_settings.aws_access_key_id != "REPLACE_ME"
and aws_settings.aws_secret_access_key != "REPLACE_ME"
):
send_to_s3_directly(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_access_key_id=aws_settings.aws_access_key_id,
aws_secret_access_key=aws_settings.aws_secret_access_key,
bucket=bucket,
file=file,
file=tmp_file,
object_key=object_key,
)
else:
send_to_s3_using_presigned_url(
api_url=api_url,
file=file,
file=tmp_file,
object_key=object_key,
)
archives_file.parent.mkdir(exist_ok=True)
file.rename(archives_file)
tmp_file.rename(archives_file)

4
gamestonk_terminal/log/constants.py
View File

@@ -1,5 +1,5 @@
ARCHIVES_FOLDER_NAME = "archives"
DEFAULT_BUCKET = "gst-restrictions"
DEFAULT_API_URL = "https://knaqi3sud7.execute-api.eu-west-3.amazonaws.com/log_api/logs"
DEFAULT_BUCKET = "gst-logging"
DEFAULT_API_URL = "https://gst.openbb.app/logging"
S3_FOLDER_SUFFIX = "-app"
TMP_FOLDER_NAME = "tmp"

48
gamestonk_terminal/log/generation/path_tracking_file_handler.py
View File

@@ -7,7 +7,6 @@ from typing import Callable
# IMPORTATION THIRDPARTY
# IMPORTATION INTERNAL
from gamestonk_terminal import feature_flags as gtff
from gamestonk_terminal.log.constants import ARCHIVES_FOLDER_NAME, TMP_FOLDER_NAME
from gamestonk_terminal.log.collection.log_sender import LogSender
from gamestonk_terminal.log.collection.logging_clock import LoggingClock, Precision
@@ -35,7 +34,7 @@ class PathTrackingFileHandler(TimedRotatingFileHandler):
@staticmethod
def build_log_sender(settings: Settings, start: bool) -> LogSender:
log_sender = LogSender(app_settings=settings.app_settings, daemon=True)
log_sender = LogSender(settings=settings, daemon=True)
if start:
log_sender.start()
@@ -132,34 +131,43 @@ class PathTrackingFileHandler(TimedRotatingFileHandler):
self,
settings: Settings,
*args,
rolling_clock: bool = False,
**kwargs,
) -> None:
# SETUP PARENT CLASS
filename = str(self.build_log_file_path(settings=settings))
frequency = settings.log_settings.frequency
kwargs["when"] = frequency
super().__init__(filename, *args, **kwargs)
self.suffix += ".log"
# SETUP CURRENT CLASS
self.__settings = settings
self.clean_expired_files(before_timestamp=get_timestamp_from_x_days(x=5))
if gtff.LOG_COLLECTION:
self.__log_sender = self.build_log_sender(settings=settings, start=True)
self.__rolling_clock = self.build_rolling_clock(
action=self.doRollover,
frequency=frequency,
start=rolling_clock,
)
self.send_expired_files(before_timestamp=get_timestamp_from_x_days(x=3))
self.__log_sender = self.build_log_sender(
settings=settings,
start=False,
)
self.__rolling_clock = self.build_rolling_clock(
action=self.doRollover,
frequency=frequency,
start=settings.log_settings.rolling_clock,
)
if LogSender.start_required():
self.__log_sender.start()
self.send_expired_files(before_timestamp=get_timestamp_from_x_days(x=1))
# OVERRIDE
def doRollover(self) -> None:
log_sender = self.__log_sender
super().doRollover()
if gtff.LOG_COLLECTION:
log_sender = self.__log_sender
if log_sender.check_sending_conditions():
to_delete_path_list = self.getFilesToDelete()
for path in to_delete_path_list:
log_sender.send_path(path=Path(path))
@@ -168,13 +176,17 @@ class PathTrackingFileHandler(TimedRotatingFileHandler):
def close(self):
"""Do not use the file logger in this function."""
super().close()
log_sender = self.__log_sender
closed_log_path = Path(self.baseFilename)
if gtff.LOG_COLLECTION:
log_sender = self.__log_sender
closed_log_path = self.baseFilename
log_sender.send_path(path=Path(closed_log_path), last=True)
if log_sender.check_sending_conditions():
log_sender.send_path(path=closed_log_path, last=True)
try:
log_sender.join(timeout=3)
except Exception:
pass
super().close()
if log_sender.check_sending_conditions():
closed_log_path.unlink(missing_ok=True)

32
gamestonk_terminal/log/generation/settings.py
View File

@@ -7,6 +7,24 @@ from pathlib import Path
# IMPORTATION INTERNAL
class AWSSettings:
@property
def aws_access_key_id(self) -> str:
return self.__aws_access_key_id
@property
def aws_secret_access_key(self) -> str:
return self.__aws_secret_access_key
def __init__(
self,
aws_access_key_id: str,
aws_secret_access_key: str,
):
self.__aws_access_key_id = aws_access_key_id
self.__aws_secret_access_key = aws_secret_access_key
class AppSettings:
@property
def name(self) -> str:
@@ -58,6 +76,10 @@ class LogSettings:
def handler_list(self) -> str:
return self.__handler_list
@property
def rolling_clock(self) -> bool:
return self.__rolling_clock
@property
def verbosity(self) -> int:
return self.__verbosity
@@ -67,6 +89,7 @@ class LogSettings:
directory: Path,
frequency: str,
handler_list: str,
rolling_clock: bool,
verbosity: int,
):
"""
@@ -74,12 +97,14 @@ class LogSettings:
directory (Path): Directory used to store log files.
frequency (str): Frequency of the log files rotation.
handler_list (str) : Comma separated list of handlers : stdout,stderr,noop,file.
rolling_clock (bool): Whether or not to start a Thread to rotate logs even when inactive.
verbosity (str): Verbosity level as defined in Python `logging` module.
"""
self.__directory = directory
self.__frequency = frequency
self.__handler_list = handler_list
self.__rolling_clock = rolling_clock
self.__verbosity = verbosity
@@ -92,9 +117,14 @@ class Settings:
def log_settings(self) -> LogSettings:
return deepcopy(self.__log_settings)
@property
def aws_settings(self) -> AWSSettings:
return deepcopy(self.__aws_settings)
def __init__(
self,
app_settings: AppSettings,
aws_settings: AWSSettings,
log_settings: LogSettings,
):
"""This model regroups all configurations used by these classes instance :
@@ -105,7 +135,9 @@ class Settings:
Args:
app_name (str): Instance of AppSettings.
log_settings (str): Instance of LogSettings.
aws_settings (str): Instance of AWSSettings.
"""
self.__app_settings = app_settings
self.__log_settings = log_settings
self.__aws_settings = aws_settings

39
gamestonk_terminal/loggers.py
View File

@@ -13,9 +13,18 @@ from typing import Optional
import git
# IMPORTATION INTERNAL
import gamestonk_terminal.config_terminal as cfg
from gamestonk_terminal.config_terminal import (
LOGGING_APP_NAME,
LOGGING_AWS_ACCESS_KEY_ID,
LOGGING_AWS_SECRET_ACCESS_KEY,
LOGGING_FREQUENCY,
LOGGING_HANDLERS,
LOGGING_ROLLING_CLOCK,
LOGGING_VERBOSITY,
)
from gamestonk_terminal.log.generation.settings import (
AppSettings,
AWSSettings,
LogSettings,
Settings,
)
@@ -27,8 +36,8 @@ from gamestonk_terminal.log.generation.formatter_with_exceptions import (
)
from gamestonk_terminal.log.generation.directories import get_log_dir
logging.getLogger("requests").setLevel(cfg.LOGGING_VERBOSITY)
logging.getLogger("urllib3").setLevel(cfg.LOGGING_VERBOSITY)
logging.getLogger("requests").setLevel(LOGGING_VERBOSITY)
logging.getLogger("urllib3").setLevel(LOGGING_VERBOSITY)
logger = logging.getLogger(__name__)
@@ -86,7 +95,7 @@ def add_noop_handler(settings: Settings):
def add_file_handler(settings: Settings):
app_settings = settings.app_settings
handler = PathTrackingFileHandler(settings=settings, rolling_clock=False)
handler = PathTrackingFileHandler(settings=settings)
formatter = FormatterWithExceptions(app_settings=app_settings)
handler.setFormatter(formatter)
logging.getLogger().addHandler(handler)
@@ -134,27 +143,39 @@ def setup_logging(
) -> None:
"""Setup Logging"""
name = app_name or cfg.LOGGING_APP_NAME
# AppSettings
commit_hash = get_commit_hash()
name = app_name or LOGGING_APP_NAME
identifier = get_app_id()
session_id = session_id or str(START_TIMESTAMP)
frequency = frequency or cfg.LOGGING_FREQUENCY
verbosity = verbosity or cfg.LOGGING_VERBOSITY
# AWSSettings
aws_access_key_id = LOGGING_AWS_ACCESS_KEY_ID
aws_secret_access_key = LOGGING_AWS_SECRET_ACCESS_KEY
# LogSettings
directory = get_log_dir()
handler_list = cfg.LOGGING_HANDLERS
frequency = frequency or LOGGING_FREQUENCY
handler_list = LOGGING_HANDLERS
rolling_clock = LOGGING_ROLLING_CLOCK
verbosity = verbosity or LOGGING_VERBOSITY
settings = Settings(
app_settings=AppSettings(
name=name,
commit_hash=commit_hash,
name=name,
identifier=identifier,
session_id=session_id,
),
aws_settings=AWSSettings(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
),
log_settings=LogSettings(
directory=directory,
frequency=frequency,
handler_list=handler_list,
rolling_clock=rolling_clock,
verbosity=verbosity,
),
)