diff --git a/gamestonk_terminal/config_terminal.py b/gamestonk_terminal/config_terminal.py index f0a4f94746d..bc01d732e5e 100644 --- a/gamestonk_terminal/config_terminal.py +++ b/gamestonk_terminal/config_terminal.py @@ -1,6 +1,11 @@ +# IMPORTATION STANDARD import os +from distutils.util import strtobool + +# IMPORTATION THIRDPARTY import dotenv +# IMPORTATION INTERNAL from .helper_classes import TerminalStyle as _TerminalStyle env_files = [f for f in os.listdir() if f.endswith(".env")] @@ -26,10 +31,14 @@ PAPERMILL_NOTEBOOK_REPORT_PORT = ( # USE IN LOG LINES + FOR FOLDER NAME INSIDE S3 BUCKET LOGGING_APP_NAME = os.getenv("GT_LOGGING_APP_NAME") or "gst" +# AWS KEYS +LOGGING_AWS_ACCESS_KEY_ID = os.getenv("GT_LOGGING_AWS_ACCESS_KEY_ID") or "REPLACE_ME" +LOGGING_AWS_SECRET_ACCESS_KEY = os.getenv("GT_LOGGING_AWS_SECRET_ACCESS_KEY") or "REPLACE_ME" # D | H | M | S LOGGING_FREQUENCY = os.getenv("GT_LOGGING_FREQUENCY") or "H" # stdout,stderr,noop,file LOGGING_HANDLERS = os.getenv("GT_LOGGING_HANDLERS") or "file" +LOGGING_ROLLING_CLOCK = strtobool(os.getenv("GT_LOGGING_ROLLING_CLOCK") or "True") # CRITICAL = 50 # FATAL = CRITICAL # ERROR = 40 @@ -144,9 +153,5 @@ API_BITQUERY_KEY = os.getenv("GT_API_BITQUERY_KEY") or "REPLACE_ME" API_SMARTSTAKE_KEY = os.getenv("GT_API_SMARTSTAKE_KEY") or "REPLACE_ME" API_SMARTSTAKE_TOKEN = os.getenv("GT_API_SMARTSTAKE_TOKEN") or "REPLACE_ME" -# AWS KEYS -AWS_ACCESS_KEY_ID = os.getenv("GT_AWS_ACCESS_KEY_ID") or "REPLACE_ME" -AWS_SECRET_ACCESS_KEY = os.getenv("GT_AWS_SECRET_ACCESS_KEY") or "REPLACE_ME" - # https://messari.io/ API_MESSARI_KEY = os.getenv("GT_API_MESSARI_KEY") or "REPLACE_ME" diff --git a/gamestonk_terminal/log/collection/log_sender.py b/gamestonk_terminal/log/collection/log_sender.py index cb2b2a8f7e7..e48db2e65ad 100644 --- a/gamestonk_terminal/log/collection/log_sender.py +++ b/gamestonk_terminal/log/collection/log_sender.py @@ -7,14 +7,14 @@ from pathlib import Path # IMPORTATION THIRDPARTY # IMPORTATION INTERNAL -from gamestonk_terminal import feature_flags as gtff +from gamestonk_terminal.feature_flags import LOG_COLLECTION from gamestonk_terminal.log.constants import ( ARCHIVES_FOLDER_NAME, S3_FOLDER_SUFFIX, TMP_FOLDER_NAME, ) from gamestonk_terminal.log.collection.s3_sender import send_to_s3 -from gamestonk_terminal.log.generation.settings import AppSettings +from gamestonk_terminal.log.generation.settings import Settings # DO NOT USE THE FILE LOGGER IN THIS MODULE @@ -37,18 +37,32 @@ class QueueItem: class LogSender(Thread): + MAX_FAILS = 3 + + @staticmethod + def start_required() -> bool: + """Check if it makes sense to start a LogsSender instance .""" + + return LOG_COLLECTION + + @property + def settings(self) -> Settings: + return deepcopy(self.__settings) + + @property + def fails(self) -> int: + return self.__fails + @property def queue(self) -> SimpleQueue: return self.__queue - @property - def app_settings(self) -> AppSettings: - return deepcopy(self.__app_settings) - # OVERRIDE def run(self): + settings = self.__settings + app_settings = settings.app_settings + aws_settings = settings.aws_settings queue = self.__queue - app_settings = self.__app_settings app_name = app_settings.name identifier = app_settings.identifier @@ -58,7 +72,7 @@ class LogSender(Thread): file = item.path last = item.last - if gtff.LOG_COLLECTION: + if self.check_sending_conditions(): archives_file = file.parent / ARCHIVES_FOLDER_NAME / f"{file.stem}.log" object_key = ( f"{app_name}{S3_FOLDER_SUFFIX}/logs/{identifier}/{file.stem}.log" @@ -68,14 +82,16 @@ class LogSender(Thread): try: send_to_s3( archives_file=archives_file, + aws_settings=aws_settings, file=file, object_key=object_key, tmp_file=tmp_file, + copy=last, ) except Exception: - pass + self.fails_increment() finally: - pass + self.fails_reset() if last: break @@ -83,16 +99,31 @@ class LogSender(Thread): # OVERRIDE def __init__( self, - app_settings: AppSettings, + settings: Settings, *args, **kwargs, ): super().__init__(*args, **kwargs) - self.__app_settings = app_settings + self.__settings = settings + self.__fails: int = 0 # type: ignore self.__queue: SimpleQueue = SimpleQueue() + def check_sending_conditions(self): + """Check if the condition are met to send data.""" + + return self.start_required() and not self.max_fails_reached() + + def fails_increment(self): + self.__fails += 1 + + def fails_reset(self): + self.__fails = 0 + + def max_fails_reached(self) -> bool: + return self.__fails > self.MAX_FAILS + def send_path(self, path: Path, last: bool = False): """Only closed files should be sent.""" diff --git a/gamestonk_terminal/log/collection/s3_sender.py b/gamestonk_terminal/log/collection/s3_sender.py index c40b4f96455..b75be190eaa 100644 --- a/gamestonk_terminal/log/collection/s3_sender.py +++ b/gamestonk_terminal/log/collection/s3_sender.py @@ -1,5 +1,6 @@ # IMPORTATION STANDARD from pathlib import Path +from shutil import copyfile from typing import Any, Dict # IMPORTATION THIRDPARTY @@ -13,8 +14,11 @@ finally: WITH_BOTO3 = True # IMPORTATION INTERNAL -from gamestonk_terminal import config_terminal as cfg -from gamestonk_terminal.log.constants import DEFAULT_API_URL, DEFAULT_BUCKET +from gamestonk_terminal.log.constants import ( + DEFAULT_API_URL, + DEFAULT_BUCKET, +) +from gamestonk_terminal.log.generation.settings import AWSSettings # DO NOT USE THE FILE LOGGER IN THIS MODULE @@ -38,7 +42,11 @@ def send_to_s3_directly( def fetch_presigned_url(api_url: str, object_key: str) -> Dict[str, Any]: - raw_response = requests.put(url=api_url, json={"object_key": object_key}) + raw_response = requests.put( + json={"object_key": object_key}, + timeout=3, + url=api_url, + ) raw_response.raise_for_status() response = raw_response.json() @@ -57,9 +65,10 @@ def send_to_s3_using_presigned_url( files = {"file": f} raw_response = requests.post( - presigned_info["url"], data=presigned_info["fields"], files=files, + timeout=3, + url=presigned_info["url"], ) raw_response.raise_for_status() @@ -68,10 +77,15 @@ def send_to_s3_using_presigned_url( pass -def send_to_s3(archives_file: Path, file: Path, object_key: str, tmp_file: Path): +def send_to_s3( + archives_file: Path, + aws_settings: AWSSettings, + file: Path, + object_key: str, + tmp_file: Path, + copy: bool = False, +): api_url = DEFAULT_API_URL - aws_access_key_id = cfg.AWS_ACCESS_KEY_ID - aws_secret_access_key = cfg.AWS_SECRET_ACCESS_KEY bucket = DEFAULT_BUCKET if file.stat().st_size <= 0: @@ -79,22 +93,29 @@ def send_to_s3(archives_file: Path, file: Path, object_key: str, tmp_file: Path) raise AttributeError(f"File is empty : {file}") tmp_file.parent.mkdir(exist_ok=True) - file = file.rename(tmp_file) - if aws_access_key_id != "REPLACE_ME" and aws_secret_access_key != "REPLACE_ME": + if copy: + copyfile(file, tmp_file) + else: + file.rename(tmp_file) + + if ( + aws_settings.aws_access_key_id != "REPLACE_ME" + and aws_settings.aws_secret_access_key != "REPLACE_ME" + ): send_to_s3_directly( - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, + aws_access_key_id=aws_settings.aws_access_key_id, + aws_secret_access_key=aws_settings.aws_secret_access_key, bucket=bucket, - file=file, + file=tmp_file, object_key=object_key, ) else: send_to_s3_using_presigned_url( api_url=api_url, - file=file, + file=tmp_file, object_key=object_key, ) archives_file.parent.mkdir(exist_ok=True) - file.rename(archives_file) + tmp_file.rename(archives_file) diff --git a/gamestonk_terminal/log/constants.py b/gamestonk_terminal/log/constants.py index 5a73ca28f29..1ec5ea31013 100644 --- a/gamestonk_terminal/log/constants.py +++ b/gamestonk_terminal/log/constants.py @@ -1,5 +1,5 @@ ARCHIVES_FOLDER_NAME = "archives" -DEFAULT_BUCKET = "gst-restrictions" -DEFAULT_API_URL = "https://knaqi3sud7.execute-api.eu-west-3.amazonaws.com/log_api/logs" +DEFAULT_BUCKET = "gst-logging" +DEFAULT_API_URL = "https://gst.openbb.app/logging" S3_FOLDER_SUFFIX = "-app" TMP_FOLDER_NAME = "tmp" diff --git a/gamestonk_terminal/log/generation/path_tracking_file_handler.py b/gamestonk_terminal/log/generation/path_tracking_file_handler.py index ca9675cfd5f..12f8e238101 100644 --- a/gamestonk_terminal/log/generation/path_tracking_file_handler.py +++ b/gamestonk_terminal/log/generation/path_tracking_file_handler.py @@ -7,7 +7,6 @@ from typing import Callable # IMPORTATION THIRDPARTY # IMPORTATION INTERNAL -from gamestonk_terminal import feature_flags as gtff from gamestonk_terminal.log.constants import ARCHIVES_FOLDER_NAME, TMP_FOLDER_NAME from gamestonk_terminal.log.collection.log_sender import LogSender from gamestonk_terminal.log.collection.logging_clock import LoggingClock, Precision @@ -35,7 +34,7 @@ class PathTrackingFileHandler(TimedRotatingFileHandler): @staticmethod def build_log_sender(settings: Settings, start: bool) -> LogSender: - log_sender = LogSender(app_settings=settings.app_settings, daemon=True) + log_sender = LogSender(settings=settings, daemon=True) if start: log_sender.start() @@ -132,34 +131,43 @@ class PathTrackingFileHandler(TimedRotatingFileHandler): self, settings: Settings, *args, - rolling_clock: bool = False, **kwargs, ) -> None: + # SETUP PARENT CLASS filename = str(self.build_log_file_path(settings=settings)) frequency = settings.log_settings.frequency kwargs["when"] = frequency + super().__init__(filename, *args, **kwargs) self.suffix += ".log" + + # SETUP CURRENT CLASS self.__settings = settings self.clean_expired_files(before_timestamp=get_timestamp_from_x_days(x=5)) - if gtff.LOG_COLLECTION: - self.__log_sender = self.build_log_sender(settings=settings, start=True) - self.__rolling_clock = self.build_rolling_clock( - action=self.doRollover, - frequency=frequency, - start=rolling_clock, - ) - self.send_expired_files(before_timestamp=get_timestamp_from_x_days(x=3)) + self.__log_sender = self.build_log_sender( + settings=settings, + start=False, + ) + self.__rolling_clock = self.build_rolling_clock( + action=self.doRollover, + frequency=frequency, + start=settings.log_settings.rolling_clock, + ) + + if LogSender.start_required(): + self.__log_sender.start() + self.send_expired_files(before_timestamp=get_timestamp_from_x_days(x=1)) # OVERRIDE def doRollover(self) -> None: + log_sender = self.__log_sender + super().doRollover() - if gtff.LOG_COLLECTION: - log_sender = self.__log_sender + if log_sender.check_sending_conditions(): to_delete_path_list = self.getFilesToDelete() for path in to_delete_path_list: log_sender.send_path(path=Path(path)) @@ -168,13 +176,17 @@ class PathTrackingFileHandler(TimedRotatingFileHandler): def close(self): """Do not use the file logger in this function.""" - super().close() + log_sender = self.__log_sender + closed_log_path = Path(self.baseFilename) - if gtff.LOG_COLLECTION: - log_sender = self.__log_sender - closed_log_path = self.baseFilename - log_sender.send_path(path=Path(closed_log_path), last=True) + if log_sender.check_sending_conditions(): + log_sender.send_path(path=closed_log_path, last=True) try: log_sender.join(timeout=3) except Exception: pass + + super().close() + + if log_sender.check_sending_conditions(): + closed_log_path.unlink(missing_ok=True) diff --git a/gamestonk_terminal/log/generation/settings.py b/gamestonk_terminal/log/generation/settings.py index 0185202afc5..910dedd6a85 100644 --- a/gamestonk_terminal/log/generation/settings.py +++ b/gamestonk_terminal/log/generation/settings.py @@ -7,6 +7,24 @@ from pathlib import Path # IMPORTATION INTERNAL +class AWSSettings: + @property + def aws_access_key_id(self) -> str: + return self.__aws_access_key_id + + @property + def aws_secret_access_key(self) -> str: + return self.__aws_secret_access_key + + def __init__( + self, + aws_access_key_id: str, + aws_secret_access_key: str, + ): + self.__aws_access_key_id = aws_access_key_id + self.__aws_secret_access_key = aws_secret_access_key + + class AppSettings: @property def name(self) -> str: @@ -58,6 +76,10 @@ class LogSettings: def handler_list(self) -> str: return self.__handler_list + @property + def rolling_clock(self) -> bool: + return self.__rolling_clock + @property def verbosity(self) -> int: return self.__verbosity @@ -67,6 +89,7 @@ class LogSettings: directory: Path, frequency: str, handler_list: str, + rolling_clock: bool, verbosity: int, ): """ @@ -74,12 +97,14 @@ class LogSettings: directory (Path): Directory used to store log files. frequency (str): Frequency of the log files rotation. handler_list (str) : Comma separated list of handlers : stdout,stderr,noop,file. + rolling_clock (bool): Whether or not to start a Thread to rotate logs even when inactive. verbosity (str): Verbosity level as defined in Python `logging` module. """ self.__directory = directory self.__frequency = frequency self.__handler_list = handler_list + self.__rolling_clock = rolling_clock self.__verbosity = verbosity @@ -92,9 +117,14 @@ class Settings: def log_settings(self) -> LogSettings: return deepcopy(self.__log_settings) + @property + def aws_settings(self) -> AWSSettings: + return deepcopy(self.__aws_settings) + def __init__( self, app_settings: AppSettings, + aws_settings: AWSSettings, log_settings: LogSettings, ): """This model regroups all configurations used by these classes instance : @@ -105,7 +135,9 @@ class Settings: Args: app_name (str): Instance of AppSettings. log_settings (str): Instance of LogSettings. + aws_settings (str): Instance of AWSSettings. """ self.__app_settings = app_settings self.__log_settings = log_settings + self.__aws_settings = aws_settings diff --git a/gamestonk_terminal/loggers.py b/gamestonk_terminal/loggers.py index c05503e0130..4e0e3182df8 100644 --- a/gamestonk_terminal/loggers.py +++ b/gamestonk_terminal/loggers.py @@ -13,9 +13,18 @@ from typing import Optional import git # IMPORTATION INTERNAL -import gamestonk_terminal.config_terminal as cfg +from gamestonk_terminal.config_terminal import ( + LOGGING_APP_NAME, + LOGGING_AWS_ACCESS_KEY_ID, + LOGGING_AWS_SECRET_ACCESS_KEY, + LOGGING_FREQUENCY, + LOGGING_HANDLERS, + LOGGING_ROLLING_CLOCK, + LOGGING_VERBOSITY, +) from gamestonk_terminal.log.generation.settings import ( AppSettings, + AWSSettings, LogSettings, Settings, ) @@ -27,8 +36,8 @@ from gamestonk_terminal.log.generation.formatter_with_exceptions import ( ) from gamestonk_terminal.log.generation.directories import get_log_dir -logging.getLogger("requests").setLevel(cfg.LOGGING_VERBOSITY) -logging.getLogger("urllib3").setLevel(cfg.LOGGING_VERBOSITY) +logging.getLogger("requests").setLevel(LOGGING_VERBOSITY) +logging.getLogger("urllib3").setLevel(LOGGING_VERBOSITY) logger = logging.getLogger(__name__) @@ -86,7 +95,7 @@ def add_noop_handler(settings: Settings): def add_file_handler(settings: Settings): app_settings = settings.app_settings - handler = PathTrackingFileHandler(settings=settings, rolling_clock=False) + handler = PathTrackingFileHandler(settings=settings) formatter = FormatterWithExceptions(app_settings=app_settings) handler.setFormatter(formatter) logging.getLogger().addHandler(handler) @@ -134,27 +143,39 @@ def setup_logging( ) -> None: """Setup Logging""" - name = app_name or cfg.LOGGING_APP_NAME + # AppSettings commit_hash = get_commit_hash() + name = app_name or LOGGING_APP_NAME identifier = get_app_id() session_id = session_id or str(START_TIMESTAMP) - frequency = frequency or cfg.LOGGING_FREQUENCY - verbosity = verbosity or cfg.LOGGING_VERBOSITY + # AWSSettings + aws_access_key_id = LOGGING_AWS_ACCESS_KEY_ID + aws_secret_access_key = LOGGING_AWS_SECRET_ACCESS_KEY + + # LogSettings directory = get_log_dir() - handler_list = cfg.LOGGING_HANDLERS + frequency = frequency or LOGGING_FREQUENCY + handler_list = LOGGING_HANDLERS + rolling_clock = LOGGING_ROLLING_CLOCK + verbosity = verbosity or LOGGING_VERBOSITY settings = Settings( app_settings=AppSettings( - name=name, commit_hash=commit_hash, + name=name, identifier=identifier, session_id=session_id, ), + aws_settings=AWSSettings( + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + ), log_settings=LogSettings( directory=directory, frequency=frequency, handler_list=handler_list, + rolling_clock=rolling_clock, verbosity=verbosity, ), )