diff --git a/gamestonk_terminal/log/collection/log_sender.py b/gamestonk_terminal/log/collection/log_sender.py index 699b532a4d1..cb2b2a8f7e7 100644 --- a/gamestonk_terminal/log/collection/log_sender.py +++ b/gamestonk_terminal/log/collection/log_sender.py @@ -8,6 +8,11 @@ from pathlib import Path # IMPORTATION INTERNAL from gamestonk_terminal import feature_flags as gtff +from gamestonk_terminal.log.constants import ( + ARCHIVES_FOLDER_NAME, + S3_FOLDER_SUFFIX, + TMP_FOLDER_NAME, +) from gamestonk_terminal.log.collection.s3_sender import send_to_s3 from gamestonk_terminal.log.generation.settings import AppSettings @@ -53,17 +58,12 @@ class LogSender(Thread): file = item.path last = item.last - # print(f"LogSender, processing : {file}") - if gtff.LOG_COLLECTION: - archives_file = file.parent / "archives" / f"{file.stem}.log" - object_key = f"{app_name}-app/logs/{identifier}/{file.stem}.log" - tmp_file = file.parent / "tmp" / f"{file.stem}.log" - - # print(f"Sending to S3, file : {file}") - # print(f"Sending to S3, archives_file : {archives_file}") - # print(f"Sending to S3, object_key : {object_key}") - # print(f"Sending to S3, tmp_file : {tmp_file}") + archives_file = file.parent / ARCHIVES_FOLDER_NAME / f"{file.stem}.log" + object_key = ( + f"{app_name}{S3_FOLDER_SUFFIX}/logs/{identifier}/{file.stem}.log" + ) + tmp_file = file.parent / TMP_FOLDER_NAME / f"{file.stem}.log" try: send_to_s3( @@ -74,7 +74,6 @@ class LogSender(Thread): ) except Exception: pass - # print(f"Sending to S3 failed : {e}") finally: pass @@ -95,7 +94,7 @@ class LogSender(Thread): self.__queue: SimpleQueue = SimpleQueue() def send_path(self, path: Path, last: bool = False): - """What ever path we send it need to the closed !""" + """Only closed files should be sent.""" queue = self.__queue queue.put(QueueItem(path=path, last=last)) diff --git a/gamestonk_terminal/log/collection/logging_clock.py b/gamestonk_terminal/log/collection/logging_clock.py index 7368f705e7a..d9219320024 100644 --- a/gamestonk_terminal/log/collection/logging_clock.py +++ b/gamestonk_terminal/log/collection/logging_clock.py @@ -75,7 +75,7 @@ class LoggingClock(Thread): @classmethod def do_action_every_sharp( cls, - action, + action: Callable, precision: Precision = Precision.hour, ): next_hour = cls.calculate_next_sharp( @@ -100,11 +100,11 @@ class LoggingClock(Thread): # OVERRIDE def run(self): - action_func = self.__action_func + action = self.__action precision = self.__precision self.do_action_every_sharp( - action=action_func, + action=action, precision=precision, ) @@ -112,7 +112,7 @@ class LoggingClock(Thread): def __init__( self, *args, - action_func: Optional[Callable] = None, + action: Optional[Callable] = None, level: int = logging.INFO, logger: Optional[logging.Logger] = None, msg: str = "Logging Clock : %s", @@ -121,13 +121,13 @@ class LoggingClock(Thread): ): super().__init__(*args, **kwargs) - self.__action_func = action_func or self.action + self.__action = action or self.default_action self.__level = level self.__logger = logger or logging.getLogger(self.__module__) self.__msg = msg self.__precision = precision - def action(self): + def default_action(self): level = self.__level logger = self.__logger msg = self.__msg % datetime.now() diff --git a/gamestonk_terminal/log/collection/s3_sender.py b/gamestonk_terminal/log/collection/s3_sender.py index 3944de0a019..167d265e260 100644 --- a/gamestonk_terminal/log/collection/s3_sender.py +++ b/gamestonk_terminal/log/collection/s3_sender.py @@ -14,12 +14,10 @@ finally: # IMPORTATION INTERNAL from gamestonk_terminal import config_terminal as cfg +from gamestonk_terminal.log.constants import DEFAULT_API_URL, DEFAULT_BUCKET # DO NOT USE THE FILE LOGGER IN THIS MODULE -DEFAULT_BUCKET = "gst-restrictions" -DEFAULT_API_URL = "https://knaqi3sud7.execute-api.eu-west-3.amazonaws.com/log_api/logs" - def send_to_s3_directly( aws_access_key: str, diff --git a/gamestonk_terminal/log/constants.py b/gamestonk_terminal/log/constants.py new file mode 100644 index 00000000000..5a73ca28f29 --- /dev/null +++ b/gamestonk_terminal/log/constants.py @@ -0,0 +1,5 @@ +ARCHIVES_FOLDER_NAME = "archives" +DEFAULT_BUCKET = "gst-restrictions" +DEFAULT_API_URL = "https://knaqi3sud7.execute-api.eu-west-3.amazonaws.com/log_api/logs" +S3_FOLDER_SUFFIX = "-app" +TMP_FOLDER_NAME = "tmp" diff --git a/gamestonk_terminal/log/generation/directories.py b/gamestonk_terminal/log/generation/directories.py index e6aff04a197..d03b18672a2 100644 --- a/gamestonk_terminal/log/generation/directories.py +++ b/gamestonk_terminal/log/generation/directories.py @@ -33,8 +33,6 @@ def get_log_dir() -> Path: if not os.path.isdir(uuid_log_dir.absolute()): os.mkdir(uuid_log_dir.absolute()) - # print(uuid_log_dir) - return uuid_log_dir diff --git a/gamestonk_terminal/log/generation/expired_files.py b/gamestonk_terminal/log/generation/expired_files.py new file mode 100644 index 00000000000..9306694dc49 --- /dev/null +++ b/gamestonk_terminal/log/generation/expired_files.py @@ -0,0 +1,29 @@ +# IMPORTATION STANDARD +from datetime import datetime +from pathlib import Path +from typing import List + + +# IMPORTATION THIRDPARTY + +# IMPORTATION INTERNAL + + +def get_timestamp_from_x_days(x: int): + timestamp_from_x_days = datetime.now().timestamp() - x * 86400 + return timestamp_from_x_days + + +def get_expired_files(directory: Path, before_timestamp: float): + expired_files = list() + if directory.exists and directory.is_dir(): + for file in directory.iterdir(): + if file.lstat().st_mtime < before_timestamp: + expired_files.append(file) + + return expired_files + + +def remove_file_list(file_list: List[Path]): + for file in file_list: + file.unlink(missing_ok=True) diff --git a/gamestonk_terminal/log/generation/formatter_with_exceptions.py b/gamestonk_terminal/log/generation/formatter_with_exceptions.py index 05419a8c76f..046951e4750 100644 --- a/gamestonk_terminal/log/generation/formatter_with_exceptions.py +++ b/gamestonk_terminal/log/generation/formatter_with_exceptions.py @@ -52,8 +52,8 @@ class FormatterWithExceptions(logging.Formatter): for word in text.split(): if ( ip_reg.search(word) - and int(max(word.split(""))) < 256 - and int(min(word.split(""))) >= 0 + # and int(max(word.split(""))) < 256 + # and int(min(word.split(""))) >= 0 ): s_list.append("suspected_ip") elif "@" in word and "." in word: diff --git a/gamestonk_terminal/log/generation/path_tracking_file_handler.py b/gamestonk_terminal/log/generation/path_tracking_file_handler.py index e3f6de08e60..8482f19bc95 100644 --- a/gamestonk_terminal/log/generation/path_tracking_file_handler.py +++ b/gamestonk_terminal/log/generation/path_tracking_file_handler.py @@ -2,15 +2,23 @@ from copy import deepcopy from logging.handlers import TimedRotatingFileHandler from pathlib import Path +from typing import Callable # IMPORTATION THIRDPARTY # IMPORTATION INTERNAL +from gamestonk_terminal import feature_flags as gtff +from gamestonk_terminal.log.constants import ARCHIVES_FOLDER_NAME, TMP_FOLDER_NAME from gamestonk_terminal.log.collection.log_sender import LogSender from gamestonk_terminal.log.collection.logging_clock import LoggingClock, Precision -from gamestonk_terminal.log.generation.settings import Settings +from gamestonk_terminal.log.generation.directories import get_log_dir, get_log_sub_dir +from gamestonk_terminal.log.generation.expired_files import ( + get_expired_files, + get_timestamp_from_x_days, + remove_file_list, +) -# DO NOT USE THE FILE LOGGER IN THIS MODULE +from gamestonk_terminal.log.generation.settings import Settings class PathTrackingFileHandler(TimedRotatingFileHandler): @@ -34,7 +42,38 @@ class PathTrackingFileHandler(TimedRotatingFileHandler): return log_sender - def build_rolling_clock(self, frequency: str, start: bool) -> LoggingClock: + @staticmethod + def clean_expired_files(before_timestamp: float): + """Deleting old files inside : archives and tmp folders. + + Only files inside the following folders are considered : + - {LOG_FOLDER_PATH}/{ARCHIVES_FOLDER_NAME} + - {LOG_FOLDER_PATH}/{TMP_FOLDER_NAME} + + Args: + before_timestamp (float): Timestamp before which files are considered expired. + """ + + archives_directory = get_log_sub_dir(name=ARCHIVES_FOLDER_NAME) + tmp_directory = get_log_sub_dir(name=TMP_FOLDER_NAME) + expired_archives_file_list = get_expired_files( + directory=archives_directory, + before_timestamp=before_timestamp, + ) + expired_tmp_file_list = get_expired_files( + directory=tmp_directory, + before_timestamp=before_timestamp, + ) + + remove_file_list(file_list=expired_archives_file_list) + remove_file_list(file_list=expired_tmp_file_list) + + @staticmethod + def build_rolling_clock( + action: Callable, + frequency: str, + start: bool, + ) -> LoggingClock: frequency = frequency.upper() if frequency == "H": @@ -45,7 +84,7 @@ class PathTrackingFileHandler(TimedRotatingFileHandler): raise AttributeError("Unsupported `logging_clock.Precision`.") rolling_clock = LoggingClock( - action_func=self.doRollover, + action=action, daemon=True, precision=precision, ) @@ -55,6 +94,27 @@ class PathTrackingFileHandler(TimedRotatingFileHandler): return rolling_clock + def send_expired_files(self, before_timestamp: float): + """Try to send files older than before_timestamp days. + + Only files inside the following folder are considered : + - {LOG_FOLDER_PATH}/ + + Args: + before_timestamp (float): Timestamp before which files are considered expired. + """ + + log_sender = self.__log_sender + log_directory = get_log_dir() + + expired_log_file_list = get_expired_files( + directory=log_directory, + before_timestamp=before_timestamp, + ) + + for file in expired_log_file_list: + log_sender.send_path(last=False, path=file) + @property def log_sender(self) -> LogSender: return self.__log_sender @@ -79,31 +139,39 @@ class PathTrackingFileHandler(TimedRotatingFileHandler): frequency = settings.log_settings.frequency kwargs["when"] = frequency super().__init__(filename, *args, **kwargs) - self.suffix += ".log" + self.suffix += ".log" self.__settings = settings - self.__log_sender = self.build_log_sender(settings=settings, start=True) - self.__rolling_clock = self.build_rolling_clock( - frequency=frequency, start=rolling_clock - ) + + self.clean_expired_files(before_timestamp=get_timestamp_from_x_days(x=5)) + + if gtff.LOG_COLLECTION: + self.__log_sender = self.build_log_sender(settings=settings, start=True) + self.__rolling_clock = self.build_rolling_clock( + action=self.doRollover, + frequency=frequency, + start=rolling_clock, + ) + self.send_expired_files(before_timestamp=get_timestamp_from_x_days(x=3)) # OVERRIDE def doRollover(self) -> None: super().doRollover() - # print("I am rolling.") - log_sender = self.__log_sender - to_delete_path_list = self.getFilesToDelete() - for path in to_delete_path_list: - # print(path) - log_sender.send_path(path=Path(path)) + if gtff.LOG_COLLECTION: + log_sender = self.__log_sender + to_delete_path_list = self.getFilesToDelete() + for path in to_delete_path_list: + log_sender.send_path(path=Path(path)) # OVERRIDE def close(self): + """Do not use the file logger in this function.""" + super().close() - log_sender = self.__log_sender - closed_log_path = self.baseFilename - log_sender.send_path(path=Path(closed_log_path), last=True) - log_sender.join() - # print("Exiting", self.baseFilename) + if gtff.LOG_COLLECTION: + log_sender = self.__log_sender + closed_log_path = self.baseFilename + log_sender.send_path(path=Path(closed_log_path), last=True) + log_sender.join() diff --git a/gamestonk_terminal/portfolio/brokers/degiro/degiro_controller.py b/gamestonk_terminal/portfolio/brokers/degiro/degiro_controller.py index 6e7e82507e1..67772524539 100644 --- a/gamestonk_terminal/portfolio/brokers/degiro/degiro_controller.py +++ b/gamestonk_terminal/portfolio/brokers/degiro/degiro_controller.py @@ -1,10 +1,12 @@ +# IMPORTATION STANDARD import argparse import logging from typing import List +# IMPORTATION THIRDPARTY from prompt_toolkit.completion import NestedCompleter -import gamestonk_terminal.config_terminal as config +# IMPORTATION INTERNAL from gamestonk_terminal import feature_flags as gtff from gamestonk_terminal.decorators import log_start_end from gamestonk_terminal.helper_funcs import parse_known_args_and_warn @@ -205,37 +207,10 @@ class DegiroController(BaseController): add_help=False, prog="login", ) - parser.add_argument( - "-u", - "--username", - type=str, - default=config.DG_USERNAME, - help="Username in Degiro's account.", - ) - parser.add_argument( - "-p", - "--password", - type=str, - default=config.DG_PASSWORD, - help="Password in Degiro's account.", - ) - parser.add_argument( - "-o", - "--otp", - type=int, - default=None, - help="One time password (2FA).", - ) - parser.add_argument( - "-s", - "--topt-secret", - type=str, - default=None, - help="TOTP SECRET (2FA).", - ) ns_parser = parse_known_args_and_warn(parser, other_args) - self.__degiro_view.login(ns_parser=ns_parser) + if ns_parser: + self.__degiro_view.login() @log_start_end(log=logger) def call_logout(self, other_args: List[str]): @@ -248,7 +223,8 @@ class DegiroController(BaseController): ) ns_parser = parse_known_args_and_warn(parser, other_args) - self.__degiro_view.logout(ns_parser=ns_parser) + if ns_parser: + self.__degiro_view.logout() @log_start_end(log=logger) def call_lookup(self, other_args: List[str]): diff --git a/gamestonk_terminal/portfolio/brokers/degiro/degiro_model.py b/gamestonk_terminal/portfolio/brokers/degiro/degiro_model.py index 1ab23c30a0a..eb0cdbed0da 100644 --- a/gamestonk_terminal/portfolio/brokers/degiro/degiro_model.py +++ b/gamestonk_terminal/portfolio/brokers/degiro/degiro_model.py @@ -265,8 +265,7 @@ class DegiroModel: return news @log_start_end(log=logger) - def login(self, credentials: Credentials): - self.__trading_api.credentials.CopyFrom(credentials) + def login(self): self.__trading_api.connect() self.__setup_extra_credentials() diff --git a/gamestonk_terminal/portfolio/brokers/degiro/degiro_view.py b/gamestonk_terminal/portfolio/brokers/degiro/degiro_view.py index 43946f511f9..831453497a9 100644 --- a/gamestonk_terminal/portfolio/brokers/degiro/degiro_view.py +++ b/gamestonk_terminal/portfolio/brokers/degiro/degiro_view.py @@ -2,6 +2,7 @@ import logging from argparse import Namespace +# IMPORTATION THIRDPARTY import pandas as pd from degiro_connector.core.helpers import pb_handler from degiro_connector.trading.models.trading_pb2 import ( @@ -306,22 +307,14 @@ class DegiroView: @log_start_end(log=logger) @check_api_key(["DG_USERNAME", "DG_PASSWORD"]) - def login(self, ns_parser: Namespace): + def login(self): # GET ATTRIBUTES degiro_model = self.__degiro_model default_credentials = degiro_model.login_default_credentials() credentials = Credentials() credentials.CopyFrom(default_credentials) - credentials.username = ns_parser.username - credentials.password = ns_parser.password - - if ns_parser.otp is not None: - credentials.one_time_password = ns_parser.otp - if ns_parser.topt_secret is not None: - credentials.totp_secret_key = ns_parser.topt_secret - - degiro_model.login(credentials=credentials) + degiro_model.login() DegiroView.__login_display_success() @@ -331,9 +324,7 @@ class DegiroView: console.print("You are now logged in !") @log_start_end(log=logger) - def logout(self, ns_parser: Namespace): - _ = ns_parser - + def logout(self): # GET ATTRIBUTES degiro_model = self.__degiro_model diff --git a/tests/gamestonk_terminal/portfolio/brokers/degiro/test_degiro_controller.py b/tests/gamestonk_terminal/portfolio/brokers/degiro/test_degiro_controller.py index 85b66e2d2b1..133c4222e88 100644 --- a/tests/gamestonk_terminal/portfolio/brokers/degiro/test_degiro_controller.py +++ b/tests/gamestonk_terminal/portfolio/brokers/degiro/test_degiro_controller.py @@ -213,7 +213,8 @@ def test_call_func_expect_queue(expected_queue, func, queue): "call_create", [ "--action=buy", - "--symbol=MOCK_SYMBOL" "--price=1", + "--symbol=MOCK_SYMBOL", + "--price=1", "--size=2", ], "DegiroView.create", diff --git a/tests/gamestonk_terminal/portfolio/brokers/degiro/test_degiro_model.py b/tests/gamestonk_terminal/portfolio/brokers/degiro/test_degiro_model.py index f782a490e0b..1a223623703 100644 --- a/tests/gamestonk_terminal/portfolio/brokers/degiro/test_degiro_model.py +++ b/tests/gamestonk_terminal/portfolio/brokers/degiro/test_degiro_model.py @@ -255,7 +255,7 @@ def test_login(mocker): new=mock_get_client_details, ) - model.login(credentials=trading_pb2.Credentials()) + model.login() mock_connect.assert_called_once() mock_get_client_details.assert_called_once()