jerry/OpenBB
1
0
Fork 0
mirror of https://github.com/OpenBB-finance/OpenBB.git synced 2026-11-03 16:02:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Logging : Cleaning feature (#1549)

Browse Source 
* Logging : add log cleaning feature

* Logging : typing

* Logging : refactor

* Logging : comment

* Logging : comment

* Logging : ignoring troublesome lines

* Degiro : remove PII
This commit is contained in:
Chavithra
2022-03-25 02:52:20 +01:00
committed by GitHub
parent 93c6e63533
commit 1ae054b2e7
13 changed files with 157 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
gamestonk_terminal/log/collection/log_sender.py
View File

@@ -8,6 +8,11 @@ from pathlib import Path
# IMPORTATION INTERNAL
from gamestonk_terminal import feature_flags as gtff
from gamestonk_terminal.log.constants import (
ARCHIVES_FOLDER_NAME,
S3_FOLDER_SUFFIX,
TMP_FOLDER_NAME,
)
from gamestonk_terminal.log.collection.s3_sender import send_to_s3
from gamestonk_terminal.log.generation.settings import AppSettings
@@ -53,17 +58,12 @@ class LogSender(Thread):
file = item.path
last = item.last
# print(f"LogSender, processing : {file}")
if gtff.LOG_COLLECTION:
archives_file = file.parent / "archives" / f"{file.stem}.log"
object_key = f"{app_name}-app/logs/{identifier}/{file.stem}.log"
tmp_file = file.parent / "tmp" / f"{file.stem}.log"
# print(f"Sending to S3, file : {file}")
# print(f"Sending to S3, archives_file : {archives_file}")
# print(f"Sending to S3, object_key : {object_key}")
# print(f"Sending to S3, tmp_file : {tmp_file}")
archives_file = file.parent / ARCHIVES_FOLDER_NAME / f"{file.stem}.log"
object_key = (
f"{app_name}{S3_FOLDER_SUFFIX}/logs/{identifier}/{file.stem}.log"
)
tmp_file = file.parent / TMP_FOLDER_NAME / f"{file.stem}.log"
try:
send_to_s3(
@@ -74,7 +74,6 @@ class LogSender(Thread):
)
except Exception:
pass
# print(f"Sending to S3 failed : {e}")
finally:
pass
@@ -95,7 +94,7 @@ class LogSender(Thread):
self.__queue: SimpleQueue = SimpleQueue()
def send_path(self, path: Path, last: bool = False):
"""What ever path we send it need to the closed !"""
"""Only closed files should be sent."""
queue = self.__queue
queue.put(QueueItem(path=path, last=last))

12
gamestonk_terminal/log/collection/logging_clock.py
View File

@@ -75,7 +75,7 @@ class LoggingClock(Thread):
@classmethod
def do_action_every_sharp(
cls,
action,
action: Callable,
precision: Precision = Precision.hour,
):
next_hour = cls.calculate_next_sharp(
@@ -100,11 +100,11 @@ class LoggingClock(Thread):
# OVERRIDE
def run(self):
action_func = self.__action_func
action = self.__action
precision = self.__precision
self.do_action_every_sharp(
action=action_func,
action=action,
precision=precision,
)
@@ -112,7 +112,7 @@ class LoggingClock(Thread):
def __init__(
self,
*args,
action_func: Optional[Callable] = None,
action: Optional[Callable] = None,
level: int = logging.INFO,
logger: Optional[logging.Logger] = None,
msg: str = "Logging Clock : %s",
@@ -121,13 +121,13 @@ class LoggingClock(Thread):
):
super().__init__(*args, **kwargs)
self.__action_func = action_func or self.action
self.__action = action or self.default_action
self.__level = level
self.__logger = logger or logging.getLogger(self.__module__)
self.__msg = msg
self.__precision = precision
def action(self):
def default_action(self):
level = self.__level
logger = self.__logger
msg = self.__msg % datetime.now()

4
gamestonk_terminal/log/collection/s3_sender.py
View File

@@ -14,12 +14,10 @@ finally:
# IMPORTATION INTERNAL
from gamestonk_terminal import config_terminal as cfg
from gamestonk_terminal.log.constants import DEFAULT_API_URL, DEFAULT_BUCKET
# DO NOT USE THE FILE LOGGER IN THIS MODULE
DEFAULT_BUCKET = "gst-restrictions"
DEFAULT_API_URL = "https://knaqi3sud7.execute-api.eu-west-3.amazonaws.com/log_api/logs"
def send_to_s3_directly(
aws_access_key: str,

5
gamestonk_terminal/log/constants.py Normal file
View File

@@ -0,0 +1,5 @@
ARCHIVES_FOLDER_NAME = "archives"
DEFAULT_BUCKET = "gst-restrictions"
DEFAULT_API_URL = "https://knaqi3sud7.execute-api.eu-west-3.amazonaws.com/log_api/logs"
S3_FOLDER_SUFFIX = "-app"
TMP_FOLDER_NAME = "tmp"

2
gamestonk_terminal/log/generation/directories.py
View File

@@ -33,8 +33,6 @@ def get_log_dir() -> Path:
if not os.path.isdir(uuid_log_dir.absolute()):
os.mkdir(uuid_log_dir.absolute())
# print(uuid_log_dir)
return uuid_log_dir

29
gamestonk_terminal/log/generation/expired_files.py Normal file
View File

@@ -0,0 +1,29 @@
# IMPORTATION STANDARD
from datetime import datetime
from pathlib import Path
from typing import List
# IMPORTATION THIRDPARTY
# IMPORTATION INTERNAL
def get_timestamp_from_x_days(x: int):
timestamp_from_x_days = datetime.now().timestamp() - x * 86400
return timestamp_from_x_days
def get_expired_files(directory: Path, before_timestamp: float):
expired_files = list()
if directory.exists and directory.is_dir():
for file in directory.iterdir():
if file.lstat().st_mtime < before_timestamp:
expired_files.append(file)
return expired_files
def remove_file_list(file_list: List[Path]):
for file in file_list:
file.unlink(missing_ok=True)

4
gamestonk_terminal/log/generation/formatter_with_exceptions.py
View File

@@ -52,8 +52,8 @@ class FormatterWithExceptions(logging.Formatter):
for word in text.split():
if (
ip_reg.search(word)
and int(max(word.split(""))) < 256
and int(min(word.split(""))) >= 0
# and int(max(word.split(""))) < 256
# and int(min(word.split(""))) >= 0
):
s_list.append("suspected_ip")
elif "@" in word and "." in word:

108
gamestonk_terminal/log/generation/path_tracking_file_handler.py
View File

@@ -2,15 +2,23 @@
from copy import deepcopy
from logging.handlers import TimedRotatingFileHandler
from pathlib import Path
from typing import Callable
# IMPORTATION THIRDPARTY
# IMPORTATION INTERNAL
from gamestonk_terminal import feature_flags as gtff
from gamestonk_terminal.log.constants import ARCHIVES_FOLDER_NAME, TMP_FOLDER_NAME
from gamestonk_terminal.log.collection.log_sender import LogSender
from gamestonk_terminal.log.collection.logging_clock import LoggingClock, Precision
from gamestonk_terminal.log.generation.settings import Settings
from gamestonk_terminal.log.generation.directories import get_log_dir, get_log_sub_dir
from gamestonk_terminal.log.generation.expired_files import (
get_expired_files,
get_timestamp_from_x_days,
remove_file_list,
)
# DO NOT USE THE FILE LOGGER IN THIS MODULE
from gamestonk_terminal.log.generation.settings import Settings
class PathTrackingFileHandler(TimedRotatingFileHandler):
@@ -34,7 +42,38 @@ class PathTrackingFileHandler(TimedRotatingFileHandler):
return log_sender
def build_rolling_clock(self, frequency: str, start: bool) -> LoggingClock:
@staticmethod
def clean_expired_files(before_timestamp: float):
"""Deleting old files inside : archives and tmp folders.
Only files inside the following folders are considered :
- {LOG_FOLDER_PATH}/{ARCHIVES_FOLDER_NAME}
- {LOG_FOLDER_PATH}/{TMP_FOLDER_NAME}
Args:
before_timestamp (float): Timestamp before which files are considered expired.
"""
archives_directory = get_log_sub_dir(name=ARCHIVES_FOLDER_NAME)
tmp_directory = get_log_sub_dir(name=TMP_FOLDER_NAME)
expired_archives_file_list = get_expired_files(
directory=archives_directory,
before_timestamp=before_timestamp,
)
expired_tmp_file_list = get_expired_files(
directory=tmp_directory,
before_timestamp=before_timestamp,
)
remove_file_list(file_list=expired_archives_file_list)
remove_file_list(file_list=expired_tmp_file_list)
@staticmethod
def build_rolling_clock(
action: Callable,
frequency: str,
start: bool,
) -> LoggingClock:
frequency = frequency.upper()
if frequency == "H":
@@ -45,7 +84,7 @@ class PathTrackingFileHandler(TimedRotatingFileHandler):
raise AttributeError("Unsupported `logging_clock.Precision`.")
rolling_clock = LoggingClock(
action_func=self.doRollover,
action=action,
daemon=True,
precision=precision,
)
@@ -55,6 +94,27 @@ class PathTrackingFileHandler(TimedRotatingFileHandler):
return rolling_clock
def send_expired_files(self, before_timestamp: float):
"""Try to send files older than before_timestamp days.
Only files inside the following folder are considered :
- {LOG_FOLDER_PATH}/
Args:
before_timestamp (float): Timestamp before which files are considered expired.
"""
log_sender = self.__log_sender
log_directory = get_log_dir()
expired_log_file_list = get_expired_files(
directory=log_directory,
before_timestamp=before_timestamp,
)
for file in expired_log_file_list:
log_sender.send_path(last=False, path=file)
@property
def log_sender(self) -> LogSender:
return self.__log_sender
@@ -79,31 +139,39 @@ class PathTrackingFileHandler(TimedRotatingFileHandler):
frequency = settings.log_settings.frequency
kwargs["when"] = frequency
super().__init__(filename, *args, **kwargs)
self.suffix += ".log"
self.suffix += ".log"
self.__settings = settings
self.__log_sender = self.build_log_sender(settings=settings, start=True)
self.__rolling_clock = self.build_rolling_clock(
frequency=frequency, start=rolling_clock
)
self.clean_expired_files(before_timestamp=get_timestamp_from_x_days(x=5))
if gtff.LOG_COLLECTION:
self.__log_sender = self.build_log_sender(settings=settings, start=True)
self.__rolling_clock = self.build_rolling_clock(
action=self.doRollover,
frequency=frequency,
start=rolling_clock,
)
self.send_expired_files(before_timestamp=get_timestamp_from_x_days(x=3))
# OVERRIDE
def doRollover(self) -> None:
super().doRollover()
# print("I am rolling.")
log_sender = self.__log_sender
to_delete_path_list = self.getFilesToDelete()
for path in to_delete_path_list:
# print(path)
log_sender.send_path(path=Path(path))
if gtff.LOG_COLLECTION:
log_sender = self.__log_sender
to_delete_path_list = self.getFilesToDelete()
for path in to_delete_path_list:
log_sender.send_path(path=Path(path))
# OVERRIDE
def close(self):
"""Do not use the file logger in this function."""
super().close()
log_sender = self.__log_sender
closed_log_path = self.baseFilename
log_sender.send_path(path=Path(closed_log_path), last=True)
log_sender.join()
# print("Exiting", self.baseFilename)
if gtff.LOG_COLLECTION:
log_sender = self.__log_sender
closed_log_path = self.baseFilename
log_sender.send_path(path=Path(closed_log_path), last=True)
log_sender.join()

38
gamestonk_terminal/portfolio/brokers/degiro/degiro_controller.py
View File

@@ -1,10 +1,12 @@
# IMPORTATION STANDARD
import argparse
import logging
from typing import List
# IMPORTATION THIRDPARTY
from prompt_toolkit.completion import NestedCompleter
import gamestonk_terminal.config_terminal as config
# IMPORTATION INTERNAL
from gamestonk_terminal import feature_flags as gtff
from gamestonk_terminal.decorators import log_start_end
from gamestonk_terminal.helper_funcs import parse_known_args_and_warn
@@ -205,37 +207,10 @@ class DegiroController(BaseController):
add_help=False,
prog="login",
)
parser.add_argument(
"-u",
"--username",
type=str,
default=config.DG_USERNAME,
help="Username in Degiro's account.",
)
parser.add_argument(
"-p",
"--password",
type=str,
default=config.DG_PASSWORD,
help="Password in Degiro's account.",
)
parser.add_argument(
"-o",
"--otp",
type=int,
default=None,
help="One time password (2FA).",
)
parser.add_argument(
"-s",
"--topt-secret",
type=str,
default=None,
help="TOTP SECRET (2FA).",
)
ns_parser = parse_known_args_and_warn(parser, other_args)
self.__degiro_view.login(ns_parser=ns_parser)
if ns_parser:
self.__degiro_view.login()
@log_start_end(log=logger)
def call_logout(self, other_args: List[str]):
@@ -248,7 +223,8 @@ class DegiroController(BaseController):
)
ns_parser = parse_known_args_and_warn(parser, other_args)
self.__degiro_view.logout(ns_parser=ns_parser)
if ns_parser:
self.__degiro_view.logout()
@log_start_end(log=logger)
def call_lookup(self, other_args: List[str]):

3
gamestonk_terminal/portfolio/brokers/degiro/degiro_model.py
View File

@@ -265,8 +265,7 @@ class DegiroModel:
return news
@log_start_end(log=logger)
def login(self, credentials: Credentials):
self.__trading_api.credentials.CopyFrom(credentials)
def login(self):
self.__trading_api.connect()
self.__setup_extra_credentials()

17
gamestonk_terminal/portfolio/brokers/degiro/degiro_view.py
View File

@@ -2,6 +2,7 @@
import logging
from argparse import Namespace
# IMPORTATION THIRDPARTY
import pandas as pd
from degiro_connector.core.helpers import pb_handler
from degiro_connector.trading.models.trading_pb2 import (
@@ -306,22 +307,14 @@ class DegiroView:
@log_start_end(log=logger)
@check_api_key(["DG_USERNAME", "DG_PASSWORD"])
def login(self, ns_parser: Namespace):
def login(self):
# GET ATTRIBUTES
degiro_model = self.__degiro_model
default_credentials = degiro_model.login_default_credentials()
credentials = Credentials()
credentials.CopyFrom(default_credentials)
credentials.username = ns_parser.username
credentials.password = ns_parser.password
if ns_parser.otp is not None:
credentials.one_time_password = ns_parser.otp
if ns_parser.topt_secret is not None:
credentials.totp_secret_key = ns_parser.topt_secret
degiro_model.login(credentials=credentials)
degiro_model.login()
DegiroView.__login_display_success()
@@ -331,9 +324,7 @@ class DegiroView:
console.print("You are now logged in !")
@log_start_end(log=logger)
def logout(self, ns_parser: Namespace):
_ = ns_parser
def logout(self):
# GET ATTRIBUTES
degiro_model = self.__degiro_model

3
tests/gamestonk_terminal/portfolio/brokers/degiro/test_degiro_controller.py
View File

@@ -213,7 +213,8 @@ def test_call_func_expect_queue(expected_queue, func, queue):
"call_create",
[
"--action=buy",
"--symbol=MOCK_SYMBOL" "--price=1",
"--symbol=MOCK_SYMBOL",
"--price=1",
"--size=2",
],
"DegiroView.create",

2
tests/gamestonk_terminal/portfolio/brokers/degiro/test_degiro_model.py
View File

@@ -255,7 +255,7 @@ def test_login(mocker):
new=mock_get_client_details,
)
model.login(credentials=trading_pb2.Credentials())
model.login()
mock_connect.assert_called_once()
mock_get_client_details.assert_called_once()