From 20ce4aa7344742172b7a4e907bac60885aa91b0c Mon Sep 17 00:00:00 2001 From: Chavithra Date: Sat, 26 Mar 2022 10:19:50 +0100 Subject: [PATCH 1/3] Log Collection : fixes (#1558) * Log Collection : expired files issue * Log Collection : add timeout at exit * Log Collection : fix access keys issue * Log Collection : linting --- gamestonk_terminal/config_terminal.py | 4 ++-- gamestonk_terminal/log/collection/s3_sender.py | 16 ++++++++-------- .../log/generation/expired_files.py | 2 +- .../log/generation/path_tracking_file_handler.py | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/gamestonk_terminal/config_terminal.py b/gamestonk_terminal/config_terminal.py index b123a2c4428..f0a4f94746d 100644 --- a/gamestonk_terminal/config_terminal.py +++ b/gamestonk_terminal/config_terminal.py @@ -145,8 +145,8 @@ API_SMARTSTAKE_KEY = os.getenv("GT_API_SMARTSTAKE_KEY") or "REPLACE_ME" API_SMARTSTAKE_TOKEN = os.getenv("GT_API_SMARTSTAKE_TOKEN") or "REPLACE_ME" # AWS KEYS -AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID") or "REPLACE_ME" -AWS_ACCESS_KEY = os.getenv("AWS_ACCESS_KEY") or "REPLACE_ME" +AWS_ACCESS_KEY_ID = os.getenv("GT_AWS_ACCESS_KEY_ID") or "REPLACE_ME" +AWS_SECRET_ACCESS_KEY = os.getenv("GT_AWS_SECRET_ACCESS_KEY") or "REPLACE_ME" # https://messari.io/ API_MESSARI_KEY = os.getenv("GT_API_MESSARI_KEY") or "REPLACE_ME" diff --git a/gamestonk_terminal/log/collection/s3_sender.py b/gamestonk_terminal/log/collection/s3_sender.py index 167d265e260..c40b4f96455 100644 --- a/gamestonk_terminal/log/collection/s3_sender.py +++ b/gamestonk_terminal/log/collection/s3_sender.py @@ -8,9 +8,9 @@ import requests try: import boto3 except ImportError: - WITH_BOTO3 = True -finally: WITH_BOTO3 = False +finally: + WITH_BOTO3 = True # IMPORTATION INTERNAL from gamestonk_terminal import config_terminal as cfg @@ -20,8 +20,8 @@ from gamestonk_terminal.log.constants import DEFAULT_API_URL, DEFAULT_BUCKET def send_to_s3_directly( - aws_access_key: str, aws_access_key_id: str, + aws_secret_access_key: str, bucket: str, file: Path, object_key: str, @@ -31,8 +31,8 @@ def send_to_s3_directly( s3_client = boto3.client( service_name="s3", - aws_access_key_id=aws_access_key, - aws_secret_access_key=aws_access_key_id, + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, ) s3_client.upload_file(str(file), bucket, object_key) @@ -70,8 +70,8 @@ def send_to_s3_using_presigned_url( def send_to_s3(archives_file: Path, file: Path, object_key: str, tmp_file: Path): api_url = DEFAULT_API_URL - aws_access_key = cfg.AWS_ACCESS_KEY aws_access_key_id = cfg.AWS_ACCESS_KEY_ID + aws_secret_access_key = cfg.AWS_SECRET_ACCESS_KEY bucket = DEFAULT_BUCKET if file.stat().st_size <= 0: @@ -81,10 +81,10 @@ def send_to_s3(archives_file: Path, file: Path, object_key: str, tmp_file: Path) tmp_file.parent.mkdir(exist_ok=True) file = file.rename(tmp_file) - if aws_access_key != "REPLACE_ME" and aws_access_key_id != "REPLACE_ME": + if aws_access_key_id != "REPLACE_ME" and aws_secret_access_key != "REPLACE_ME": send_to_s3_directly( - aws_access_key=aws_access_key_id, aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, bucket=bucket, file=file, object_key=object_key, diff --git a/gamestonk_terminal/log/generation/expired_files.py b/gamestonk_terminal/log/generation/expired_files.py index 7da9870a055..4b5b46c58ac 100644 --- a/gamestonk_terminal/log/generation/expired_files.py +++ b/gamestonk_terminal/log/generation/expired_files.py @@ -18,7 +18,7 @@ def get_expired_file_list(directory: Path, before_timestamp: float): expired_files = list() if directory.exists and directory.is_dir(): for file in directory.iterdir(): - if file.lstat().st_mtime < before_timestamp: + if file.is_file() and file.lstat().st_mtime < before_timestamp: expired_files.append(file) return expired_files diff --git a/gamestonk_terminal/log/generation/path_tracking_file_handler.py b/gamestonk_terminal/log/generation/path_tracking_file_handler.py index a3014102290..ca9675cfd5f 100644 --- a/gamestonk_terminal/log/generation/path_tracking_file_handler.py +++ b/gamestonk_terminal/log/generation/path_tracking_file_handler.py @@ -175,6 +175,6 @@ class PathTrackingFileHandler(TimedRotatingFileHandler): closed_log_path = self.baseFilename log_sender.send_path(path=Path(closed_log_path), last=True) try: - log_sender.join() + log_sender.join(timeout=3) except Exception: pass From c2b035c4da79b4522b9412530a7c3ffa4dfceb56 Mon Sep 17 00:00:00 2001 From: Colin Delahunty <72827203+colin99d@users.noreply.github.com> Date: Sat, 26 Mar 2022 08:53:42 -0400 Subject: [PATCH 2/3] Fixing tests (#1559) --- .github/workflows/mac.yml | 2 +- .github/workflows/test.yml | 1 - .github/workflows/windows.yml | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 292c73c6545..0367618ede1 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -65,7 +65,6 @@ jobs: restore-keys: | macos-pip- - # TODO: Is there a better way to do this? - name: Uninstall Brotlipy shell: bash -l {0} run: | @@ -76,6 +75,7 @@ jobs: run: | conda info pip install --requirement requirements-full.txt + pip install --requirements bots/requirements.txt - name: List installed packages (Bash) shell: bash -l {0} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index efb7be94879..a16c62cc97e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -237,7 +237,6 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - # TODO: Is there a better way to do this? - name: Uninstall Brotlipy shell: bash -l {0} run: | diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 4ce2375c19f..d77971c01a8 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -77,7 +77,6 @@ jobs: restore-keys: | windows-pip- - # TODO: Is there a better way to do this? - name: Uninstall Brotlipy shell: bash -l {0} run: | @@ -88,6 +87,7 @@ jobs: run: | conda info pip install --requirement requirements-full.txt + pip install --requirements bots/requirements.txt - name: List installed packages (Bash) shell: bash -l {0} From 2444b4e53f2ed698ec24f2586c13a9a49c7b219d Mon Sep 17 00:00:00 2001 From: Colin Delahunty <72827203+colin99d@users.noreply.github.com> Date: Sat, 26 Mar 2022 09:35:27 -0400 Subject: [PATCH 3/3] Fix main tests (#1560) * Fixing tests * Fixed embarassing mistake --- .github/workflows/mac.yml | 2 +- .github/workflows/windows.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 0367618ede1..3080646e1cd 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -75,7 +75,7 @@ jobs: run: | conda info pip install --requirement requirements-full.txt - pip install --requirements bots/requirements.txt + pip install --requirement bots/requirements.txt - name: List installed packages (Bash) shell: bash -l {0} diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index d77971c01a8..01509353059 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -87,7 +87,7 @@ jobs: run: | conda info pip install --requirement requirements-full.txt - pip install --requirements bots/requirements.txt + pip install --requirement bots/requirements.txt - name: List installed packages (Bash) shell: bash -l {0}