jerry/OpenBB
1
0
Fork 0
mirror of https://github.com/OpenBB-finance/OpenBB.git synced 2026-10-27 04:31:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove terminal CI actions (#6221)

Browse Source 
* remove ultima + althub

* move twitter keys

* Build + remove v3 docs

* Delete terminal lints

* Remove builds + remove terminal nightly

* more builds + website gen

* more terminal actions
This commit is contained in:
James Maslek
2024-03-14 16:37:47 -04:00
committed by GitHub
parent a46404091e
commit 9b0d34076f
13 changed files with 2 additions and 1892 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
.github/workflows/build-release.yml vendored
View File

@@ -1,43 +0,0 @@
name: Build & Release
env:
OPENBB_ENABLE_QUICK_EXIT: true
OPENBB_LOG_COLLECT: false
OPENBB_USE_PROMPT_TOOLKIT: false
PIP_DEFAULT_TIMEOUT: 100
PYTHONNOUSERSITE: 1
on:
push:
branches:
- release/v3/*
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
permissions:
actions: write
jobs:
trigger-windows-build:
runs-on: ubuntu-latest
steps:
- name: Trigger Windows Build
uses: aurelien-baudet/workflow-dispatch@v2
with:
workflow: windows10_build.yml
token: ${{ secrets.GITHUB_TOKEN }}
wait-for-completion-timeout: 2h
trigger-macos-build:
runs-on: ubuntu-latest
steps:
- name: Trigger Intel & M1 MacOs Build
uses: aurelien-baudet/workflow-dispatch@v2
with:
workflow: macos-build.yml
token: ${{ secrets.GITHUB_TOKEN }}
wait-for-completion-timeout: 2h

11
.github/workflows/gh-pages.yml vendored
View File

@@ -51,17 +51,6 @@ jobs:
- name: Install dependencies
run: poetry install --no-interaction --no-root
- name: Install packages
run: poetry install --no-interaction -E forecast -E doc -E optimization
- name: Generate SDK v3 Markdown
run: |
source .venv/bin/activate
pip uninstall Brotli -y
pytest tests/website --autodoc
poetry run python website/generate_sdk_v3_markdown.py
poetry run python website/generate_terminal_v3_markdown.py
- name: Generate Platform Markdown
run: |
source .venv/bin/activate

13
.github/workflows/linting.yml vendored
View File

@@ -45,12 +45,10 @@ jobs:
python-version: "3.9"
architecture: x64
- name: Get changed files in openbb_terminal and openbb_platform for PR
- name: Get changed files in openbb_platform for PR
if: github.event_name == 'pull_request'
run: |
# "Checking PR diff"
echo "terminal_files=$(git diff --diff-filter=d --name-only origin/${{ github.base_ref }}...${{ github.head_ref }} | grep 'openbb_terminal/.*\.py$' | grep -v 'openbb_terminal/openbb/package' | grep -v 'integration' | grep -v 'tests' | xargs)" >> $GITHUB_ENV
echo $terminal_files
echo "platform_files=$(git diff --diff-filter=d --name-only origin/${{ github.base_ref }}...${{ github.head_ref }} | grep 'openbb_platform/.*\.py$' | grep -v 'openbb_platform/openbb/package' | grep -v 'integration' | grep -v 'tests' | xargs)" >> $GITHUB_ENV
echo $platform_files
@@ -67,15 +65,6 @@ jobs:
- run: black --diff --check .
- run: codespell --ignore-words=.codespell.ignore --skip="$(tr '\n' ',' < .codespell.skip | sed 's/,$//')" --quiet-level=2
- run: ruff .
# TODO: Enable linters for openbb_terminal once we're ready
# - run: |
# # Run linters for openbb_terminal
# if [ -n "${{ env.terminal_files }}" ]; then
# mypy --ignore-missing-imports ${{ env.terminal_files }}
# pylint terminal.py ${{ env.terminal_files }}
# else
# echo "No Python files changed in openbb_terminal"
# fi
- run: |
# Run linters for openbb_platform
if [ -n "${{ env.platform_files }}" ]; then

280
.github/workflows/macos-build.yml vendored
View File

@@ -1,280 +0,0 @@
name: MacOS Build
env:
OPENBB_LOG_COLLECT: false
OPENBB_USE_PROMPT_TOOLKIT: false
OPENBB_FILE_OVERWRITE: true
PIP_DEFAULT_TIMEOUT: 100
PYTHONNOUSERSITE: 1
on: workflow_dispatch
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
M1-MacOs-Build:
name: M1 MacOS Build
runs-on: [self-hosted, macos, ARM64]
steps:
# Checkout repository main branch. this allows for the commit hashes to line up
- name: Checkout
uses: actions/checkout@v3
- name: Git Log
run: git log
# The following commands to clear previous PATHS and restore to defaults since we have to maintain the instance ourselves
- name: Clean Previous Path
run: |
export PATH=""
export PATH="/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"
echo $PATH
# Set up caching for conda env so that the workflow runs quickly after the first time
- name: Setup Conda Caching
uses: actions/cache@v3
with:
path: ~/conda_pkgs_dir
key: conda-macos-3-9-${{ hashFiles('build/conda/conda-3-9-env.yaml') }}
# Set up miniconda using the environment yaml file within the repo
- name: Setup Miniconda
uses: conda-incubator/setup-miniconda@v2.2.0
with:
miniconda-version: "latest"
auto-update-conda: true
channels: conda-forge,defaults
show-channel-urls: true
channel-priority: flexible
environment-file: build/conda/conda-3-9-env.yaml
activate-environment: obb
auto-activate-base: false
use-only-tar-bz2: true # Needed for caching some reason
- name: Install Dependencies
shell: bash -l {0}
run: |
conda info
pip list
build/conda/cleanup_artifacts.sh
poetry install -E all -E installer
pip uninstall papermill -y
pip install git+https://github.com/nteract/papermill.git@main
pip list
- name: Build Bundle
run: build/pyinstaller/build4mac.sh
shell: bash -l {0}
- name: Creating Application Keychain
env:
MACOS_CERTIFICATE: ${{ secrets.MACOS_CERTIFICATE }}
MACOS_CERTIFICATE_PWD: ${{ secrets.MACOS_CERTIFICATE_PWD }}
MACOS_KEYCHAIN_PWD: ${{ secrets.MACOS_KEYCHAIN_PWD }}
MACOS_CODESIGN_IDENTITY: ${{ secrets.MACOS_CODESIGN_IDENTITY }}
run:
| # when pushing to main, make to generate new cert, and utilize secrets to store new password, and identity
echo "Ensuring Keychain with same name does not exist"
rm -rf /Users/openbb/Library/Keychains/build.keychain-db
echo "Decoding certificate"
echo $MACOS_CERTIFICATE | base64 --decode > certificate.p12
echo "Creating Keychain"
security create-keychain -p $MACOS_KEYCHAIN_PWD build.keychain
echo "Setting Default Keychain"
security default-keychain -s build.keychain
echo "Unlocking Keychain"
security unlock-keychain -p $MACOS_KEYCHAIN_PWD build.keychain
echo "Importing Keychain"
security import certificate.p12 -k build.keychain -P $MACOS_CERTIFICATE_PWD -T /usr/bin/codesign
echo "Setting Partition List"
security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k $MACOS_KEYCHAIN_PWD build.keychain
- name: Create Signed Package
env:
APPLE_DEVELOPER_CERTIFICATE_ID: ${{ secrets.MACOS_INSTALLER_CODESIGN_IDENTITY }}
MACOS_CERTIFICATE_PWD: ${{ secrets.MACOS_CERTIFICATE_PWD }}
MACOS_KEYCHAIN_PWD: ${{ secrets.MACOS_KEYCHAIN_PWD }}
MACOS_INSTALLER_KEYCHAIN_PWD: ${{ secrets.MACOS_INSTALLER_KEYCHAIN_PWD }}
APPLE_SIGNING_IDENTITY: ${{ secrets.MACOS_CODESIGN_IDENTITY }}
run: |
security unlock-keychain -p $MACOS_INSTALLER_KEYCHAIN_PWD install.keychain
build/pyinstaller/macOS/build-macos.sh OpenBBTerminal 0.0.1
mv build/pyinstaller/macOS/target/pkg/OpenBBTerminalM1.pkg OpenBBTerminal.pkg
shell: bash -l {0}
- name: Deleting Previous Keychain
run: |
echo "Deleting Previous Keychain to Clean Instance"
rm -rf /Users/openbb/Library/Keychains/build.keychain-db
- name: Clean up Build Artifacts
run: |
rm -rf build/terminal
rm -rf dist
rm -rf DMG
- name: Notorize DMG
env:
NOTARIZE_APPLE_ID: ${{ secrets.NOTARIZE_APPLE_ID }}
NOTARIZE_APPLE_PWD: ${{ secrets.NOTARIZE_APPLE_PWD }}
NOTARIZE_APPLE_TEAM_ID: ${{ secrets.NOTARIZE_APPLE_TEAM_ID }}
run: |
xcrun notarytool submit OpenBBTerminal.pkg --apple-id "$NOTARIZE_APPLE_ID" --password "$NOTARIZE_APPLE_PWD" --team-id "$NOTARIZE_APPLE_TEAM_ID" --wait
- name: Staple
run: |
xcrun stapler staple OpenBBTerminal.pkg
- name: Clean up Build Artifacts
run: |
rm -rf build/terminal
rm -rf dist
rm -rf DMG
- name: Save Build Artifact PKG
uses: actions/upload-artifact@v3
with:
name: OpenBBM1.pkg
path: OpenBBTerminal.pkg
- name: Get contents from PkgUtil
run: |
pkgutil --expand-full OpenBBTerminal.pkg extract/
rm -rf OpenBBTerminal.pkg
- name: Remove OpenBB Folder
run: |
rm -rf /Users/openbb/Desktop/OpenBB\ Terminal
rm -rf ~/Desktop/OPENBB-exports
rm -rf extract/
# Job to build the MacOS Intel version of the Terminal===================================
Intel-MacOs-Build:
name: Intel MacOS Build
runs-on: [self-hosted, macos, x64]
steps:
# Checkout repository main branch. this allows for the commit hashes to line up
- name: Checkout
uses: actions/checkout@v3
- name: Git Log
run: git log
# The following commands to clear previous PATHS and restore to defaults since we have to maintain the instance ourselves
- name: Clean Previous Path
run: |
export PATH=""
export PATH="/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"
echo $PATH
# Set up caching for conda env so that the workflow runs quickly after the first time
- name: Setup Conda Caching
uses: actions/cache@v3
with:
path: ~/conda_pkgs_dir
key: conda-macos-3-9-${{ hashFiles('build/conda/conda-3-9-env.yaml') }}
# Set up miniconda using the environment yaml file within the repo
- name: Setup Miniconda
uses: conda-incubator/setup-miniconda@v2.2.0
with:
miniconda-version: "latest"
auto-update-conda: true
channels: conda-forge,defaults
show-channel-urls: true
channel-priority: flexible
environment-file: build/conda/conda-3-9-env.yaml
activate-environment: obb
auto-activate-base: false
use-only-tar-bz2: true # Needed for caching some reason
- name: Install Dependencies
shell: bash -l {0}
run: |
conda info
pip list
build/conda/cleanup_artifacts.sh
poetry install -E all -E installer
pip uninstall papermill -y
pip install git+https://github.com/nteract/papermill.git@main
pip list
- name: Build Bundle
run: build/pyinstaller/build4mac.sh
shell: bash -l {0}
- name: Creating Application Keychain
env:
MACOS_CERTIFICATE: ${{ secrets.MACOS_CERTIFICATE }}
MACOS_CERTIFICATE_PWD: ${{ secrets.MACOS_CERTIFICATE_PWD }}
MACOS_KEYCHAIN_PWD: ${{ secrets.MACOS_KEYCHAIN_PWD }}
MACOS_CODESIGN_IDENTITY: ${{ secrets.MACOS_CODESIGN_IDENTITY }}
run:
| # when pushing to main, make to generate new cert, and utilize secrets to store new password, and identity
echo "Ensuring Keychain with same name does not exist"
rm -rf /Users/openbb/Library/Keychains/build.keychain-db
echo "Decoding certificate"
echo $MACOS_CERTIFICATE | base64 --decode > certificate.p12
echo "Creating Keychain"
security create-keychain -p $MACOS_KEYCHAIN_PWD build.keychain
echo "Setting Default Keychain"
security default-keychain -s build.keychain
echo "Unlocking Keychain"
security unlock-keychain -p $MACOS_KEYCHAIN_PWD build.keychain
echo "Importing Keychain"
security import certificate.p12 -k build.keychain -P $MACOS_CERTIFICATE_PWD -T /usr/bin/codesign
echo "Setting Partition List"
security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k $MACOS_KEYCHAIN_PWD build.keychain
- name: Create Signed Package
env:
APPLE_DEVELOPER_CERTIFICATE_ID: ${{ secrets.MACOS_INSTALLER_CODESIGN_IDENTITY }}
MACOS_CERTIFICATE_PWD: ${{ secrets.MACOS_CERTIFICATE_PWD }}
MACOS_KEYCHAIN_PWD: ${{ secrets.MACOS_KEYCHAIN_PWD }}
MACOS_INSTALLER_KEYCHAIN_PWD: ${{ secrets.MACOS_INSTALLER_KEYCHAIN_PWD }}
APPLE_SIGNING_IDENTITY: ${{ secrets.MACOS_CODESIGN_IDENTITY }}
run: |
security unlock-keychain -p $MACOS_INSTALLER_KEYCHAIN_PWD install.keychain
build/pyinstaller/macOS/build-macos.sh OpenBBTerminal 0.0.1
mv build/pyinstaller/macOS/target/pkg/OpenBBTerminalM1.pkg OpenBBTerminal.pkg
shell: bash -l {0}
- name: Deleting Previous Keychain
run: |
echo "Deleting Previous Keychain to Clean Instance"
rm -rf /Users/openbb/Library/Keychains/build.keychain-db
- name: Clean up Build Artifacts
run: |
rm -rf build/terminal
rm -rf dist
rm -rf DMG
- name: Notorize DMG
env:
NOTARIZE_APPLE_ID: ${{ secrets.NOTARIZE_APPLE_ID }}
NOTARIZE_APPLE_PWD: ${{ secrets.NOTARIZE_APPLE_PWD }}
NOTARIZE_APPLE_TEAM_ID: ${{ secrets.NOTARIZE_APPLE_TEAM_ID }}
run: |
xcrun notarytool submit OpenBBTerminal.pkg --apple-id "$NOTARIZE_APPLE_ID" --password "$NOTARIZE_APPLE_PWD" --team-id "$NOTARIZE_APPLE_TEAM_ID" --wait
- name: Staple
run: |
xcrun stapler staple OpenBBTerminal.pkg
- name: Clean up Build Artifacts
run: |
rm -rf build/terminal
rm -rf dist
rm -rf DMG
- name: Save Build Artifact PKG
uses: actions/upload-artifact@v3
with:
name: OpenBBIntel.pkg
path: OpenBBTerminal.pkg
- name: Get contents from PkgUtil
run: |
pkgutil --expand-full OpenBBTerminal.pkg extract/
rm -rf OpenBBTerminal.pkg
- name: Remove OpenBB Folder
run: |
rm -rf /Users/openbb/Desktop/OpenBB\ Terminal
rm -rf ~/Desktop/OPENBB-exports
rm -rf extract/

92
.github/workflows/macos-ml.yml vendored
View File

@@ -1,92 +0,0 @@
name: Mac OS X Full Clean Build with ML
env:
OPENBB_ENABLE_QUICK_EXIT: true
OPENBB_LOG_COLLECT: false
OPENBB_USE_PROMPT_TOOLKIT: false
PIP_DEFAULT_TIMEOUT: 100
OPENBB_FILE_OVERWRITE: true
PYTHONNOUSERSITE: 1
on:
push:
branches:
- "main"
workflow_dispatch:
inputs:
comments:
required: true
description: "Test scenario tags"
jobs:
macos-conda:
name: MacOS-latest, Conda-latest, Python 3.9
runs-on: macos-latest
steps:
- name: Checkout
uses: actions/checkout@v2
with:
# Need to fetch more than the last commit so that setuptools-scm can
# create the correct version string. If the number of commits since
# the last release is greater than this, the version still be wrong.
# Increase if necessary.
fetch-depth: 100
# The GitHub token is preserved by default but this job doesn't need
# to be able to push to GitHub.
persist-credentials: false
- name: Setup caching for conda packages
uses: actions/cache@v2
with:
path: ~/conda_pkgs_dir
key: conda-macos-3-9-${{ hashFiles('build/conda/conda-3-9-env.yaml') }}
- name: Setup Miniconda
uses: conda-incubator/setup-miniconda@v2
with:
miniconda-version: "latest"
auto-update-conda: true
channels: conda-forge,defaults
show-channel-urls: true
channel-priority: flexible
environment-file: build/conda/conda-3-9-env.yaml
activate-environment: testing
# Needed for caching
use-only-tar-bz2: true
- name: Get pip cache dir
shell: bash -l {0}
id: pip-cache
run: |
echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
- name: pip cache
uses: actions/cache@v2
with:
path: ${{ steps.pip-cache.outputs.dir }}
key: macos-3-9-pip-${{ hashFiles('**/requirements-full.txt') }}
restore-keys: |
macos-pip-
- name: Uninstall Brotlipy
shell: bash -l {0}
run: |
conda remove brotlipy -y
- name: Install dependencies (Bash)
shell: bash -l {0}
run: |
conda info
pip install --requirement requirements-full.txt
pip install docstring_parser
- name: List installed packages (Bash)
shell: bash -l {0}
run: |
conda list
pip list
- name: Run tests (Bash)
shell: bash -l {0}
run: pytest tests/
- name: Start Terminal and exit
shell: bash -l {0}
run: python terminal.py

149
.github/workflows/manual-unit-test.yml vendored
View File

@@ -1,149 +0,0 @@
name: Manually Triggered Unit Test
env:
OPENBB_ENABLE_QUICK_EXIT: true
OPENBB_LOG_COLLECT: false
OPENBB_USE_PROMPT_TOOLKIT: false
OPENBB_FILE_OVERWRITE: true
OPENBB_ENABLE_CHECK_API: false
OPENBB_PREVIOUS_USE: true
OPENBB_USE_INTERACTIVE_DF: false
PIP_DEFAULT_TIMEOUT: 100
on:
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
tests-python:
name: Vanilla Python Tests - ${{ matrix.python-version }}
runs-on: ${{ matrix.os }}
strategy:
fail-fast: true
matrix:
python-version: ["3.8", "3.9", "3.10"]
os: [ubuntu-latest]
exclude:
- os: ubuntu-latest
python-version: "3.9"
steps:
- name: Checkout Code
uses: actions/checkout@v3
- name: Setup Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
architecture: x64
- name: Install Poetry
uses: snok/install-poetry@v1
with:
version: 1.4.0
virtualenvs-create: true
virtualenvs-in-project: true
- name: Setup sudo apt installs for ubuntu-latest
run: |
sudo apt-get update
sudo apt-get install -y \
libgtk-3-dev \
libwebkit2gtk-4.0-dev
- name: Load cached venv
id: cached-poetry-dependencies
uses: actions/cache@v3
with:
path: .venv
key: venv-${{ runner.os }}-v1-${{ hashFiles('**/poetry.lock') }}
- name: Install dependencies
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
run: poetry install --no-interaction -E optimization
- name: List installed packages
shell: bash -l {0}
run: |
source $VENV
pip list
- name: Run tests
env:
MPLBACKEND: Agg
run: |
source $VENV
pytest tests/ --optimization --autodoc -n auto --timeout=30
- name: Start Terminal and exit
run: |
source $VENV
python terminal.py
full-test:
name: Full Test Suite - (${{ matrix.python-version }}, ${{ matrix.os }})
runs-on: ${{ matrix.os }}
# Run only when a PR is merged or a push to release/* branch
strategy:
fail-fast: false
matrix:
python-version: ["3.8", "3.9", "3.10"]
os: [ubuntu-latest, macos-latest]
exclude:
- os: ubuntu-latest
python-version: "3.9"
steps:
- name: Checkout Code
uses: actions/checkout@v3
- name: Setup Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
architecture: x64
- name: Install Poetry
uses: snok/install-poetry@v1
with:
version: 1.4.0
virtualenvs-create: true
virtualenvs-in-project: true
- name: Setup sudo apt installs for ubuntu-latest
if: runner.os == 'Linux'
run: |
sudo apt-get update
sudo apt-get install -y \
libgtk-3-dev \
libwebkit2gtk-4.0-dev
- name: Load cached venv
id: cached-poetry-dependencies
uses: actions/cache@v3
with:
path: .venv
key: venv-${{ runner.os }}-v1-${{ hashFiles('**/poetry.lock') }}
- name: Install dependencies
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
run: |
poetry install --no-interaction -E optimization
- name: List installed packages
shell: bash -l {0}
run: |
source $VENV
pip list
- name: Start Terminal and exit
run: |
source $VENV
python terminal.py
- name: Run tests
env:
MPLBACKEND: Agg
run: |
source $VENV
pytest tests/ --optimization --autodoc -n auto --timeout=30

31
.github/workflows/nightly-build.yml vendored
View File

@@ -16,35 +16,6 @@ permissions:
actions: write
jobs:
trigger-windows-build:
runs-on: ubuntu-latest
steps:
- name: Trigger Windows Build
uses: aurelien-baudet/workflow-dispatch@v2
with:
workflow: windows10_build.yml
token: ${{ secrets.GITHUB_TOKEN }}
wait-for-completion-timeout: 3h
trigger-macos-build:
runs-on: ubuntu-latest
steps:
- name: Trigger M1 & Intel MacOs Build
uses: aurelien-baudet/workflow-dispatch@v2
with:
workflow: macos-build.yml
token: ${{ secrets.GITHUB_TOKEN }}
wait-for-completion-timeout: 2h
trigger-docker-build:
runs-on: ubuntu-latest
steps:
- name: Trigger Docker Build
uses: aurelien-baudet/workflow-dispatch@v2
with:
workflow: docker-build.yml
token: ${{ secrets.GITHUB_TOKEN }}
trigger-pypi-build:
runs-on: ubuntu-latest
steps:
@@ -61,4 +32,4 @@ jobs:
uses: aurelien-baudet/workflow-dispatch@v2
with:
workflow: platform-api-integration-test.yml
token: ${{ secrets.GITHUB_TOKEN }}
token: ${{ secrets.GITHUB_TOKEN }}

22
.github/workflows/pypi-nightly.yml vendored
View File

@@ -23,28 +23,6 @@ jobs:
python -m
pip install
build
--user
- name: OpenBBTerminal - Update version in pyproject.toml and Edit docs
run: |
sed -i 's/name = ".*"/name = "openbb-terminal-nightly"/' pyproject.toml
sed -i "3s/version = \"\(.*\)\"/version = \"\1.dev$(date +%Y%m%d%H%M)\"/" pyproject.toml
sed -i 's|pip install openbb-terminal|pip install openbb-terminal-nightly|g' ./website/pypi.md
- name: OpenBBTerminal - Build a binary wheel and a source tarball
run: >-
python -m
build
--sdist
--wheel
--outdir dist/
.
- name: OpenBBTerminal - Publish distribution 📦 to PyPI
uses: pypa/gh-action-pypi-publish@release/v1
with:
packages-dir: dist/
password: ${{ secrets.NIGHTLY_PYPI_API_TOKEN_OBB_TERMINAL_NIGHTLY }}
- name: OpenBB Platform - Update version in pyproject.toml and Edit docs
run: |

184
.github/workflows/unit-test.yml vendored
View File

@@ -70,187 +70,3 @@ jobs:
run: |
echo "check-changes=${{ steps.check-changes.outputs.check-changes }}"
echo "check-platform-changes=${{ steps.check-platform-changes.outputs.check-platform-changes }}"
base-test:
name: Base Tests - Ubuntu-latest - Python 3.9
needs: [check-files-changed]
runs-on: ubuntu-latest
if: needs.check-files-changed.outputs.check-changes == 'true' && github.event.pull_request.base.ref == 'develop' && needs.check-files-changed.outputs.check-platform-changes == 'false'
steps:
- name: Checkout Code
uses: actions/checkout@v3
- name: Setup Python 3.9
uses: actions/setup-python@v4
with:
python-version: "3.9"
architecture: x64
- name: Install Poetry
uses: snok/install-poetry@v1
with:
version: 1.5.1
virtualenvs-create: true
virtualenvs-in-project: true
- name: Setup sudo apt installs for ubuntu-latest
run: |
sudo apt-get update
sudo apt-get install -y \
libgtk-3-dev \
libwebkit2gtk-4.0-dev
- name: Load cached venv
id: cached-poetry-dependencies
uses: actions/cache@v3
with:
path: .venv
key: venv-${{ runner.os }}-v1-${{ hashFiles('**/poetry.lock') }}
- name: Install dependencies
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
run: poetry install --no-interaction -E optimization
- name: Run tests and coverage test
env:
MPLBACKEND: Agg
run: |
source $VENV
pytest tests/ --optimization --cov --cov-fail-under=50 --autodoc -n auto --durations=10 --timeout=30 --cov-report=xml
- name: Start Terminal and exit
run: |
source $VENV
python terminal.py
tests-python:
name: Vanilla Python Tests - ${{ matrix.python-version }}
needs: [base-test]
runs-on: ${{ matrix.os }}
strategy:
fail-fast: true
matrix:
python-version: ["3.8", "3.9", "3.10", "3.11"]
os: [ubuntu-latest]
exclude:
- os: ubuntu-latest
python-version: "3.9"
steps:
- name: Checkout Code
uses: actions/checkout@v3
- name: Setup Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
architecture: x64
- name: Install Poetry
uses: snok/install-poetry@v1
with:
version: 1.5.1
virtualenvs-create: true
virtualenvs-in-project: true
- name: Setup sudo apt installs for ubuntu-latest
run: |
sudo apt-get update
sudo apt-get install -y \
libgtk-3-dev \
libwebkit2gtk-4.0-dev
- name: Load cached venv
id: cached-poetry-dependencies
uses: actions/cache@v3
with:
path: .venv
key: venv-${{ runner.os }}-v1-${{ hashFiles('**/poetry.lock') }}
- name: Install dependencies
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
run: poetry install --no-interaction -E optimization
- name: List installed packages
shell: bash -l {0}
run: |
source $VENV
pip list
- name: Run tests
env:
MPLBACKEND: Agg
run: |
source $VENV
pytest tests/ --optimization --autodoc -n auto --timeout=30
- name: Start Terminal and exit
run: |
source $VENV
python terminal.py
full-test:
name: Full Test Suite - (${{ matrix.python-version }}, ${{ matrix.os }})
runs-on: ${{ matrix.os }}
# Run only when a PR is merged or a push to release/* branch
if: github.event_name == 'push' || github.event.pull_request.merged == true
strategy:
fail-fast: false
matrix:
python-version: ["3.8", "3.9", "3.10", "3.11"]
os: [ubuntu-latest, macos-latest]
exclude:
- os: ubuntu-latest
python-version: "3.9"
steps:
- name: Checkout Code
uses: actions/checkout@v3
- name: Setup Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
architecture: x64
- name: Install Poetry
uses: snok/install-poetry@v1
with:
version: 1.5.1
virtualenvs-create: true
virtualenvs-in-project: true
- name: Setup sudo apt installs for ubuntu-latest
if: runner.os == 'Linux'
run: |
sudo apt-get update
sudo apt-get install -y \
libgtk-3-dev \
libwebkit2gtk-4.0-dev
- name: Load cached venv
id: cached-poetry-dependencies
uses: actions/cache@v3
with:
path: .venv
key: venv-${{ runner.os }}-v1-${{ hashFiles('**/poetry.lock') }}
- name: Install dependencies
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
run: |
poetry install --no-interaction -E optimization
- name: List installed packages
shell: bash -l {0}
run: |
source $VENV
pip list
- name: Start Terminal and exit
run: |
source $VENV
python terminal.py
- name: Run tests
env:
MPLBACKEND: Agg
run: |
source $VENV
pytest tests/ --optimization --autodoc -n auto --timeout=30

131
.github/workflows/windows10_build.yml vendored
View File

@@ -1,131 +0,0 @@
name: Windows10 Build
env:
OPENBB_ENABLE_QUICK_EXIT: true
OPENBB_LOG_COLLECT: false
OPENBB_USE_PROMPT_TOOLKIT: false
PIP_DEFAULT_TIMEOUT: 100
PYTHONNOUSERSITE: 1
on: workflow_dispatch
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
Windows-Build:
name: Windows10 Build ENV
runs-on: windows-latest
# Configuring -------------
steps:
- name: Setup Windows Git Configuration # This is under the assumption that git is already installed on the system
run: |
Set-ItemProperty 'HKLM:\System\CurrentControlSet\Control\FileSystem' -Name 'LongPathsEnabled' -value 1
git config --system core.longpaths true
- name: Checkout
uses: actions/checkout@v3
- name: Set up Python & Create OpenBB Environment
uses: actions/setup-python@v4
with:
python-version: '3.10'
- name: Create and start virtual environment
run: |
python -m venv venv --upgrade-deps
source venv/Scripts/activate
python -m pip install setuptools==65.4.1 wheel
python -m pip install -r requirements-full.txt --no-cache-dir
python -m pip uninstall papermill -y
python -m pip install git+https://github.com/nteract/papermill.git@main
python -m PyInstaller build/pyinstaller/terminal.spec --clean
pwd
shell: bash
- name: zip the file
run: |
Get-Location
Compress-Archive dist release.zip
shell: pwsh
- name: Upload Artifact
uses: actions/upload-artifact@v3
with:
name: OpenBB-Windows10-ENV
path: release.zip
Build-Exe:
name: Windows10 Build-Exe
runs-on: [self-hosted, Windows, x64]
needs: Windows-Build
# Configuring -------------
steps:
- name: Setup Windows Git Configuration # This is under the assumption that git is already installed on the system
run: |
Set-ItemProperty 'HKLM:\System\CurrentControlSet\Control\FileSystem' -Name 'LongPathsEnabled' -value 1
git config --system core.longpaths true
- name: Checkout
uses: actions/checkout@v3
- name: Download Artifact
uses: actions/download-artifact@v3
with:
name: OpenBB-Windows10-ENV
path: .
- name: unzip artifact for deployment
run: Expand-Archive release.zip -DestinationPath .
shell: pwsh
- name: Move Files into App Folder
run: cp -r .\dist\OpenBBTerminal\ .\build\nsis\app\
# don't need this because log collection is turned off
# - name: Remove Log Data
# run: Remove-Item .\build\nsis\app\openbb_terminal\logs\ -Recurse
- name: Build Exe File
run: |
C:\'.\Program Files (x86)\'\nsis\makensis.exe .\build\nsis\setup.nsi
# This should autosign the installer now - We really need to only do this when needed though. ON RELEASE
- name: Sign Installer
if: startsWith(github.ref, 'refs/heads/main')
run: |
& 'C:\Program Files (x86)\Windows Kits\10\bin\10.0.19041.0\x86\signtool.exe' sign /fd sha256 /tr http://ts.ssl.com /td sha256 /sha1 0bf95b74b3b8763434e1782df36052d42c5c0bef 'C:\Users\Administrator\actions-runner\_work\OpenBBTerminal\OpenBBTerminal\build\nsis\OpenBB Terminal Setup.exe'
- name: Save Build Artifact
uses: actions/upload-artifact@v3
with:
name: Windows EXE Artifact
path: '.\build\nsis\OpenBB Terminal Setup.exe'
- name: Run Integration Tests
run: |
dist\OpenBBTerminal\OpenBBTerminal.exe -t -s forecast cryptocurrency reports alternative economy futures econometrics dashboards portfolio stocks/test_stocks_options_screen.openbb stocks/test_stocks_options.openbb forex etf stocks/test_stocks_fa.openbb | tee result.txt
Select-String "================================ Integration Test Summary ================================" result.txt -Context 0,100 | Select-Object -Last 2000 | Out-File summary.txt
Write-output `n >> summary.txt
- name: Run Integration Tests Coverage Report
run: |
dist\OpenBBTerminal\OpenBBTerminal.exe -t --coverage | tee result.txt
Select-String "============================== Integration Coverage Summary ==============================" result.txt -Context 0,300 | Select-Object -Last 2000 >> summary.txt
- name: Upload summary to Slack
uses: adrey/slack-file-upload-action@master
with:
token: ${{ secrets.SLACK_API_TOKEN }}
initial_comment: "Windows Build Integration test summary"
title: "Windows Build Integration test summary"
path: summary.txt
channel: ${{ secrets.SLACK_CHANNEL_ID }}
# Cleaning ------------------------
# Make sure to add Remove-Item C:\Users\Administrator\Desktop\OPENBB-exports -Recurse whenever integration tests get fixed
- name: Remove Previous Build
run: |
Remove-Item '.\build\nsis\OpenBB Terminal Setup.exe' -Recurse
Remove-Item .\build\nsis\app\ -Recurse
Remove-Item .\dist\ -Recurse

107
.github/workflows/windows_ml.yml vendored
View File

@@ -1,107 +0,0 @@
name: Windows 10 Full Clean Build with ML
env:
OPENBB_ENABLE_QUICK_EXIT: true
OPENBB_LOG_COLLECT: false
OPENBB_USE_PROMPT_TOOLKIT: false
OPENBB_FILE_OVERWRITE: true
PIP_DEFAULT_TIMEOUT: 100
PYTHONNOUSERSITE: 1
on:
push:
branches:
- "main"
workflow_dispatch:
inputs:
comments:
required: true
description: "Test scenario tags"
jobs:
windows-conda:
name: Windows-latest, Conda-latest, Python 3.9
runs-on: windows-latest
steps:
- name: Set git to use LF
if: runner.os == 'Windows'
run: |
git config --global core.autocrlf false
git config --global core.eol lf
- name: Checkout
uses: actions/checkout@v2
with:
# Need to fetch more than the last commit so that setuptools-scm can
# create the correct version string. If the number of commits since
# the last release is greater than this, the version still be wrong.
# Increase if necessary.
fetch-depth: 100
# The GitHub token is preserved by default but this job doesn't need
# to be able to push to GitHub.
persist-credentials: false
- name: Setup caching for conda packages
uses: actions/cache@v2
with:
path: ~/conda_pkgs_dir
key: conda-windows-3-9-${{ hashFiles('build/conda/conda-3-9-env.yaml') }}
- name: Setup Miniconda
uses: conda-incubator/setup-miniconda@v2
with:
miniconda-version: "latest"
auto-update-conda: true
channels: conda-forge,defaults
show-channel-urls: true
channel-priority: flexible
environment-file: build/conda/conda-3-9-env.yaml
activate-environment: obb
# Needed for caching
use-only-tar-bz2: true
- name: Update Temp permissions on windows
if: runner.os == 'Windows'
shell: cmd /C CALL {0}
run: >-
c:\windows\system32\icacls C:\Users\runneradmin\AppData\Local\Temp /grant "everyone":F /t
- name: Get pip cache dir
shell: bash -l {0}
id: pip-cache
run: |
echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
- name: pip cache
uses: actions/cache@v2
with:
path: ${{ steps.pip-cache.outputs.dir }}
key: windows-3-9-pip-${{ hashFiles('requirements-full.txt') }}
restore-keys: |
windows-pip-
- name: Uninstall Brotlipy
shell: bash -l {0}
run: |
conda remove brotlipy -y
- name: Install dependencies (Bash)
shell: bash -l {0}
run: |
conda info
build\conda\cleanup_artifacts.bat
pip install docstring_parser
poetry install -E all -E installer
- name: List installed packages (Bash)
shell: bash -l {0}
run: |
conda list
pip list
- name: Run tests (Bash)
env:
MPLBACKEND: Agg
shell: bash -l {0}
run: pytest tests/ -m "not linux"
- name: Start Terminal and exit
shell: bash -l {0}
run: python terminal.py

396
website/generate_sdk_v3_markdown.py
View File

@@ -1,396 +0,0 @@
import inspect
import json
import re
import shutil
from pathlib import Path
from textwrap import shorten
from typing import Any, Dict, List, Literal, TextIO, Union
from docstring_parser import parse
from openbb_terminal.core.sdk.trailmap import Trailmap, get_trailmaps
from openbb_terminal.core.session.current_system import set_system_variable
set_system_variable("TEST_MODE", True)
set_system_variable("LOG_COLLECT", False)
website_path = Path(__file__).parent.absolute()
CONTENT_PATH = website_path / "content/sdk/reference"
SEO_META: Dict[str, Dict[str, Union[str, List[str]]]] = json.loads(
(website_path / "metadata/sdk_v3_seo_metadata.json").read_text()
)
reference_import = (
'import ReferenceCard from "@site/src/components/General/NewReferenceCard";\n\n'
)
refrence_ul_element = """<ul className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4 gap-4 -ml-6">"""
def get_function_meta(trailmap: Trailmap, trail_type: Literal["model", "view"]):
"""Gets the function meta data."""
func_attr = trailmap.func_attrs[trail_type]
if not func_attr.func_unwrapped:
return None
doc_parsed = parse(func_attr.long_doc) # type: ignore
line = func_attr.lineon
path = func_attr.full_path
func_def = func_attr.func_def
source_code_url = (
f"https://github.com/OpenBB-finance/OpenBBTerminal/tree/main/{path}#L{line}"
)
function_name = trailmap.view if trail_type == "view" else trailmap.model
params = []
for param in doc_parsed.params:
arg_default = (
func_attr.params[param.arg_name].default
if param.arg_name in func_attr.params
else None
)
params.append(
{
"name": param.arg_name,
"doc": param.description if param.description else "",
"type": param.type_name,
"default": (
arg_default if arg_default is not inspect.Parameter.empty else None
),
"optional": bool(arg_default is not inspect.Parameter.empty)
or param.is_optional,
}
)
if doc_parsed.returns:
returns = {
"doc": doc_parsed.returns.description,
"type": doc_parsed.returns.type_name,
}
else:
returns = None
examples = []
for example in doc_parsed.examples:
examples.append(
{
"snippet": example.snippet,
"description": example.description.strip(), # type: ignore
}
)
desc = doc_parsed.short_description or doc_parsed.long_description
cmd_meta = {
"name": trailmap.class_attr,
"path": path,
"function_name": function_name,
"func_def": func_def,
"source_code_url": source_code_url,
"description": desc if desc else "",
"params": params,
"returns": returns,
"examples": examples,
}
default_desc = (doc_parsed.short_description or " .").split(".").pop(
0
).strip().replace(":", "") or trailmap.class_attr
keywords = "\n- ".join(
k for k in trailmap.location_path + [trailmap.class_attr] if k
)
header = f"title: {trailmap.class_attr}\ndescription: {default_desc}\nkeywords:\n- {keywords}"
key = f"{'.'.join(trailmap.location_path)}.{trailmap.class_attr}"
if seo_meta := SEO_META.get(key, None):
keywords = "\n- ".join(seo_meta["keywords"])
header = f"title: {seo_meta['title']}\ndescription: {seo_meta['description']}\nkeywords:\n- {keywords}"
cmd_meta[
"header"
] = f"""---\n{header}\n---\n
import HeadTitle from '@site/src/components/General/HeadTitle.tsx';
<HeadTitle title="{key} - Reference | OpenBB SDK Docs" />\n\n"""
return cmd_meta
def generate_markdown(meta_model: dict, meta_view: dict):
main_model = meta_model
if not meta_model:
if not meta_view:
raise ValueError("No model or view")
main_model = meta_view
markdown = main_model["header"]
if meta_view and meta_model:
markdown += """import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';\n\n"""
if meta_view and meta_model:
markdown += f"""<Tabs>
<TabItem value="model" label="Model" default>\n
{generate_markdown_section(meta_model)}\n
</TabItem>
<TabItem value="view" label="Chart">\n
{generate_markdown_section(meta_view)}\n
</TabItem>
</Tabs>"""
else:
markdown += generate_markdown_section(main_model)
return markdown
def generate_markdown_section(meta: Dict[str, Any]):
# head meta https://docusaurus.io/docs/markdown-features/head-metadata
# use real description but need to parse it
markdown = (
f"{meta['description']}\n\nSource Code: [[link]({meta['source_code_url']})]\n\n"
)
markdown += f"```python wordwrap\n{meta['func_def']}\n```\n\n"
markdown += "---\n\n## Parameters\n\n"
if meta["params"]:
markdown += "| Name | Type | Description | Default | Optional |\n"
markdown += "| ---- | ---- | ----------- | ------- | -------- |\n"
for param in meta["params"]:
description = param["doc"].replace("\n", "<br/>") if param["doc"] else ""
markdown += f"| {param['name']} | {param['type']} | {description} | {param['default']} | {param['optional']} |\n" # noqa
markdown += "\n\n"
else:
markdown += "This function does not take any parameters.\n\n"
markdown += "---\n\n## Returns\n\n"
if meta["returns"]:
markdown += "| Type | Description |\n"
markdown += "| ---- | ----------- |\n"
return_desc = (
meta["returns"]["doc"].replace("\n", "<br/>")
if meta["returns"]["doc"]
else ""
)
markdown += f"| {meta['returns']['type']} | {return_desc} |\n"
else:
markdown += "This function does not return anything\n\n"
markdown += "---\n\n## Examples\n\n" if meta["examples"] else ""
prev_snippet = " "
for example in meta["examples"]:
if isinstance(example["snippet"], str) and ">>>" in example["snippet"]:
snippet = example["snippet"].replace(">>> ", "")
markdown += f"```python\n{snippet}\n```\n\n"
if example["description"] and prev_snippet != "":
markdown += f"```\n{example['description']}\n```\n"
prev_snippet = snippet.strip()
elif example["description"]:
markdown += f"\n{example['description']}\n\n"
else:
if example["description"]:
markdown += f"\n{example['description']}\n\n"
prev_snippet = ""
markdown += "---\n\n"
return markdown
def create_nested_menus_card(folder: Path, url: str) -> str:
sub_categories = [
sub.stem
for sub in folder.glob("**/**/*.md*")
if sub.is_file() and sub.stem != "index"
]
categories = shorten(
", ".join(sub_categories or [""]), width=116, placeholder="..."
)
url = f"/sdk/reference/{url}/{folder.name}".replace("//", "/")
index_card = f"""<ReferenceCard
title="{folder.name.replace("_", " ").title()}"
description="{categories}"
url="{url}"
/>\n"""
return index_card
def create_cmd_cards(cmd_text: List[Dict[str, str]]) -> str:
cmd_cards = ""
for cmd in cmd_text:
url = f"/sdk/reference/{cmd['url']}/{cmd['title']}".replace("//", "/")
description = shorten(f"{cmd['description']}", width=116, placeholder="...")
cmd_cards += f"""<ReferenceCard
title="{cmd["title"].replace("_", " ").title()}"
description="{description.replace('None', '')}"
url="{url}"
command
/>\n"""
return cmd_cards
def write_reference_index(
reference_cards: Dict[Path, List[Dict[str, str]]],
fname: str,
path: Path,
rel_path: Path,
f: TextIO,
) -> None:
"""Write to the corresponding index.mdx file for a given folder, with the
appropriate nested menus and command cards.
Parameters
----------
reference_cards : Dict[Path, List[Dict[str, str]]]
Dictionary of command cards to be written to the index.mdx file.
fname : str
Name of the index.mdx file.
path : Path
Path to the folder to be written.
rel_path : Path
Relative path to the folder to be written.
f : TextIO
File to write to.
"""
f.write(f"# {fname}\n\n{reference_import}")
sub_folders = [sub for sub in path.glob("*") if sub.is_dir()]
menus = []
for folder in sub_folders:
menus.append(create_nested_menus_card(folder, "/".join(rel_path.parts)))
if menus:
f.write(f"### Menus\n{refrence_ul_element}\n{''.join(menus)}</ul>\n")
folder_cmd_cards: List[Dict[str, str]] = reference_cards.get(path, {}) # type: ignore
if folder_cmd_cards:
f.writelines(
[
f"\n\n### Commands\n{refrence_ul_element}\n",
create_cmd_cards(folder_cmd_cards),
"</ul>\n",
]
)
def main() -> bool:
"""Generate markdown files for OpenBB SDK Documentation."""
print("Loading trailmaps...")
trailmaps = get_trailmaps()
kwargs = {"encoding": "utf-8", "newline": "\n"}
print("Generating markdown files...")
reference_cards: Dict[Path, List[Dict[str, str]]] = {}
for file in CONTENT_PATH.glob("*"):
if file.is_file():
file.unlink()
else:
shutil.rmtree(file)
for trailmap in trailmaps:
try:
if trailmap.location_path[0] == "root":
trailmap.location_path[0] = ""
model_meta = (
get_function_meta(trailmap, "model") if trailmap.model else None
)
view_meta = get_function_meta(trailmap, "view") if trailmap.view else None
markdown = generate_markdown(model_meta, view_meta)
if trailmap.class_attr == "index":
trailmap.class_attr = "index_cmd"
filepath = (
CONTENT_PATH
/ "/".join(trailmap.location_path)
/ f"{trailmap.class_attr}.md"
)
func = trailmap.func_attrs.get(
"model", trailmap.func_attrs.get("view", None)
)
reference_cards.setdefault(filepath.parent, []).append(
dict(
title=trailmap.class_attr,
description=func.short_doc, # type: ignore
url="/".join(trailmap.location_path),
)
)
filepath.parent.mkdir(parents=True, exist_ok=True)
with open(filepath, "w", **kwargs) as f: # type: ignore
f.write(markdown)
except Exception as e:
print(
f"Error generating {trailmap.location_path} {trailmap.class_attr} - {e}"
)
return False
# Sort reference_cards
reference_cards = dict(sorted(reference_cards.items(), key=lambda item: item[0]))
with open(CONTENT_PATH / "index.mdx", "w", **kwargs) as f: # type: ignore
fname = "OpenBB SDK Reference"
rel_path = CONTENT_PATH.relative_to(CONTENT_PATH)
write_reference_index(reference_cards, fname, CONTENT_PATH, rel_path, f)
with open(CONTENT_PATH / "_category_.json", "w", **kwargs) as f: # type: ignore
"""Generate category json"""
f.write(json.dumps({"label": "Reference", "position": 4}, indent=2))
def gen_category_json(fname: str, path: Path, position: int = 1):
"""Generate category json"""
with open(path / "_category_.json", "w", **kwargs) as f: # type: ignore
f.write(
json.dumps({"label": fname.title(), "position": position}, indent=2)
)
with open(path / "index.mdx", "w", **kwargs) as f: # type: ignore
rel_path = path.relative_to(CONTENT_PATH)
write_reference_index(reference_cards, fname, path, rel_path, f)
def gen_category_recursive(nested_path: Path):
"""Generate category json recursively"""
position = 1
for folder in nested_path.iterdir():
if folder.is_dir():
gen_category_json(folder.name, folder, position)
gen_category_recursive(folder) # pylint: disable=cell-var-from-loop
position += 1
gen_category_recursive(CONTENT_PATH)
print("Markdown files generated, check the functions folder")
return True
def save_metadata() -> Dict[str, Dict[str, Union[str, List[str]]]]:
"""Save SEO metadata to json file."""
regex = re.compile(
r"---\ntitle: (.*)\ndescription: (.*)\nkeywords:(.*)\n---\n\nimport HeadTitle",
re.MULTILINE | re.DOTALL,
)
metadata = {}
for file in CONTENT_PATH.rglob("*/**/*.md"):
context = file.read_text(encoding="utf-8")
match = regex.search(context)
if match:
title, description, keywords = match.groups()
key = file.relative_to(CONTENT_PATH).as_posix().removesuffix(".md")
metadata[key.replace("/", ".")] = {
"title": title,
"description": description,
"keywords": [
keyword.strip() for keyword in keywords.split("\n- ") if keyword
],
}
filepath = website_path / "metadata/sdk_v3_seo_metadata.json"
with open(filepath, "w", encoding="utf-8", newline="\n") as f:
json.dump(metadata, f, indent=2)
return metadata
if __name__ == "__main__":
main()

435
website/generate_terminal_v3_markdown.py
View File

@@ -1,435 +0,0 @@
import json
import re
import shutil
import traceback
from datetime import datetime
from pathlib import Path
from textwrap import shorten
from typing import Dict, List, TextIO, Union
from openbb_terminal.core.session.current_system import set_system_variable
from openbb_terminal.core.session.current_user import get_current_user
from website.controller_doc_classes import (
ControllerDoc,
LoadControllersDoc,
console,
sub_names_full as subnames,
)
set_system_variable("TEST_MODE", True)
set_system_variable("LOG_COLLECT", False)
website_path = Path(__file__).parent.absolute()
CONTENT_PATH: Path = website_path / "content/terminal/reference"
EXAMPLES_META: Dict[str, Dict[str, Union[str, List[str]]]] = json.loads(
(website_path / "metadata/terminal_v3_examples.json").read_text()
)
SEO_META: Dict[str, Dict[str, Union[str, List[str]]]] = json.loads(
(website_path / "metadata/terminal_v3_seo_metadata.json").read_text()
)
reference_import = (
'import ReferenceCard from "@site/src/components/General/NewReferenceCard";\n\n'
)
refrence_ul_element = """<ul className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4 gap-4 -ml-6">"""
USER_PATH = (
f"{get_current_user().preferences.USER_DATA_DIRECTORY}",
"`USER_DATA_DIRECTORY`",
)
def existing_markdown_file_examples(
trailmap: str, cmd_dict: Dict[str, str]
) -> Dict[str, Union[str, List[str]]]:
"""Get existing markdown file examples
Parameters
----------
trailmap : str
The trailmap to the function
cmd_dict : Dict[str, str]
The dictionary of the command
Returns
-------
Dict[str, Union[str, List[str]]]
Updated dictionary of the command with examples and images keys
"""
trail = trailmap.split(".")
for sub in trail:
if sub in ["ba", "ta", "qa"]:
trail.remove(trailmap.split(".")[0])
trail.append(cmd_dict["cmd_name"])
return EXAMPLES_META.get(".".join(trail), {})
# pylint: disable=isinstance-second-argument-not-valid-type
def process_cmd_parsers(
ctrl: ControllerDoc, ctrl_trailmap: str
) -> List[Dict[str, str]]:
"""Process command parsers from ControllerDoc"""
commands = []
for cmd, parser in ctrl.cmd_parsers.items():
actions = []
for action in parser._actions: # pylint: disable=protected-access
if action.dest == "help":
continue
if (default := action.default) is not None:
if isinstance(default, list):
default = ", ".join([str(x) for x in default])
elif isinstance(default, datetime):
if "start" in action.dest:
default = "datetime.now() - timedelta(days=365)"
elif "end" in action.dest or "date" in action.dest:
default = "datetime.now()"
if (choices := action.choices) is not None:
# We do this so that range(0, N) objects are not converted to a list
if isinstance(choices, (dict, type({}.keys()), list)):
listdict: list = []
for choice in choices:
# We check for nested dicts
if isinstance(choice, (dict, type({}.keys()))):
listdict.append([f"{k}" for k in choice])
choices = listdict or [f"{x}" for x in choices]
choices = ", ".join(choices) if len(choices) > 0 else "None"
if action.dest == "file":
new_desc = "File in `EXPORTS` or `CUSTOM_IMPORTS` directories"
if (file_ext := re.compile(r"\.(\w{3,5})").findall(choices)) != []: # type: ignore
exts = set(file_ext)
ext_examples = "file_name." + ", file_name.".join(exts)
new_desc += f" (e.g: `{ext_examples}`)"
action.choices = [f"`{ext_examples}`"]
choices = new_desc
if (doc := action.help) is not None:
# We do this to fix multiline docstrings for the markdown
doc = " ".join(doc.split()).replace(*USER_PATH)
if (flags := action.option_strings) is not None:
flags = " ".join(flags)
actions.append(
{
"opt_name": action.dest if action.dest else "",
"flags": flags if flags else "",
"doc": doc if doc else "",
"default": f"{default}".replace(*USER_PATH),
"optional": not action.required,
"choices": choices,
}
)
if (desc := parser.description) is not None:
# We do this to fix multiline docstrings for the markdown
desc = " ".join(desc.split()).replace(*USER_PATH)
cmd_name = cmd.replace("call_", "")
param = {
"cmd_name": cmd_name,
"actions": actions,
"usage": " ".join(parser.format_usage().split()).replace("usage: ", ""),
"description": desc if desc else "",
}
default_desc = (desc or " .").split(".").pop(0).strip().replace(":", "")
header = f"title: {cmd_name}\ndescription: {default_desc}\nkeywords:\n- {ctrl_trailmap}\n- {cmd_name}"
key = f"{ctrl_trailmap}.{cmd_name}"
if cmd_meta := SEO_META.get(key, None):
keywords = "\n- ".join(cmd_meta["keywords"])
header = f"title: {cmd_meta['title']}\ndescription: {cmd_meta['description']}\nkeywords:\n- {keywords}"
title = key.split(".")
title[0] += " "
param[
"header"
] = f"""---\n{header}\n---\n
import HeadTitle from '@site/src/components/General/HeadTitle.tsx';
<HeadTitle title="{'/'.join(title)} - Reference | OpenBB Terminal Docs" />\n\n"""
commands.append(param)
return commands # type: ignore
def generate_markdown(
cmd_meta: Dict[str, str], examples: Dict[str, Union[str, List[str]]]
) -> str:
"""Generate markdown section
Parameters
----------
cmd_meta : Dict[str, str]
Command metadata
examples : Dict[str, Union[str, List[str]]]
Command examples
Returns
-------
str
Markdown string
"""
if not cmd_meta:
raise ValueError("No command metadata found")
# head meta https://docusaurus.io/docs/markdown-features/head-metadata
markdown = f"{cmd_meta['description']}\n\n"
markdown += f"### Usage\n\n```python wordwrap\n{cmd_meta['usage']}\n```\n\n"
markdown += "---\n\n## Parameters\n\n"
if cmd_meta["actions"]:
markdown += (
"| Name | Parameter | Description | Default | Optional | Choices |\n"
"| ---- | --------- | ----------- | ------- | -------- | ------- |\n"
)
for param in cmd_meta["actions"]:
if isinstance(param, dict):
markdown += (
f"| {param['opt_name']} | {param['flags']} | {param['doc']} "
f"| {param['default']} | {param['optional']} | {param['choices']} |\n"
)
else:
markdown += "This command has no parameters\n\n"
if examples.get("example", ""):
markdown += "\n\n---\n\n## Examples\n\n"
markdown += f"```python\n{examples['example']}\n```"
markdown += "\n"
if isinstance(examples.get("images", None), list):
for image in examples["images"]:
markdown += f"{image}\n\n"
markdown += "---\n"
return markdown.replace("<", "").replace(">", "")
def create_nested_menus_card(folder: Path, url: str) -> str:
sub_categories = [
sub.stem
for sub in folder.rglob("**/**/*.md*")
if sub.is_file() and sub.stem != "index"
]
categories = shorten(", ".join(sub_categories), width=116, placeholder="...")
url = f"/terminal/reference/{url}/{folder.name}".replace("//", "/")
index_card = f"""<ReferenceCard
title="{folder.name.title()}"
description="{categories}"
url="{url}"
/>\n"""
return index_card
def create_cmd_cards(cmd_text: List[Dict[str, str]]) -> str:
cmd_cards = ""
for cmd in cmd_text:
url = f"/terminal/reference/{cmd['url']}/{cmd['title']}".replace("//", "/")
description = shorten(cmd["description"], width=116, placeholder="...")
cmd_cards += f"""<ReferenceCard
title="{cmd["title"].title()}"
description="{description.replace('"', "'")}"
url="{url}"
command
/>\n"""
return cmd_cards
def write_reference_index(
reference_cards: Dict[Path, List[Dict[str, str]]],
fname: str,
path: Path,
rel_path: Path,
f: TextIO,
) -> None:
"""Write to the corresponding index.mdx file for a given folder, with the
appropriate nested menus and command cards.
Parameters
----------
reference_cards : Dict[Path, List[Dict[str, str]]]
Dictionary of command cards to be written to the index.md file.
fname : str
Name of the index.md file.
path : Path
Path to the folder to be written.
rel_path : Path
Relative path to the folder to be written.
f : TextIO
File to write to.
"""
f.write(f"# {fname}\n\n{reference_import}")
sub_folders = [sub for sub in path.glob("*") if sub.is_dir()]
menus = []
for folder in sub_folders:
menus.append(create_nested_menus_card(folder, "/".join(rel_path.parts)))
if menus:
f.write(f"### Menus\n{refrence_ul_element}\n{''.join(menus)}</ul>\n")
folder_cmd_cards: List[Dict[str, str]] = reference_cards.get(path, {}) # type: ignore
if folder_cmd_cards:
f.writelines(
[
f"\n\n### Commands\n{refrence_ul_element}\n",
create_cmd_cards(folder_cmd_cards),
"</ul>\n",
]
)
def main() -> bool:
"""Main function to generate markdown files"""
console.print(
"[bright_yellow]Loading Controllers... Please wait and ignore any errors, this is normal.[/bright_yellow]"
)
load_ctrls = LoadControllersDoc()
ctrls = load_ctrls.available_controllers()
wopen_kwargs = {"encoding": "utf-8", "newline": "\n"}
console.print(
"[bright_yellow]Generating markdown files... Don't ignore any errors now[/bright_yellow]"
)
reference_cards: Dict[Path, List[Dict[str, str]]] = {}
global_cmds = {"askobb": False, "hold": False, "about": False}
for file in CONTENT_PATH.glob("*"):
if file.is_file():
file.unlink()
else:
shutil.rmtree(file)
for ctrl_trailmap in ctrls:
try:
ctrl = load_ctrls.get_controller_doc(ctrl_trailmap)
ctrl_cmds = process_cmd_parsers(ctrl, ctrl_trailmap)
for cmd in ctrl_cmds:
trail = ctrl_trailmap.split(".")
if (created := global_cmds.get(cmd["cmd_name"], None)) is not None:
if created:
continue
global_cmds[cmd["cmd_name"]] = True
trail = []
examples = existing_markdown_file_examples(ctrl_trailmap, cmd)
markdown = cmd["header"] + generate_markdown(cmd, examples)
if cmd["cmd_name"] == "index":
cmd["cmd_name"] = "index_cmd"
filepath = CONTENT_PATH / f"{'/'.join(trail + [cmd['cmd_name']])}.md"
reference_cards.setdefault(filepath.parent, []).append(
dict(
title=cmd["cmd_name"],
description=cmd["description"],
url="/".join(trail),
)
)
filepath.parent.mkdir(parents=True, exist_ok=True)
with open(filepath, "w", **wopen_kwargs) as f: # type: ignore
f.write(markdown)
except Exception as e:
traceback.print_exc()
console.print(
f"[red]Failed to generate markdown for {ctrl_trailmap}: {e}[/red]"
)
return False
# Sort reference_cards
reference_cards = dict(sorted(reference_cards.items(), key=lambda item: item[0]))
# Generate root "_category_.json" file
with open(CONTENT_PATH / "_category_.json", "w", **wopen_kwargs) as f: # type: ignore
f.write(json.dumps({"label": "Reference", "position": 4}, indent=2))
# Generate root "index.md" file
with open(CONTENT_PATH / "index.mdx", "w", **wopen_kwargs) as f: # type: ignore
fname = "OpenBB Terminal Features"
rel_path = CONTENT_PATH.relative_to(CONTENT_PATH)
write_reference_index(reference_cards, fname, CONTENT_PATH, rel_path, f)
def gen_category_json(fname: str, path: Path, position: int = 1):
"""Generate category json"""
fname = subnames[fname.lower()] if fname.lower() in subnames else fname.title()
with open(path / "_category_.json", "w", **wopen_kwargs) as f: # type: ignore
f.write(json.dumps({"label": fname, "position": position}, indent=2))
with open(path / "index.mdx", "w", **wopen_kwargs) as f: # type: ignore
rel_path = path.relative_to(CONTENT_PATH)
write_reference_index(reference_cards, fname, path, rel_path, f)
def gen_category_recursive(nested_path: Path):
"""Generate category json recursively"""
position = 1
for folder in nested_path.iterdir():
if folder.is_dir():
gen_category_json(folder.name, folder, position)
gen_category_recursive(folder) # pylint: disable=cell-var-from-loop
position += 1
# Generate modules/sub category json and index files
gen_category_recursive(CONTENT_PATH)
console.print(
"[green]Markdown files generated, check the website/content/terminal/reference/ folder[/green]"
)
return True
def save_metadata() -> Dict[str, Dict[str, Union[str, List[str]]]]:
"""Save SEO metadata to json file."""
regex = re.compile(
r"---\ntitle: (.*)\ndescription: (.*)\nkeywords:(.*)\n---\n\nimport HeadTitle",
re.MULTILINE | re.DOTALL,
)
metadata = {}
for file in CONTENT_PATH.rglob("*/**/*.md"):
context = file.read_text(encoding="utf-8")
match = regex.search(context)
if match:
title, description, keywords = match.groups()
key = file.relative_to(CONTENT_PATH).as_posix().removesuffix(".md")
metadata[key.replace("/", ".")] = {
"title": title,
"description": description,
"keywords": [
keyword.strip() for keyword in keywords.split("\n- ") if keyword
],
}
filepath = website_path / "metadata/sdk_v3_seo_metadata.json"
with open(filepath, "w", encoding="utf-8", newline="\n") as f:
json.dump(metadata, f, indent=2)
return metadata
if __name__ == "__main__":
main()