diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml deleted file mode 100644 index cdcb8ca708e..00000000000 --- a/.github/workflows/build-release.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Build & Release - -env: - OPENBB_ENABLE_QUICK_EXIT: true - OPENBB_LOG_COLLECT: false - OPENBB_USE_PROMPT_TOOLKIT: false - PIP_DEFAULT_TIMEOUT: 100 - PYTHONNOUSERSITE: 1 - -on: - push: - branches: - - release/v3/* - workflow_dispatch: - - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -permissions: - actions: write - -jobs: - trigger-windows-build: - runs-on: ubuntu-latest - steps: - - name: Trigger Windows Build - uses: aurelien-baudet/workflow-dispatch@v2 - with: - workflow: windows10_build.yml - token: ${{ secrets.GITHUB_TOKEN }} - wait-for-completion-timeout: 2h - - trigger-macos-build: - runs-on: ubuntu-latest - steps: - - name: Trigger Intel & M1 MacOs Build - uses: aurelien-baudet/workflow-dispatch@v2 - with: - workflow: macos-build.yml - token: ${{ secrets.GITHUB_TOKEN }} - wait-for-completion-timeout: 2h diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 0ebe358c4ca..6f4bf8634e6 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -51,17 +51,6 @@ jobs: - name: Install dependencies run: poetry install --no-interaction --no-root - - name: Install packages - run: poetry install --no-interaction -E forecast -E doc -E optimization - - - name: Generate SDK v3 Markdown - run: | - source .venv/bin/activate - pip uninstall Brotli -y - pytest tests/website --autodoc - poetry run python website/generate_sdk_v3_markdown.py - poetry run python website/generate_terminal_v3_markdown.py - - name: Generate Platform Markdown run: | source .venv/bin/activate diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index e71252bbeb7..edf6025d597 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -45,12 +45,10 @@ jobs: python-version: "3.9" architecture: x64 - - name: Get changed files in openbb_terminal and openbb_platform for PR + - name: Get changed files in openbb_platform for PR if: github.event_name == 'pull_request' run: | # "Checking PR diff" - echo "terminal_files=$(git diff --diff-filter=d --name-only origin/${{ github.base_ref }}...${{ github.head_ref }} | grep 'openbb_terminal/.*\.py$' | grep -v 'openbb_terminal/openbb/package' | grep -v 'integration' | grep -v 'tests' | xargs)" >> $GITHUB_ENV - echo $terminal_files echo "platform_files=$(git diff --diff-filter=d --name-only origin/${{ github.base_ref }}...${{ github.head_ref }} | grep 'openbb_platform/.*\.py$' | grep -v 'openbb_platform/openbb/package' | grep -v 'integration' | grep -v 'tests' | xargs)" >> $GITHUB_ENV echo $platform_files @@ -67,15 +65,6 @@ jobs: - run: black --diff --check . - run: codespell --ignore-words=.codespell.ignore --skip="$(tr '\n' ',' < .codespell.skip | sed 's/,$//')" --quiet-level=2 - run: ruff . - # TODO: Enable linters for openbb_terminal once we're ready - # - run: | - # # Run linters for openbb_terminal - # if [ -n "${{ env.terminal_files }}" ]; then - # mypy --ignore-missing-imports ${{ env.terminal_files }} - # pylint terminal.py ${{ env.terminal_files }} - # else - # echo "No Python files changed in openbb_terminal" - # fi - run: | # Run linters for openbb_platform if [ -n "${{ env.platform_files }}" ]; then diff --git a/.github/workflows/macos-build.yml b/.github/workflows/macos-build.yml deleted file mode 100644 index 6c7a2971b17..00000000000 --- a/.github/workflows/macos-build.yml +++ /dev/null @@ -1,280 +0,0 @@ -name: MacOS Build - -env: - OPENBB_LOG_COLLECT: false - OPENBB_USE_PROMPT_TOOLKIT: false - OPENBB_FILE_OVERWRITE: true - PIP_DEFAULT_TIMEOUT: 100 - PYTHONNOUSERSITE: 1 - -on: workflow_dispatch - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - M1-MacOs-Build: - name: M1 MacOS Build - runs-on: [self-hosted, macos, ARM64] - steps: - # Checkout repository main branch. this allows for the commit hashes to line up - - name: Checkout - uses: actions/checkout@v3 - - name: Git Log - run: git log - # The following commands to clear previous PATHS and restore to defaults since we have to maintain the instance ourselves - - name: Clean Previous Path - run: | - export PATH="" - export PATH="/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" - echo $PATH - # Set up caching for conda env so that the workflow runs quickly after the first time - - name: Setup Conda Caching - uses: actions/cache@v3 - with: - path: ~/conda_pkgs_dir - key: conda-macos-3-9-${{ hashFiles('build/conda/conda-3-9-env.yaml') }} - # Set up miniconda using the environment yaml file within the repo - - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v2.2.0 - with: - miniconda-version: "latest" - auto-update-conda: true - channels: conda-forge,defaults - show-channel-urls: true - channel-priority: flexible - environment-file: build/conda/conda-3-9-env.yaml - activate-environment: obb - auto-activate-base: false - use-only-tar-bz2: true # Needed for caching some reason - - - name: Install Dependencies - shell: bash -l {0} - run: | - conda info - pip list - build/conda/cleanup_artifacts.sh - poetry install -E all -E installer - pip uninstall papermill -y - pip install git+https://github.com/nteract/papermill.git@main - pip list - - - name: Build Bundle - run: build/pyinstaller/build4mac.sh - shell: bash -l {0} - - - name: Creating Application Keychain - env: - MACOS_CERTIFICATE: ${{ secrets.MACOS_CERTIFICATE }} - MACOS_CERTIFICATE_PWD: ${{ secrets.MACOS_CERTIFICATE_PWD }} - MACOS_KEYCHAIN_PWD: ${{ secrets.MACOS_KEYCHAIN_PWD }} - MACOS_CODESIGN_IDENTITY: ${{ secrets.MACOS_CODESIGN_IDENTITY }} - run: - | # when pushing to main, make to generate new cert, and utilize secrets to store new password, and identity - echo "Ensuring Keychain with same name does not exist" - rm -rf /Users/openbb/Library/Keychains/build.keychain-db - echo "Decoding certificate" - echo $MACOS_CERTIFICATE | base64 --decode > certificate.p12 - echo "Creating Keychain" - security create-keychain -p $MACOS_KEYCHAIN_PWD build.keychain - echo "Setting Default Keychain" - security default-keychain -s build.keychain - echo "Unlocking Keychain" - security unlock-keychain -p $MACOS_KEYCHAIN_PWD build.keychain - echo "Importing Keychain" - security import certificate.p12 -k build.keychain -P $MACOS_CERTIFICATE_PWD -T /usr/bin/codesign - echo "Setting Partition List" - security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k $MACOS_KEYCHAIN_PWD build.keychain - - - name: Create Signed Package - env: - APPLE_DEVELOPER_CERTIFICATE_ID: ${{ secrets.MACOS_INSTALLER_CODESIGN_IDENTITY }} - MACOS_CERTIFICATE_PWD: ${{ secrets.MACOS_CERTIFICATE_PWD }} - MACOS_KEYCHAIN_PWD: ${{ secrets.MACOS_KEYCHAIN_PWD }} - MACOS_INSTALLER_KEYCHAIN_PWD: ${{ secrets.MACOS_INSTALLER_KEYCHAIN_PWD }} - APPLE_SIGNING_IDENTITY: ${{ secrets.MACOS_CODESIGN_IDENTITY }} - run: | - security unlock-keychain -p $MACOS_INSTALLER_KEYCHAIN_PWD install.keychain - build/pyinstaller/macOS/build-macos.sh OpenBBTerminal 0.0.1 - mv build/pyinstaller/macOS/target/pkg/OpenBBTerminalM1.pkg OpenBBTerminal.pkg - shell: bash -l {0} - - - name: Deleting Previous Keychain - run: | - echo "Deleting Previous Keychain to Clean Instance" - rm -rf /Users/openbb/Library/Keychains/build.keychain-db - - - name: Clean up Build Artifacts - run: | - rm -rf build/terminal - rm -rf dist - rm -rf DMG - - - name: Notorize DMG - env: - NOTARIZE_APPLE_ID: ${{ secrets.NOTARIZE_APPLE_ID }} - NOTARIZE_APPLE_PWD: ${{ secrets.NOTARIZE_APPLE_PWD }} - NOTARIZE_APPLE_TEAM_ID: ${{ secrets.NOTARIZE_APPLE_TEAM_ID }} - run: | - xcrun notarytool submit OpenBBTerminal.pkg --apple-id "$NOTARIZE_APPLE_ID" --password "$NOTARIZE_APPLE_PWD" --team-id "$NOTARIZE_APPLE_TEAM_ID" --wait - - - name: Staple - run: | - xcrun stapler staple OpenBBTerminal.pkg - - - name: Clean up Build Artifacts - run: | - rm -rf build/terminal - rm -rf dist - rm -rf DMG - - - name: Save Build Artifact PKG - uses: actions/upload-artifact@v3 - with: - name: OpenBBM1.pkg - path: OpenBBTerminal.pkg - - - name: Get contents from PkgUtil - run: | - pkgutil --expand-full OpenBBTerminal.pkg extract/ - rm -rf OpenBBTerminal.pkg - - - name: Remove OpenBB Folder - run: | - rm -rf /Users/openbb/Desktop/OpenBB\ Terminal - rm -rf ~/Desktop/OPENBB-exports - rm -rf extract/ - - # Job to build the MacOS Intel version of the Terminal=================================== - Intel-MacOs-Build: - name: Intel MacOS Build - runs-on: [self-hosted, macos, x64] - steps: - # Checkout repository main branch. this allows for the commit hashes to line up - - name: Checkout - uses: actions/checkout@v3 - - name: Git Log - run: git log - # The following commands to clear previous PATHS and restore to defaults since we have to maintain the instance ourselves - - name: Clean Previous Path - run: | - export PATH="" - export PATH="/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" - echo $PATH - # Set up caching for conda env so that the workflow runs quickly after the first time - - name: Setup Conda Caching - uses: actions/cache@v3 - with: - path: ~/conda_pkgs_dir - key: conda-macos-3-9-${{ hashFiles('build/conda/conda-3-9-env.yaml') }} - # Set up miniconda using the environment yaml file within the repo - - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v2.2.0 - with: - miniconda-version: "latest" - auto-update-conda: true - channels: conda-forge,defaults - show-channel-urls: true - channel-priority: flexible - environment-file: build/conda/conda-3-9-env.yaml - activate-environment: obb - auto-activate-base: false - use-only-tar-bz2: true # Needed for caching some reason - - - name: Install Dependencies - shell: bash -l {0} - run: | - conda info - pip list - build/conda/cleanup_artifacts.sh - poetry install -E all -E installer - pip uninstall papermill -y - pip install git+https://github.com/nteract/papermill.git@main - pip list - - - name: Build Bundle - run: build/pyinstaller/build4mac.sh - shell: bash -l {0} - - - name: Creating Application Keychain - env: - MACOS_CERTIFICATE: ${{ secrets.MACOS_CERTIFICATE }} - MACOS_CERTIFICATE_PWD: ${{ secrets.MACOS_CERTIFICATE_PWD }} - MACOS_KEYCHAIN_PWD: ${{ secrets.MACOS_KEYCHAIN_PWD }} - MACOS_CODESIGN_IDENTITY: ${{ secrets.MACOS_CODESIGN_IDENTITY }} - run: - | # when pushing to main, make to generate new cert, and utilize secrets to store new password, and identity - echo "Ensuring Keychain with same name does not exist" - rm -rf /Users/openbb/Library/Keychains/build.keychain-db - echo "Decoding certificate" - echo $MACOS_CERTIFICATE | base64 --decode > certificate.p12 - echo "Creating Keychain" - security create-keychain -p $MACOS_KEYCHAIN_PWD build.keychain - echo "Setting Default Keychain" - security default-keychain -s build.keychain - echo "Unlocking Keychain" - security unlock-keychain -p $MACOS_KEYCHAIN_PWD build.keychain - echo "Importing Keychain" - security import certificate.p12 -k build.keychain -P $MACOS_CERTIFICATE_PWD -T /usr/bin/codesign - echo "Setting Partition List" - security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k $MACOS_KEYCHAIN_PWD build.keychain - - - name: Create Signed Package - env: - APPLE_DEVELOPER_CERTIFICATE_ID: ${{ secrets.MACOS_INSTALLER_CODESIGN_IDENTITY }} - MACOS_CERTIFICATE_PWD: ${{ secrets.MACOS_CERTIFICATE_PWD }} - MACOS_KEYCHAIN_PWD: ${{ secrets.MACOS_KEYCHAIN_PWD }} - MACOS_INSTALLER_KEYCHAIN_PWD: ${{ secrets.MACOS_INSTALLER_KEYCHAIN_PWD }} - APPLE_SIGNING_IDENTITY: ${{ secrets.MACOS_CODESIGN_IDENTITY }} - run: | - security unlock-keychain -p $MACOS_INSTALLER_KEYCHAIN_PWD install.keychain - build/pyinstaller/macOS/build-macos.sh OpenBBTerminal 0.0.1 - mv build/pyinstaller/macOS/target/pkg/OpenBBTerminalM1.pkg OpenBBTerminal.pkg - shell: bash -l {0} - - - name: Deleting Previous Keychain - run: | - echo "Deleting Previous Keychain to Clean Instance" - rm -rf /Users/openbb/Library/Keychains/build.keychain-db - - - name: Clean up Build Artifacts - run: | - rm -rf build/terminal - rm -rf dist - rm -rf DMG - - - name: Notorize DMG - env: - NOTARIZE_APPLE_ID: ${{ secrets.NOTARIZE_APPLE_ID }} - NOTARIZE_APPLE_PWD: ${{ secrets.NOTARIZE_APPLE_PWD }} - NOTARIZE_APPLE_TEAM_ID: ${{ secrets.NOTARIZE_APPLE_TEAM_ID }} - run: | - xcrun notarytool submit OpenBBTerminal.pkg --apple-id "$NOTARIZE_APPLE_ID" --password "$NOTARIZE_APPLE_PWD" --team-id "$NOTARIZE_APPLE_TEAM_ID" --wait - - - name: Staple - run: | - xcrun stapler staple OpenBBTerminal.pkg - - - name: Clean up Build Artifacts - run: | - rm -rf build/terminal - rm -rf dist - rm -rf DMG - - - name: Save Build Artifact PKG - uses: actions/upload-artifact@v3 - with: - name: OpenBBIntel.pkg - path: OpenBBTerminal.pkg - - - name: Get contents from PkgUtil - run: | - pkgutil --expand-full OpenBBTerminal.pkg extract/ - rm -rf OpenBBTerminal.pkg - - - name: Remove OpenBB Folder - run: | - rm -rf /Users/openbb/Desktop/OpenBB\ Terminal - rm -rf ~/Desktop/OPENBB-exports - rm -rf extract/ diff --git a/.github/workflows/macos-ml.yml b/.github/workflows/macos-ml.yml deleted file mode 100644 index f11f5502fc4..00000000000 --- a/.github/workflows/macos-ml.yml +++ /dev/null @@ -1,92 +0,0 @@ -name: Mac OS X Full Clean Build with ML -env: - OPENBB_ENABLE_QUICK_EXIT: true - OPENBB_LOG_COLLECT: false - OPENBB_USE_PROMPT_TOOLKIT: false - PIP_DEFAULT_TIMEOUT: 100 - OPENBB_FILE_OVERWRITE: true - PYTHONNOUSERSITE: 1 -on: - push: - branches: - - "main" - workflow_dispatch: - inputs: - comments: - required: true - description: "Test scenario tags" -jobs: - macos-conda: - name: MacOS-latest, Conda-latest, Python 3.9 - runs-on: macos-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - # Need to fetch more than the last commit so that setuptools-scm can - # create the correct version string. If the number of commits since - # the last release is greater than this, the version still be wrong. - # Increase if necessary. - fetch-depth: 100 - # The GitHub token is preserved by default but this job doesn't need - # to be able to push to GitHub. - persist-credentials: false - - - name: Setup caching for conda packages - uses: actions/cache@v2 - with: - path: ~/conda_pkgs_dir - key: conda-macos-3-9-${{ hashFiles('build/conda/conda-3-9-env.yaml') }} - - - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v2 - with: - miniconda-version: "latest" - auto-update-conda: true - channels: conda-forge,defaults - show-channel-urls: true - channel-priority: flexible - environment-file: build/conda/conda-3-9-env.yaml - activate-environment: testing - # Needed for caching - use-only-tar-bz2: true - - - name: Get pip cache dir - shell: bash -l {0} - id: pip-cache - run: | - echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT - - - name: pip cache - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: macos-3-9-pip-${{ hashFiles('**/requirements-full.txt') }} - restore-keys: | - macos-pip- - - - name: Uninstall Brotlipy - shell: bash -l {0} - run: | - conda remove brotlipy -y - - - name: Install dependencies (Bash) - shell: bash -l {0} - run: | - conda info - pip install --requirement requirements-full.txt - pip install docstring_parser - - - name: List installed packages (Bash) - shell: bash -l {0} - run: | - conda list - pip list - - - name: Run tests (Bash) - shell: bash -l {0} - run: pytest tests/ - - - name: Start Terminal and exit - shell: bash -l {0} - run: python terminal.py diff --git a/.github/workflows/manual-unit-test.yml b/.github/workflows/manual-unit-test.yml deleted file mode 100644 index 5c9be9d89e5..00000000000 --- a/.github/workflows/manual-unit-test.yml +++ /dev/null @@ -1,149 +0,0 @@ -name: Manually Triggered Unit Test - -env: - OPENBB_ENABLE_QUICK_EXIT: true - OPENBB_LOG_COLLECT: false - OPENBB_USE_PROMPT_TOOLKIT: false - OPENBB_FILE_OVERWRITE: true - OPENBB_ENABLE_CHECK_API: false - OPENBB_PREVIOUS_USE: true - OPENBB_USE_INTERACTIVE_DF: false - PIP_DEFAULT_TIMEOUT: 100 - -on: - workflow_dispatch: -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - tests-python: - name: Vanilla Python Tests - ${{ matrix.python-version }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: true - matrix: - python-version: ["3.8", "3.9", "3.10"] - os: [ubuntu-latest] - exclude: - - os: ubuntu-latest - python-version: "3.9" - steps: - - name: Checkout Code - uses: actions/checkout@v3 - - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - architecture: x64 - - - name: Install Poetry - uses: snok/install-poetry@v1 - with: - version: 1.4.0 - virtualenvs-create: true - virtualenvs-in-project: true - - - name: Setup sudo apt installs for ubuntu-latest - run: | - sudo apt-get update - sudo apt-get install -y \ - libgtk-3-dev \ - libwebkit2gtk-4.0-dev - - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@v3 - with: - path: .venv - key: venv-${{ runner.os }}-v1-${{ hashFiles('**/poetry.lock') }} - - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: poetry install --no-interaction -E optimization - - - name: List installed packages - shell: bash -l {0} - run: | - source $VENV - pip list - - - name: Run tests - env: - MPLBACKEND: Agg - run: | - source $VENV - pytest tests/ --optimization --autodoc -n auto --timeout=30 - - - name: Start Terminal and exit - run: | - source $VENV - python terminal.py - - full-test: - name: Full Test Suite - (${{ matrix.python-version }}, ${{ matrix.os }}) - runs-on: ${{ matrix.os }} - # Run only when a PR is merged or a push to release/* branch - strategy: - fail-fast: false - matrix: - python-version: ["3.8", "3.9", "3.10"] - os: [ubuntu-latest, macos-latest] - exclude: - - os: ubuntu-latest - python-version: "3.9" - steps: - - name: Checkout Code - uses: actions/checkout@v3 - - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - architecture: x64 - - - name: Install Poetry - uses: snok/install-poetry@v1 - with: - version: 1.4.0 - virtualenvs-create: true - virtualenvs-in-project: true - - - name: Setup sudo apt installs for ubuntu-latest - if: runner.os == 'Linux' - run: | - sudo apt-get update - sudo apt-get install -y \ - libgtk-3-dev \ - libwebkit2gtk-4.0-dev - - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@v3 - with: - path: .venv - key: venv-${{ runner.os }}-v1-${{ hashFiles('**/poetry.lock') }} - - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: | - poetry install --no-interaction -E optimization - - - name: List installed packages - shell: bash -l {0} - run: | - source $VENV - pip list - - - name: Start Terminal and exit - run: | - source $VENV - python terminal.py - - - name: Run tests - env: - MPLBACKEND: Agg - run: | - source $VENV - pytest tests/ --optimization --autodoc -n auto --timeout=30 diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index d1b8792779f..90d68cb9de5 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -16,35 +16,6 @@ permissions: actions: write jobs: - trigger-windows-build: - runs-on: ubuntu-latest - steps: - - name: Trigger Windows Build - uses: aurelien-baudet/workflow-dispatch@v2 - with: - workflow: windows10_build.yml - token: ${{ secrets.GITHUB_TOKEN }} - wait-for-completion-timeout: 3h - - trigger-macos-build: - runs-on: ubuntu-latest - steps: - - name: Trigger M1 & Intel MacOs Build - uses: aurelien-baudet/workflow-dispatch@v2 - with: - workflow: macos-build.yml - token: ${{ secrets.GITHUB_TOKEN }} - wait-for-completion-timeout: 2h - - trigger-docker-build: - runs-on: ubuntu-latest - steps: - - name: Trigger Docker Build - uses: aurelien-baudet/workflow-dispatch@v2 - with: - workflow: docker-build.yml - token: ${{ secrets.GITHUB_TOKEN }} - trigger-pypi-build: runs-on: ubuntu-latest steps: @@ -61,4 +32,4 @@ jobs: uses: aurelien-baudet/workflow-dispatch@v2 with: workflow: platform-api-integration-test.yml - token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/pypi-nightly.yml b/.github/workflows/pypi-nightly.yml index be5ec160acd..3a3f653df00 100644 --- a/.github/workflows/pypi-nightly.yml +++ b/.github/workflows/pypi-nightly.yml @@ -23,28 +23,6 @@ jobs: python -m pip install build - --user - - - name: OpenBBTerminal - Update version in pyproject.toml and Edit docs - run: | - sed -i 's/name = ".*"/name = "openbb-terminal-nightly"/' pyproject.toml - sed -i "3s/version = \"\(.*\)\"/version = \"\1.dev$(date +%Y%m%d%H%M)\"/" pyproject.toml - sed -i 's|pip install openbb-terminal|pip install openbb-terminal-nightly|g' ./website/pypi.md - - - name: OpenBBTerminal - Build a binary wheel and a source tarball - run: >- - python -m - build - --sdist - --wheel - --outdir dist/ - . - - - name: OpenBBTerminal - Publish distribution 📦 to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - packages-dir: dist/ - password: ${{ secrets.NIGHTLY_PYPI_API_TOKEN_OBB_TERMINAL_NIGHTLY }} - name: OpenBB Platform - Update version in pyproject.toml and Edit docs run: | diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index f8426e9b28b..32dfc58d5aa 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -70,187 +70,3 @@ jobs: run: | echo "check-changes=${{ steps.check-changes.outputs.check-changes }}" echo "check-platform-changes=${{ steps.check-platform-changes.outputs.check-platform-changes }}" - - base-test: - name: Base Tests - Ubuntu-latest - Python 3.9 - needs: [check-files-changed] - runs-on: ubuntu-latest - if: needs.check-files-changed.outputs.check-changes == 'true' && github.event.pull_request.base.ref == 'develop' && needs.check-files-changed.outputs.check-platform-changes == 'false' - steps: - - name: Checkout Code - uses: actions/checkout@v3 - - - name: Setup Python 3.9 - uses: actions/setup-python@v4 - with: - python-version: "3.9" - architecture: x64 - - - name: Install Poetry - uses: snok/install-poetry@v1 - with: - version: 1.5.1 - virtualenvs-create: true - virtualenvs-in-project: true - - - name: Setup sudo apt installs for ubuntu-latest - run: | - sudo apt-get update - sudo apt-get install -y \ - libgtk-3-dev \ - libwebkit2gtk-4.0-dev - - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@v3 - with: - path: .venv - key: venv-${{ runner.os }}-v1-${{ hashFiles('**/poetry.lock') }} - - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: poetry install --no-interaction -E optimization - - - name: Run tests and coverage test - env: - MPLBACKEND: Agg - run: | - source $VENV - pytest tests/ --optimization --cov --cov-fail-under=50 --autodoc -n auto --durations=10 --timeout=30 --cov-report=xml - - - name: Start Terminal and exit - run: | - source $VENV - python terminal.py - - tests-python: - name: Vanilla Python Tests - ${{ matrix.python-version }} - needs: [base-test] - runs-on: ${{ matrix.os }} - strategy: - fail-fast: true - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] - os: [ubuntu-latest] - exclude: - - os: ubuntu-latest - python-version: "3.9" - steps: - - name: Checkout Code - uses: actions/checkout@v3 - - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - architecture: x64 - - - name: Install Poetry - uses: snok/install-poetry@v1 - with: - version: 1.5.1 - virtualenvs-create: true - virtualenvs-in-project: true - - - name: Setup sudo apt installs for ubuntu-latest - run: | - sudo apt-get update - sudo apt-get install -y \ - libgtk-3-dev \ - libwebkit2gtk-4.0-dev - - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@v3 - with: - path: .venv - key: venv-${{ runner.os }}-v1-${{ hashFiles('**/poetry.lock') }} - - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: poetry install --no-interaction -E optimization - - - name: List installed packages - shell: bash -l {0} - run: | - source $VENV - pip list - - name: Run tests - env: - MPLBACKEND: Agg - run: | - source $VENV - pytest tests/ --optimization --autodoc -n auto --timeout=30 - - - name: Start Terminal and exit - run: | - source $VENV - python terminal.py - - full-test: - name: Full Test Suite - (${{ matrix.python-version }}, ${{ matrix.os }}) - runs-on: ${{ matrix.os }} - # Run only when a PR is merged or a push to release/* branch - if: github.event_name == 'push' || github.event.pull_request.merged == true - strategy: - fail-fast: false - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] - os: [ubuntu-latest, macos-latest] - exclude: - - os: ubuntu-latest - python-version: "3.9" - steps: - - name: Checkout Code - uses: actions/checkout@v3 - - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - architecture: x64 - - - name: Install Poetry - uses: snok/install-poetry@v1 - with: - version: 1.5.1 - virtualenvs-create: true - virtualenvs-in-project: true - - - name: Setup sudo apt installs for ubuntu-latest - if: runner.os == 'Linux' - run: | - sudo apt-get update - sudo apt-get install -y \ - libgtk-3-dev \ - libwebkit2gtk-4.0-dev - - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@v3 - with: - path: .venv - key: venv-${{ runner.os }}-v1-${{ hashFiles('**/poetry.lock') }} - - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: | - poetry install --no-interaction -E optimization - - - name: List installed packages - shell: bash -l {0} - run: | - source $VENV - pip list - - - name: Start Terminal and exit - run: | - source $VENV - python terminal.py - - - name: Run tests - env: - MPLBACKEND: Agg - run: | - source $VENV - pytest tests/ --optimization --autodoc -n auto --timeout=30 \ No newline at end of file diff --git a/.github/workflows/windows10_build.yml b/.github/workflows/windows10_build.yml deleted file mode 100644 index 005ada68fe0..00000000000 --- a/.github/workflows/windows10_build.yml +++ /dev/null @@ -1,131 +0,0 @@ -name: Windows10 Build - -env: - OPENBB_ENABLE_QUICK_EXIT: true - OPENBB_LOG_COLLECT: false - OPENBB_USE_PROMPT_TOOLKIT: false - PIP_DEFAULT_TIMEOUT: 100 - PYTHONNOUSERSITE: 1 - -on: workflow_dispatch - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - Windows-Build: - name: Windows10 Build ENV - runs-on: windows-latest - # Configuring ------------- - steps: - - name: Setup Windows Git Configuration # This is under the assumption that git is already installed on the system - run: | - Set-ItemProperty 'HKLM:\System\CurrentControlSet\Control\FileSystem' -Name 'LongPathsEnabled' -value 1 - git config --system core.longpaths true - - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up Python & Create OpenBB Environment - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - - name: Create and start virtual environment - run: | - python -m venv venv --upgrade-deps - source venv/Scripts/activate - python -m pip install setuptools==65.4.1 wheel - python -m pip install -r requirements-full.txt --no-cache-dir - python -m pip uninstall papermill -y - python -m pip install git+https://github.com/nteract/papermill.git@main - python -m PyInstaller build/pyinstaller/terminal.spec --clean - pwd - shell: bash - - - name: zip the file - run: | - Get-Location - Compress-Archive dist release.zip - shell: pwsh - - - name: Upload Artifact - uses: actions/upload-artifact@v3 - with: - name: OpenBB-Windows10-ENV - path: release.zip - - - Build-Exe: - name: Windows10 Build-Exe - runs-on: [self-hosted, Windows, x64] - needs: Windows-Build - # Configuring ------------- - steps: - - name: Setup Windows Git Configuration # This is under the assumption that git is already installed on the system - run: | - Set-ItemProperty 'HKLM:\System\CurrentControlSet\Control\FileSystem' -Name 'LongPathsEnabled' -value 1 - git config --system core.longpaths true - - - name: Checkout - uses: actions/checkout@v3 - - - name: Download Artifact - uses: actions/download-artifact@v3 - with: - name: OpenBB-Windows10-ENV - path: . - - - name: unzip artifact for deployment - run: Expand-Archive release.zip -DestinationPath . - shell: pwsh - - - name: Move Files into App Folder - run: cp -r .\dist\OpenBBTerminal\ .\build\nsis\app\ - # don't need this because log collection is turned off - # - name: Remove Log Data - # run: Remove-Item .\build\nsis\app\openbb_terminal\logs\ -Recurse - - name: Build Exe File - run: | - C:\'.\Program Files (x86)\'\nsis\makensis.exe .\build\nsis\setup.nsi - - # This should autosign the installer now - We really need to only do this when needed though. ON RELEASE - - name: Sign Installer - if: startsWith(github.ref, 'refs/heads/main') - run: | - & 'C:\Program Files (x86)\Windows Kits\10\bin\10.0.19041.0\x86\signtool.exe' sign /fd sha256 /tr http://ts.ssl.com /td sha256 /sha1 0bf95b74b3b8763434e1782df36052d42c5c0bef 'C:\Users\Administrator\actions-runner\_work\OpenBBTerminal\OpenBBTerminal\build\nsis\OpenBB Terminal Setup.exe' - - - name: Save Build Artifact - uses: actions/upload-artifact@v3 - with: - name: Windows EXE Artifact - path: '.\build\nsis\OpenBB Terminal Setup.exe' - - - name: Run Integration Tests - run: | - dist\OpenBBTerminal\OpenBBTerminal.exe -t -s forecast cryptocurrency reports alternative economy futures econometrics dashboards portfolio stocks/test_stocks_options_screen.openbb stocks/test_stocks_options.openbb forex etf stocks/test_stocks_fa.openbb | tee result.txt - Select-String "================================ Integration Test Summary ================================" result.txt -Context 0,100 | Select-Object -Last 2000 | Out-File summary.txt - Write-output `n >> summary.txt - - - name: Run Integration Tests Coverage Report - run: | - dist\OpenBBTerminal\OpenBBTerminal.exe -t --coverage | tee result.txt - Select-String "============================== Integration Coverage Summary ==============================" result.txt -Context 0,300 | Select-Object -Last 2000 >> summary.txt - - - name: Upload summary to Slack - uses: adrey/slack-file-upload-action@master - with: - token: ${{ secrets.SLACK_API_TOKEN }} - initial_comment: "Windows Build Integration test summary" - title: "Windows Build Integration test summary" - path: summary.txt - channel: ${{ secrets.SLACK_CHANNEL_ID }} - - # Cleaning ------------------------ - # Make sure to add Remove-Item C:\Users\Administrator\Desktop\OPENBB-exports -Recurse whenever integration tests get fixed - - name: Remove Previous Build - run: | - Remove-Item '.\build\nsis\OpenBB Terminal Setup.exe' -Recurse - Remove-Item .\build\nsis\app\ -Recurse - Remove-Item .\dist\ -Recurse diff --git a/.github/workflows/windows_ml.yml b/.github/workflows/windows_ml.yml deleted file mode 100644 index 3d5663a2d47..00000000000 --- a/.github/workflows/windows_ml.yml +++ /dev/null @@ -1,107 +0,0 @@ -name: Windows 10 Full Clean Build with ML -env: - OPENBB_ENABLE_QUICK_EXIT: true - OPENBB_LOG_COLLECT: false - OPENBB_USE_PROMPT_TOOLKIT: false - OPENBB_FILE_OVERWRITE: true - PIP_DEFAULT_TIMEOUT: 100 - PYTHONNOUSERSITE: 1 -on: - push: - branches: - - "main" - workflow_dispatch: - inputs: - comments: - required: true - description: "Test scenario tags" -jobs: - windows-conda: - name: Windows-latest, Conda-latest, Python 3.9 - runs-on: windows-latest - steps: - - name: Set git to use LF - if: runner.os == 'Windows' - run: | - git config --global core.autocrlf false - git config --global core.eol lf - - - name: Checkout - uses: actions/checkout@v2 - with: - # Need to fetch more than the last commit so that setuptools-scm can - # create the correct version string. If the number of commits since - # the last release is greater than this, the version still be wrong. - # Increase if necessary. - fetch-depth: 100 - # The GitHub token is preserved by default but this job doesn't need - # to be able to push to GitHub. - persist-credentials: false - - - name: Setup caching for conda packages - uses: actions/cache@v2 - with: - path: ~/conda_pkgs_dir - key: conda-windows-3-9-${{ hashFiles('build/conda/conda-3-9-env.yaml') }} - - - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v2 - with: - miniconda-version: "latest" - auto-update-conda: true - channels: conda-forge,defaults - show-channel-urls: true - channel-priority: flexible - environment-file: build/conda/conda-3-9-env.yaml - activate-environment: obb - # Needed for caching - use-only-tar-bz2: true - - - name: Update Temp permissions on windows - if: runner.os == 'Windows' - shell: cmd /C CALL {0} - run: >- - c:\windows\system32\icacls C:\Users\runneradmin\AppData\Local\Temp /grant "everyone":F /t - - - name: Get pip cache dir - shell: bash -l {0} - id: pip-cache - run: | - echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT - - - name: pip cache - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: windows-3-9-pip-${{ hashFiles('requirements-full.txt') }} - restore-keys: | - windows-pip- - - - name: Uninstall Brotlipy - shell: bash -l {0} - run: | - conda remove brotlipy -y - - - name: Install dependencies (Bash) - shell: bash -l {0} - run: | - conda info - build\conda\cleanup_artifacts.bat - pip install docstring_parser - poetry install -E all -E installer - - - name: List installed packages (Bash) - shell: bash -l {0} - run: | - conda list - pip list - - - name: Run tests (Bash) - env: - MPLBACKEND: Agg - shell: bash -l {0} - run: pytest tests/ -m "not linux" - - - name: Start Terminal and exit - shell: bash -l {0} - run: python terminal.py diff --git a/website/generate_sdk_v3_markdown.py b/website/generate_sdk_v3_markdown.py deleted file mode 100644 index 00f8d03790c..00000000000 --- a/website/generate_sdk_v3_markdown.py +++ /dev/null @@ -1,396 +0,0 @@ -import inspect -import json -import re -import shutil -from pathlib import Path -from textwrap import shorten -from typing import Any, Dict, List, Literal, TextIO, Union - -from docstring_parser import parse - -from openbb_terminal.core.sdk.trailmap import Trailmap, get_trailmaps -from openbb_terminal.core.session.current_system import set_system_variable - -set_system_variable("TEST_MODE", True) -set_system_variable("LOG_COLLECT", False) -website_path = Path(__file__).parent.absolute() -CONTENT_PATH = website_path / "content/sdk/reference" -SEO_META: Dict[str, Dict[str, Union[str, List[str]]]] = json.loads( - (website_path / "metadata/sdk_v3_seo_metadata.json").read_text() -) - -reference_import = ( - 'import ReferenceCard from "@site/src/components/General/NewReferenceCard";\n\n' -) - -refrence_ul_element = """\n") - - folder_cmd_cards: List[Dict[str, str]] = reference_cards.get(path, {}) # type: ignore - - if folder_cmd_cards: - f.writelines( - [ - f"\n\n### Commands\n{refrence_ul_element}\n", - create_cmd_cards(folder_cmd_cards), - "\n", - ] - ) - - -def main() -> bool: - """Generate markdown files for OpenBB SDK Documentation.""" - print("Loading trailmaps...") - trailmaps = get_trailmaps() - kwargs = {"encoding": "utf-8", "newline": "\n"} - - print("Generating markdown files...") - reference_cards: Dict[Path, List[Dict[str, str]]] = {} - - for file in CONTENT_PATH.glob("*"): - if file.is_file(): - file.unlink() - else: - shutil.rmtree(file) - for trailmap in trailmaps: - try: - if trailmap.location_path[0] == "root": - trailmap.location_path[0] = "" - - model_meta = ( - get_function_meta(trailmap, "model") if trailmap.model else None - ) - view_meta = get_function_meta(trailmap, "view") if trailmap.view else None - markdown = generate_markdown(model_meta, view_meta) - - if trailmap.class_attr == "index": - trailmap.class_attr = "index_cmd" - - filepath = ( - CONTENT_PATH - / "/".join(trailmap.location_path) - / f"{trailmap.class_attr}.md" - ) - func = trailmap.func_attrs.get( - "model", trailmap.func_attrs.get("view", None) - ) - - reference_cards.setdefault(filepath.parent, []).append( - dict( - title=trailmap.class_attr, - description=func.short_doc, # type: ignore - url="/".join(trailmap.location_path), - ) - ) - - filepath.parent.mkdir(parents=True, exist_ok=True) - with open(filepath, "w", **kwargs) as f: # type: ignore - f.write(markdown) - - except Exception as e: - print( - f"Error generating {trailmap.location_path} {trailmap.class_attr} - {e}" - ) - return False - - # Sort reference_cards - reference_cards = dict(sorted(reference_cards.items(), key=lambda item: item[0])) - - with open(CONTENT_PATH / "index.mdx", "w", **kwargs) as f: # type: ignore - fname = "OpenBB SDK Reference" - rel_path = CONTENT_PATH.relative_to(CONTENT_PATH) - write_reference_index(reference_cards, fname, CONTENT_PATH, rel_path, f) - - with open(CONTENT_PATH / "_category_.json", "w", **kwargs) as f: # type: ignore - """Generate category json""" - f.write(json.dumps({"label": "Reference", "position": 4}, indent=2)) - - def gen_category_json(fname: str, path: Path, position: int = 1): - """Generate category json""" - with open(path / "_category_.json", "w", **kwargs) as f: # type: ignore - f.write( - json.dumps({"label": fname.title(), "position": position}, indent=2) - ) - - with open(path / "index.mdx", "w", **kwargs) as f: # type: ignore - rel_path = path.relative_to(CONTENT_PATH) - write_reference_index(reference_cards, fname, path, rel_path, f) - - def gen_category_recursive(nested_path: Path): - """Generate category json recursively""" - position = 1 - for folder in nested_path.iterdir(): - if folder.is_dir(): - gen_category_json(folder.name, folder, position) - gen_category_recursive(folder) # pylint: disable=cell-var-from-loop - position += 1 - - gen_category_recursive(CONTENT_PATH) - print("Markdown files generated, check the functions folder") - - return True - - -def save_metadata() -> Dict[str, Dict[str, Union[str, List[str]]]]: - """Save SEO metadata to json file.""" - - regex = re.compile( - r"---\ntitle: (.*)\ndescription: (.*)\nkeywords:(.*)\n---\n\nimport HeadTitle", - re.MULTILINE | re.DOTALL, - ) - - metadata = {} - for file in CONTENT_PATH.rglob("*/**/*.md"): - context = file.read_text(encoding="utf-8") - match = regex.search(context) - if match: - title, description, keywords = match.groups() - key = file.relative_to(CONTENT_PATH).as_posix().removesuffix(".md") - metadata[key.replace("/", ".")] = { - "title": title, - "description": description, - "keywords": [ - keyword.strip() for keyword in keywords.split("\n- ") if keyword - ], - } - - filepath = website_path / "metadata/sdk_v3_seo_metadata.json" - with open(filepath, "w", encoding="utf-8", newline="\n") as f: - json.dump(metadata, f, indent=2) - - return metadata - - -if __name__ == "__main__": - main() diff --git a/website/generate_terminal_v3_markdown.py b/website/generate_terminal_v3_markdown.py deleted file mode 100644 index fe142db37c1..00000000000 --- a/website/generate_terminal_v3_markdown.py +++ /dev/null @@ -1,435 +0,0 @@ -import json -import re -import shutil -import traceback -from datetime import datetime -from pathlib import Path -from textwrap import shorten -from typing import Dict, List, TextIO, Union - -from openbb_terminal.core.session.current_system import set_system_variable -from openbb_terminal.core.session.current_user import get_current_user -from website.controller_doc_classes import ( - ControllerDoc, - LoadControllersDoc, - console, - sub_names_full as subnames, -) - -set_system_variable("TEST_MODE", True) -set_system_variable("LOG_COLLECT", False) -website_path = Path(__file__).parent.absolute() -CONTENT_PATH: Path = website_path / "content/terminal/reference" -EXAMPLES_META: Dict[str, Dict[str, Union[str, List[str]]]] = json.loads( - (website_path / "metadata/terminal_v3_examples.json").read_text() -) -SEO_META: Dict[str, Dict[str, Union[str, List[str]]]] = json.loads( - (website_path / "metadata/terminal_v3_seo_metadata.json").read_text() -) - -reference_import = ( - 'import ReferenceCard from "@site/src/components/General/NewReferenceCard";\n\n' -) - -refrence_ul_element = """\n") - - folder_cmd_cards: List[Dict[str, str]] = reference_cards.get(path, {}) # type: ignore - - if folder_cmd_cards: - f.writelines( - [ - f"\n\n### Commands\n{refrence_ul_element}\n", - create_cmd_cards(folder_cmd_cards), - "\n", - ] - ) - - -def main() -> bool: - """Main function to generate markdown files""" - console.print( - "[bright_yellow]Loading Controllers... Please wait and ignore any errors, this is normal.[/bright_yellow]" - ) - - load_ctrls = LoadControllersDoc() - ctrls = load_ctrls.available_controllers() - wopen_kwargs = {"encoding": "utf-8", "newline": "\n"} - - console.print( - "[bright_yellow]Generating markdown files... Don't ignore any errors now[/bright_yellow]" - ) - reference_cards: Dict[Path, List[Dict[str, str]]] = {} - global_cmds = {"askobb": False, "hold": False, "about": False} - - for file in CONTENT_PATH.glob("*"): - if file.is_file(): - file.unlink() - else: - shutil.rmtree(file) - - for ctrl_trailmap in ctrls: - try: - ctrl = load_ctrls.get_controller_doc(ctrl_trailmap) - ctrl_cmds = process_cmd_parsers(ctrl, ctrl_trailmap) - - for cmd in ctrl_cmds: - trail = ctrl_trailmap.split(".") - if (created := global_cmds.get(cmd["cmd_name"], None)) is not None: - if created: - continue - global_cmds[cmd["cmd_name"]] = True - trail = [] - - examples = existing_markdown_file_examples(ctrl_trailmap, cmd) - - markdown = cmd["header"] + generate_markdown(cmd, examples) - - if cmd["cmd_name"] == "index": - cmd["cmd_name"] = "index_cmd" - - filepath = CONTENT_PATH / f"{'/'.join(trail + [cmd['cmd_name']])}.md" - - reference_cards.setdefault(filepath.parent, []).append( - dict( - title=cmd["cmd_name"], - description=cmd["description"], - url="/".join(trail), - ) - ) - filepath.parent.mkdir(parents=True, exist_ok=True) - with open(filepath, "w", **wopen_kwargs) as f: # type: ignore - f.write(markdown) - - except Exception as e: - traceback.print_exc() - console.print( - f"[red]Failed to generate markdown for {ctrl_trailmap}: {e}[/red]" - ) - return False - - # Sort reference_cards - reference_cards = dict(sorted(reference_cards.items(), key=lambda item: item[0])) - - # Generate root "_category_.json" file - with open(CONTENT_PATH / "_category_.json", "w", **wopen_kwargs) as f: # type: ignore - f.write(json.dumps({"label": "Reference", "position": 4}, indent=2)) - - # Generate root "index.md" file - with open(CONTENT_PATH / "index.mdx", "w", **wopen_kwargs) as f: # type: ignore - fname = "OpenBB Terminal Features" - rel_path = CONTENT_PATH.relative_to(CONTENT_PATH) - write_reference_index(reference_cards, fname, CONTENT_PATH, rel_path, f) - - def gen_category_json(fname: str, path: Path, position: int = 1): - """Generate category json""" - fname = subnames[fname.lower()] if fname.lower() in subnames else fname.title() - with open(path / "_category_.json", "w", **wopen_kwargs) as f: # type: ignore - f.write(json.dumps({"label": fname, "position": position}, indent=2)) - - with open(path / "index.mdx", "w", **wopen_kwargs) as f: # type: ignore - rel_path = path.relative_to(CONTENT_PATH) - write_reference_index(reference_cards, fname, path, rel_path, f) - - def gen_category_recursive(nested_path: Path): - """Generate category json recursively""" - position = 1 - for folder in nested_path.iterdir(): - if folder.is_dir(): - gen_category_json(folder.name, folder, position) - gen_category_recursive(folder) # pylint: disable=cell-var-from-loop - position += 1 - - # Generate modules/sub category json and index files - gen_category_recursive(CONTENT_PATH) - - console.print( - "[green]Markdown files generated, check the website/content/terminal/reference/ folder[/green]" - ) - - return True - - -def save_metadata() -> Dict[str, Dict[str, Union[str, List[str]]]]: - """Save SEO metadata to json file.""" - - regex = re.compile( - r"---\ntitle: (.*)\ndescription: (.*)\nkeywords:(.*)\n---\n\nimport HeadTitle", - re.MULTILINE | re.DOTALL, - ) - - metadata = {} - for file in CONTENT_PATH.rglob("*/**/*.md"): - context = file.read_text(encoding="utf-8") - match = regex.search(context) - if match: - title, description, keywords = match.groups() - key = file.relative_to(CONTENT_PATH).as_posix().removesuffix(".md") - metadata[key.replace("/", ".")] = { - "title": title, - "description": description, - "keywords": [ - keyword.strip() for keyword in keywords.split("\n- ") if keyword - ], - } - - filepath = website_path / "metadata/sdk_v3_seo_metadata.json" - with open(filepath, "w", encoding="utf-8", newline="\n") as f: - json.dump(metadata, f, indent=2) - - return metadata - - -if __name__ == "__main__": - main()