From f1f6dad691649bb81dbe378ae6fa6e98be853efa Mon Sep 17 00:00:00 2001 From: Luqman Date: Wed, 12 Apr 2023 14:44:55 +0100 Subject: [PATCH] Separate Macos Build to different Jobs (#4694) * remove concurrency * comment int test * Seperate Macos Build * Int test check * crypto * reduce int test * Code cleanup * revert integration test * Edit the building steps * papermill * Fix papermill on both jobs --------- Co-authored-by: James Maslek --- .github/workflows/macos-build.yml | 175 +++++++++++++++++++++++++++--- 1 file changed, 160 insertions(+), 15 deletions(-) diff --git a/.github/workflows/macos-build.yml b/.github/workflows/macos-build.yml index 5c0d023602a..aaabc002f89 100644 --- a/.github/workflows/macos-build.yml +++ b/.github/workflows/macos-build.yml @@ -16,15 +16,9 @@ concurrency: cancel-in-progress: true jobs: - Build: - name: MacOS Build - runs-on: ${{ matrix.items.os }} - strategy: - matrix: - items: [ - {os: [self-hosted, macos, ARM64], name: "OpenBBM1.pkg"}, - {os: [self-hosted, macos, x64], name: "OpenBBIntel.pkg"} - ] + M1-MacOs-Build: + name: M1 MacOS Build + runs-on: [self-hosted, macos, ARM64] steps: # Checkout repository main branch. this allows for the commit hashes to line up - name: Checkout @@ -53,20 +47,24 @@ jobs: show-channel-urls: true channel-priority: flexible environment-file: build/conda/conda-3-9-env.yaml - activate-environment: build_env + activate-environment: obb auto-activate-base: false use-only-tar-bz2: true # Needed for caching some reason - name: Install Dependencies + shell: bash -l {0} run: | + conda info pip list - python -m pip install -r requirements-full.txt - pip install -e . + build/conda/cleanup_artifacts.sh + poetry install -E all -E installer + pip uninstall papermill -y + pip install git+https://github.com/nteract/papermill.git@main pip list - name: Build Bundle run: build/pyinstaller/build4mac.sh - shell: bash + shell: bash -l {0} - name: Creating Application Keychain env: @@ -101,7 +99,7 @@ jobs: security unlock-keychain -p $MACOS_INSTALLER_KEYCHAIN_PWD install.keychain build/pyinstaller/macOS/build-macos.sh OpenBBTerminal 0.0.1 mv build/pyinstaller/macOS/target/pkg/OpenBBTerminalM1.pkg OpenBBTerminal.pkg - shell: bash + shell: bash -l {0} - name: Deleting Previous Keychain run: | @@ -135,7 +133,7 @@ jobs: - name: Save Build Artifact PKG uses: actions/upload-artifact@v3 with: - name: ${{ matrix.items.name }} + name: OpenBBM1.pkg path: OpenBBTerminal.pkg - name: Get contents from PkgUtil @@ -163,3 +161,150 @@ jobs: rm -rf /Users/openbb/Desktop/OpenBB\ Terminal rm -rf ~/Desktop/OPENBB-exports rm -rf extract/ + +# Job to build the MacOS Intel version of the Terminal=================================== + Intel-MacOs-Build: + name: Intel MacOS Build + runs-on: [self-hosted, macos, x64] + steps: + # Checkout repository main branch. this allows for the commit hashes to line up + - name: Checkout + uses: actions/checkout@v3 + - name: Git Log + run: git log + # The following commands to clear previous PATHS and restore to defaults since we have to maintain the instance ourselves + - name: Clean Previous Path + run: | + export PATH="" + export PATH="/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" + echo $PATH + # Set up caching for conda env so that the workflow runs quickly after the first time + - name: Setup Conda Caching + uses: actions/cache@v3 + with: + path: ~/conda_pkgs_dir + key: conda-macos-3-9-${{ hashFiles('build/conda/conda-3-9-env.yaml') }} + # Set up miniconda using the environment yaml file within the repo + - name: Setup Miniconda + uses: conda-incubator/setup-miniconda@v2.2.0 + with: + miniconda-version: "latest" + auto-update-conda: true + channels: conda-forge,defaults + show-channel-urls: true + channel-priority: flexible + environment-file: build/conda/conda-3-9-env.yaml + activate-environment: obb + auto-activate-base: false + use-only-tar-bz2: true # Needed for caching some reason + + - name: Install Dependencies + shell: bash -l {0} + run: | + conda info + pip list + build/conda/cleanup_artifacts.sh + poetry install -E all -E installer + pip uninstall papermill -y + pip install git+https://github.com/nteract/papermill.git@main + pip list + + - name: Build Bundle + run: build/pyinstaller/build4mac.sh + shell: bash -l {0} + + - name: Creating Application Keychain + env: + MACOS_CERTIFICATE: ${{ secrets.MACOS_CERTIFICATE }} + MACOS_CERTIFICATE_PWD: ${{ secrets.MACOS_CERTIFICATE_PWD }} + MACOS_KEYCHAIN_PWD: ${{ secrets.MACOS_KEYCHAIN_PWD }} + MACOS_CODESIGN_IDENTITY: ${{ secrets.MACOS_CODESIGN_IDENTITY }} + run: | # when pushing to main, make to generate new cert, and utilize secrets to store new password, and identity + echo "Ensuring Keychain with same name does not exist" + rm -rf /Users/openbb/Library/Keychains/build.keychain-db + echo "Decoding certificate" + echo $MACOS_CERTIFICATE | base64 --decode > certificate.p12 + echo "Creating Keychain" + security create-keychain -p $MACOS_KEYCHAIN_PWD build.keychain + echo "Setting Default Keychain" + security default-keychain -s build.keychain + echo "Unlocking Keychain" + security unlock-keychain -p $MACOS_KEYCHAIN_PWD build.keychain + echo "Importing Keychain" + security import certificate.p12 -k build.keychain -P $MACOS_CERTIFICATE_PWD -T /usr/bin/codesign + echo "Setting Partition List" + security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k $MACOS_KEYCHAIN_PWD build.keychain + + - name: Create Signed Package + env: + APPLE_DEVELOPER_CERTIFICATE_ID: ${{ secrets.MACOS_INSTALLER_CODESIGN_IDENTITY }} + MACOS_CERTIFICATE_PWD: ${{ secrets.MACOS_CERTIFICATE_PWD }} + MACOS_KEYCHAIN_PWD: ${{ secrets.MACOS_KEYCHAIN_PWD }} + MACOS_INSTALLER_KEYCHAIN_PWD: ${{ secrets.MACOS_INSTALLER_KEYCHAIN_PWD }} + APPLE_SIGNING_IDENTITY: ${{ secrets.MACOS_CODESIGN_IDENTITY }} + run: | + security unlock-keychain -p $MACOS_INSTALLER_KEYCHAIN_PWD install.keychain + build/pyinstaller/macOS/build-macos.sh OpenBBTerminal 0.0.1 + mv build/pyinstaller/macOS/target/pkg/OpenBBTerminalM1.pkg OpenBBTerminal.pkg + shell: bash -l {0} + + - name: Deleting Previous Keychain + run: | + echo "Deleting Previous Keychain to Clean Instance" + rm -rf /Users/openbb/Library/Keychains/build.keychain-db + + - name: Clean up Build Artifacts + run: | + rm -rf build/terminal + rm -rf dist + rm -rf DMG + + - name: Notorize DMG + env: + NOTARIZE_APPLE_ID: ${{ secrets.NOTARIZE_APPLE_ID }} + NOTARIZE_APPLE_PWD: ${{ secrets.NOTARIZE_APPLE_PWD }} + NOTARIZE_APPLE_TEAM_ID: ${{ secrets.NOTARIZE_APPLE_TEAM_ID }} + run: | + xcrun notarytool submit OpenBBTerminal.pkg --apple-id "$NOTARIZE_APPLE_ID" --password "$NOTARIZE_APPLE_PWD" --team-id "$NOTARIZE_APPLE_TEAM_ID" --wait + + - name: Staple + run: | + xcrun stapler staple OpenBBTerminal.pkg + + - name: Clean up Build Artifacts + run: | + rm -rf build/terminal + rm -rf dist + rm -rf DMG + + - name: Save Build Artifact PKG + uses: actions/upload-artifact@v3 + with: + name: OpenBBIntel.pkg + path: OpenBBTerminal.pkg + + - name: Get contents from PkgUtil + run: | + pkgutil --expand-full OpenBBTerminal.pkg extract/ + rm -rf OpenBBTerminal.pkg + + - name: Run Integration Tests + run: | + extract/OpenBBTerminal.pkg/Payload/Applications/OpenBB\ Terminal/.OpenBB/OpenBBTerminal -t -s forecast cryptocurrency reports alternative economy futures econometrics dashboards portfolio stocks/test_stocks_options_screen.openbb stocks/test_stocks_options.openbb forex etf stocks/test_stocks_fa.openbb > result.txt + cat result.txt + grep "================================ integration test summary ================================" result.txt -A100 | tail --bytes=2000 > failed_tests.txt + + - name: Upload summary to Slack + uses: adrey/slack-file-upload-action@master + with: + token: ${{ secrets.SLACK_API_TOKEN }} + initial_comment: "MacOs Build Integration test summary" + title: "Intel MacOs Build Integration test summary" + path: failed_tests.txt + channel: ${{ secrets.SLACK_CHANNEL_ID }} + + - name: Remove OpenBB Folder + run: | + rm -rf /Users/openbb/Desktop/OpenBB\ Terminal + rm -rf ~/Desktop/OPENBB-exports + rm -rf extract/