diff --git a/build/pypi/openbb_platform/PUBLISH.md b/build/pypi/openbb_platform/PUBLISH.md index 05c008d3c39..b012474feb7 100644 --- a/build/pypi/openbb_platform/PUBLISH.md +++ b/build/pypi/openbb_platform/PUBLISH.md @@ -7,6 +7,41 @@ Publishing checklist: 1. Ensure all unit tests pass: `pytest openbb_platform -m "not integration"` 2. Ensure all integration tests pass: `pytest openbb_platform -m integration` 3. Run the publishing script: `python build/pypi/openbb_platform/publish.py` + + Consider using the `--dry-run` flag to check if everything is correct before publishing. + Also, it might be a good idea to run the script in batches to ensure that the packages are published correctly and the dependencies pick the correct versions. + Example, the `openbb-core` needs to pick the latest `openbb-provider` version and extension packages need to pick the latest `openbb-core` version. + Suggested batch order: + 1. Batch 1. Core + 1. `openbb-provider` + 2. `openbb-core` + 2. Batch 2. Extensions + 1. `openbb-charting` + 2. `openbb-crypto` + 3. `openbb-currency` + 4. ... + 3. Batch 3. Toolkits + 1. `openbb-quantitative` + 2. `openbb-technical` + 3. `openbb-econometrics` + 4. ... + 4. Batch 4. Providers + 1. `openbb-alpha-vantage` + 2. `openbb-benzinga` + 3. `openbb-biztoc` + 4. ... + 5. Batch 5. The meta-packages + 1. `openbb` + When publishing this package: + - Bump the dependency package versions + - Re-build the static assets that are bundled with the package + + > Note that, in order for packages to pick up the latest versions of dependencies, it might be necessary to clear the local cache of the dependencies: + > + > We can do that with `pip cache purge` and `poetry cache clear pypi --all` + > + > Also, sometimes there might be some delay in the PyPI API, so it might be necessary to wait a few minutes before publishing the next package. + 4. Update poetry files: `python build/pypi/openbb_platform/poetry_update.py` 5. Open a PR so that changes are reflected on the main branch diff --git a/build/pypi/openbb_platform/publish.py b/build/pypi/openbb_platform/publish.py index 31d05791de9..1d15b026d10 100644 --- a/build/pypi/openbb_platform/publish.py +++ b/build/pypi/openbb_platform/publish.py @@ -10,7 +10,8 @@ CORE_PACKAGES = ["platform/provider", "platform/core"] EXTENSION_PACKAGES = ["extensions", "providers"] CMD = [sys.executable, "-m", "poetry"] -CORE_BUMP_CMD = ["add", "openbb-core=latest"] +CORE_DEPENDENCIES_UPDATE_CMD = ["add", "openbb-provider=latest"] +EXTENSION_DEPENDENCIES_UPDATE_CMD = ["add", "openbb-core=latest"] VERSION_BUMP_CMD = ["version", "prerelease", "--next-phase"] PUBLISH_CMD = ["publish", "--build"] @@ -20,43 +21,81 @@ def parse_args(): description="Publish OpenBB Platform to PyPi with optional core or extensions flag." ) parser.add_argument( - "-c", "--core", action="store_true", help="Publish core packages only." + "-c", "--core", action="store_true", help="Publish core packages.", dest="core" ) parser.add_argument( "-e", "--extensions", action="store_true", - help="Publish extension packages only.", + help="Publish extension packages.", + dest="extensions", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Run the commands without actually publishing.", + default=False, + dest="dry_run", ) return parser.parse_args() -def run_cmds(directory: Path): - """Run the commands for publishing""" - print(f"Publishing: {directory.name}") # noqa: T201 - - # TODO: Uncomment the following lines depending on your needs - # subprocess.run(CMD + CORE_BUMP_CMD, cwd=directory, check=True) # noqa: S603 - subprocess.run(CMD + VERSION_BUMP_CMD, cwd=directory, check=True) # noqa: S603 - # subprocess.run(CMD + PUBLISH_CMD, cwd=directory, check=True) # noqa: S603 +def update_core_dependencies(path: Path): + """Update the core dependencies""" + subprocess.run( + CMD + CORE_DEPENDENCIES_UPDATE_CMD, # noqa: S603 + cwd=path.parent, + check=True, + ) -def publish(core=False, extensions=False): +def update_extension_dependencies(path: Path): + """Update the extension dependencies""" + subprocess.run( + CMD + EXTENSION_DEPENDENCIES_UPDATE_CMD, # noqa: S603 + cwd=path.parent, + check=True, + ) + + +def bump_version(path: Path): + """Bump the version of the package""" + subprocess.run(CMD + VERSION_BUMP_CMD, cwd=path.parent, check=True) # noqa: S603 + + +def publish(dry_run: bool = False, core: bool = False, extensions: bool = False): """Publish the Platform to PyPi with optional core or extensions.""" + package_paths = [] if core: - package_paths = CORE_PACKAGES print("Working with core packages...") # noqa: T201 + package_paths.extend(CORE_PACKAGES) if extensions: - package_paths = ( - package_paths.extend(EXTENSION_PACKAGES) if core else EXTENSION_PACKAGES - ) print("Working with extensions...") # noqa: T201 + package_paths.extend(EXTENSION_PACKAGES) + for sub_path in package_paths: + is_core = sub_path in CORE_PACKAGES + is_extension = sub_path in EXTENSION_PACKAGES + for path in PLATFORM_PATH.rglob(f"{sub_path}/**/pyproject.toml"): - run_cmds(path.parent) - # TODO: Uncomment the following lines depending on your needs - # # openbb - # run_cmds(PLATFORM_PATH) + try: + # Update dependencies + if is_core: + # if it's the provider package no need to update the dependencies + if "provider" in str(path.parent): + continue + update_core_dependencies(path) + if is_extension: + update_extension_dependencies(path) + # Bump version + bump_version(path) + # Publish (if not dry run) + if not dry_run: + subprocess.run( + CMD + PUBLISH_CMD, cwd=path.parent, check=True # noqa: S603 + ) + except Exception as e: + print(f"Error publishing {path.parent}:\n{e}") # noqa: T201 if __name__ == "__main__": @@ -70,4 +109,14 @@ if __name__ == "__main__": res = input(f"{msg}\n\nDo you want to continue? [y/N] ") if res.lower() == "y": - publish(core=args.core, extensions=args.extensions) + publish(dry_run=args.dry_run, core=args.core, extensions=args.extensions) + + openbb_package_msg = """ + In order to publish the `openbb` package you need to manually update the + versions in the `pyproject.toml` file. Follow the steps below: + 1. Bump version: `poetry version prerelease --next-phase` + 2. Re-build the static assets that are bundled with the package + 3. Publish: `poetry publish --build` + """ + + print(openbb_package_msg) # noqa: T201