jerry/OpenBB
1
0
Fork 0
mirror of https://github.com/OpenBB-finance/OpenBB.git synced 2026-10-27 12:40:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

[Docs] Fred Autogen fix (#3658)

Browse Source 
* Update controller_doc_classes.py

* test gh-actions

* Update gh-pages.yml

* Update generate_terminal_markdown.py

* cleanup generate_terminal_markdown.py, patch binance error preventing generation

Co-authored-by: andrewkenreich <andrew.kenreich@gmail.com>
Co-authored-by: James Maslek <jmaslek11@gmail.com>
This commit is contained in:
teh_coderer
2022-11-30 19:35:51 -06:00
committed by GitHub
parent 32bfba4874
commit 2494342d4c
4 changed files with 185 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/gh-pages.yml vendored
View File

@@ -13,10 +13,10 @@ jobs:
- name: Git checkout
uses: actions/checkout@v3 # actions/checkout v3.0.2
- name: Setup Python 3.10
- name: Setup Python 3.9
uses: actions/setup-python@v4
with:
python-version: "3.10"
python-version: "3.9"
architecture: x64
- name: Install Poetry

45
openbb_terminal/cryptocurrency/due_diligence/ccxt_model.py
View File

@@ -2,8 +2,10 @@
__docformat__ = "numpy"
from typing import Any, Dict, List
import ccxt
import pandas as pd
from openbb_terminal.cryptocurrency.dataframe_helpers import prettify_column_names
@@ -37,10 +39,45 @@ def get_binance_currencies() -> List[str]:
# Refactor this eventually to allow for any entered exchange -
# right now only works on default binace for "ob" and "trades"
exchange = ccxt.binance({"fetchCurrencies": True})
exchange.load_markets()
currencies = exchange.quoteCurrencies
return [c["code"] for c in currencies.values()]
# Commented out for now, since binance started blocking requests
# exchange = ccxt.binance({"fetchCurrencies": True})
# exchange.load_markets()
# currencies = exchange.quoteCurrencies
# return [c["code"] for c in currencies.values()]
return [
"AUD",
"BIDR",
"BKRW",
"BNB",
"BRL",
"BTC",
"BUSD",
"BVND",
"DAI",
"DOGE",
"DOT",
"ETH",
"EUR",
"GBP",
"IDRT",
"NGN",
"PAX",
"PLN",
"RUB",
"TRX",
"TRY",
"TUSD",
"UAH",
"USDC",
"USDP",
"USDS",
"USDT",
"UST",
"VAI",
"XRP",
"ZAR",
]
def get_orderbook(exchange: str, symbol: str, to_symbol: str) -> Dict[str, Any]:

3
website/controller_doc_classes.py
View File

@@ -13,8 +13,11 @@ import pandas as pd
from pandas.tseries.holiday import USFederalHolidayCalendar
import openbb_terminal
from openbb_terminal.decorators import disable_check_api
from openbb_terminal.parent_classes import BaseController, CryptoBaseController
disable_check_api()
DF_STOCK = pd.DataFrame.from_dict(
data={
pd.Timestamp("2020-11-30 00:00:00"): {

210
website/generate_terminal_markdown.py
View File

@@ -1,11 +1,13 @@
import json
import os
import re
import shutil
import traceback
from datetime import datetime
from pathlib import Path
from typing import Dict, List, Optional, Union
from typing import Dict, List, Union
from openbb_terminal.core.config.paths import USER_DATA_DIRECTORY
from openbb_terminal.rich_config import console
from website.controller_doc_classes import (
ControllerDoc,
@@ -14,28 +16,42 @@ from website.controller_doc_classes import (
)
website_path = Path(__file__).parent.absolute()
USER_PATH = (f"{USER_DATA_DIRECTORY}", "`USER_DATA_DIRECTORY`")
def existing_markdown_file_examples(
ctrl: ControllerDoc, cat: Dict[str, str]
) -> Dict[str, Optional[Union[str, List[str]]]]:
"""Get existing markdown file examples"""
trail = ctrl.trailmap.split(".")
trailmap: str, cmd_dict: Dict[str, str]
) -> Dict[str, Union[str, List[str]]]:
"""Get existing markdown file examples
Parameters
----------
trailmap : str
The trailmap to the function
cmd_dict : Dict[str, str]
The dictionary of the command
Returns
-------
Dict[str, Union[str, List[str]]]
Updated dictionary of the command with examples and images keys
"""
trail = trailmap.split(".")
for sub in trail:
if sub in ["ba", "ta", "qa"]:
trail.remove(ctrl.trailmap.split(".")[0])
trail.remove(trailmap.split(".")[0])
examples_path = (
f"old_content/terminal/{'/'.join(trail)}/{cat['cmd_name']}/_index.md"
f"old_content/terminal/{'/'.join(trail)}/{cmd_dict['cmd_name']}/_index.md"
)
examples_dict: Dict[str, Optional[Union[str, List[str]]]] = {}
examples_dict: Dict[str, Union[str, List[str]]] = {}
if os.path.exists(website_path / examples_path):
with open(website_path / examples_path, encoding="utf-8") as f:
content = f.read()
examples: Optional[str] = None
examples: str = ""
if "Example:" in content:
example_split = content.split("Example:")[1].split("```")
if example_split and len(example_split) > 1:
@@ -51,8 +67,8 @@ def existing_markdown_file_examples(
# pylint: disable=isinstance-second-argument-not-valid-type
def get_parser(ctrl: ControllerDoc) -> Dict[str, List[Dict[str, str]]]:
"""Get commands and parsers from ControllerDoc"""
def process_cmd_parsers(ctrl: ControllerDoc) -> List[Dict[str, str]]:
"""Process command parsers from ControllerDoc"""
commands = []
for cmd, parser in ctrl.cmd_parsers.items():
@@ -62,54 +78,59 @@ def get_parser(ctrl: ControllerDoc) -> Dict[str, List[Dict[str, str]]]:
if action.dest == "help":
continue
default = action.default
if default is not None:
if (default := action.default) is not None:
if isinstance(default, list):
default = ", ".join([str(x) for x in default])
elif isinstance(default, datetime):
if "start" in action.dest:
default = "datetime.now() - timedelta(days=365)"
elif "end" in action.dest or "date" in action.dest:
default = "datetime.now()"
choices = action.choices
if choices is not None:
if (choices := action.choices) is not None:
if isinstance(choices, list):
listdict = []
# We do this so that range(0, N) objects are not converted to a list
if isinstance(choices, (dict, type({}.keys()), list)):
listdict: list = []
for choice in choices:
# We check for nested dicts
if isinstance(choice, (dict, type({}.keys()))):
listdict.append([f"{k}" for k in choice])
if listdict:
choices = listdict
else:
choices = [f"{x}" for x in choices]
choices = ", ".join(choices) if len(choices) > 0 else None
choices = listdict or [f"{x}" for x in choices]
choices = ", ".join(choices) if len(choices) > 0 else "None"
elif isinstance(choices, (dict, type({}.keys()))):
choices = [f"{k}" for k in choices]
choices = ", ".join(choices) if len(choices) > 0 else None
if action.dest == "file":
new_desc = "File in `EXPORTS` or `CUSTOM_IMPORTS` directories"
doc = action.help
if doc is not None:
if (file_ext := re.compile(r"\.(\w{3,5})").findall(choices)) != []:
exts = set(file_ext)
ext_examples = "file_name." + ", file_name.".join(exts)
new_desc += f" (e.g: `{ext_examples}`)"
action.choices = [f"`{ext_examples}`"]
choices = new_desc
if (doc := action.help) is not None:
# We do this to fix multiline docstrings for the markdown
doc = " ".join(doc.split())
doc = " ".join(doc.split()).replace(*USER_PATH)
actions.append(
{
"opt_name": action.dest if action.dest else "",
"doc": doc if doc else "",
"default": default,
"default": f"{default}".replace(*USER_PATH),
"optional": not action.required,
"choices": choices,
}
)
desc = parser.description
if desc is not None:
if (desc := parser.description) is not None:
# We do this to fix multiline docstrings for the markdown
desc = " ".join(desc.split())
desc = " ".join(desc.split()).replace(*USER_PATH)
param = {
"cmd_name": cmd.replace("call_", ""),
@@ -119,37 +140,45 @@ def get_parser(ctrl: ControllerDoc) -> Dict[str, List[Dict[str, str]]]:
}
commands.append(param)
return {"category_name": ctrl.name, "cmds": commands}
return commands
def generate_markdown(cmd_meta: Dict[str, str], examples: Dict[str, str]):
"""Generate markdown string"""
def generate_markdown(
cmd_meta: Dict[str, str], examples: Dict[str, Union[str, List[str]]]
) -> str:
"""Generate markdown section
Parameters
----------
cmd_meta : Dict[str, str]
Command metadata
examples : Dict[str, Union[str, List[str]]]
Command examples
Returns
-------
str
Markdown string
"""
if not cmd_meta:
raise ValueError("No command metadata found")
markdown = f"""---
########### THIS FILE IS AUTO GENERATED - ANY CHANGES WILL BE VOID ###########
title: {cmd_meta["cmd_name"]}
description: OpenBB Terminal Function
---\n\n"""
markdown += generate_markdown_section(cmd_meta, examples)
return markdown
def generate_markdown_section(meta: Dict[str, str], examples: Dict[str, str]) -> str:
"""Generate markdown section"""
# head meta https://docusaurus.io/docs/markdown-features/head-metadata
markdown = f"# {meta['cmd_name']}\n\n{meta['description']}\n\n"
markdown += f"### Usage\n\n```python\n{meta['usage']}\n```\n\n"
markdown += f"# {cmd_meta['cmd_name']}\n\n{cmd_meta['description']}\n\n"
markdown += f"### Usage\n\n```python\n{cmd_meta['usage']}\n```\n\n"
markdown += "---\n\n## Parameters\n\n"
if meta["actions"]:
if cmd_meta["actions"]:
markdown += "| Name | Description | Default | Optional | Choices |\n"
markdown += "| ---- | ----------- | ------- | -------- | ------- |\n"
for param in meta["actions"]:
for param in cmd_meta["actions"]:
if isinstance(param, dict):
markdown += (
f"| {param['opt_name']} | {param['doc']} | {param['default']} "
@@ -158,13 +187,13 @@ def generate_markdown_section(meta: Dict[str, str], examples: Dict[str, str]) ->
else:
markdown += "This command has no parameters\n\n"
if examples.get("example", None):
if examples.get("example", ""):
markdown += "\n\n---\n\n## Examples\n\n"
markdown += f"```python\n{examples['example']}\n```"
markdown += "\n"
if examples.get("images", []):
if isinstance(examples.get("images", None), list):
for image in examples["images"]:
markdown += f"{image}\n\n"
@@ -173,7 +202,25 @@ def generate_markdown_section(meta: Dict[str, str], examples: Dict[str, str]) ->
def add_todict(d: dict, location_path: list, cmd_name: str, full_path: str) -> dict:
"""Adds the trailmap to the dictionary. This function creates the dictionary paths to the function."""
"""Adds the trailmap to the dictionary. This function creates the dictionary paths to the function.
Parameters
----------
d : dict
The dictionary to add the path to
location_path : list
The path to the function
cmd_name : str
The name of the function
full_path : str
The full path to the function
Returns
-------
dict
The updated dictionary
"""
if location_path[0] not in d:
d[location_path[0]] = {}
@@ -196,13 +243,13 @@ def main() -> bool:
load_ctrls = LoadControllersDoc()
ctrls = load_ctrls.available_controllers()
kwargs = {"encoding": "utf-8", "newline": "\n"}
wopen_kwargs = {"encoding": "utf-8", "newline": "\n"}
console.print(
"[bright_yellow]Generating markdown files... Don't ignore any errors now[/bright_yellow]"
)
content_path = website_path / "content/terminal/reference"
terminal_ref = {}
content_path: Path = website_path / "content/terminal/reference"
terminal_ref: Dict[str, dict] = {}
for file in content_path.glob("*"):
if file.is_file():
@@ -210,30 +257,32 @@ def main() -> bool:
else:
shutil.rmtree(file)
for ctrlstr in ctrls:
for ctrl_trailmap in ctrls:
try:
ctrl = load_ctrls.get_controller_doc(ctrlstr)
cmd_meta = get_parser(ctrl)
ctrl = load_ctrls.get_controller_doc(ctrl_trailmap)
ctrl_cmds = process_cmd_parsers(ctrl)
for cat in cmd_meta["cmds"]:
examples = existing_markdown_file_examples(ctrl, cat)
markdown = generate_markdown(cat, examples)
for cmd in ctrl_cmds:
examples = existing_markdown_file_examples(ctrl_trailmap, cmd)
markdown = generate_markdown(cmd, examples)
if cat["cmd_name"] == "index":
cat["cmd_name"] = "index_cmd"
if cmd["cmd_name"] == "index":
cmd["cmd_name"] = "index_cmd"
trail = ctrl.trailmap.split(".")
trail = ctrl_trailmap.split(".")
terminal_ref = add_todict(terminal_ref, trail, cat["cmd_name"], trail)
filepath = f"{str(content_path)}/{'/'.join(trail)}/{cat['cmd_name']}.md"
terminal_ref = add_todict(terminal_ref, trail, cmd["cmd_name"], trail)
filepath = content_path / f"{'/'.join(trail)}/{cmd['cmd_name']}.md"
os.makedirs(os.path.dirname(filepath), exist_ok=True)
with open(filepath, "w", **kwargs) as f:
filepath.parent.mkdir(parents=True, exist_ok=True)
with open(filepath, "w", **wopen_kwargs) as f: # type: ignore
f.write(markdown)
except Exception as e:
traceback.print_exc()
console.print(f"[red]Failed to generate markdown for {ctrlstr}: {e}[/red]")
console.print(
f"[red]Failed to generate markdown for {ctrl_trailmap}: {e}[/red]"
)
return False
terminal_ref = {
@@ -241,10 +290,12 @@ def main() -> bool:
for k, v in sorted(terminal_ref.items(), key=lambda item: item[0])
}
with open(content_path / "_category_.json", "w", **kwargs) as f:
# Generate root "_category_.json" file
with open(content_path / "_category_.json", "w", **wopen_kwargs) as f: # type: ignore
f.write(json.dumps({"label": "Terminal Reference", "position": 4}, indent=2))
with open(content_path / "index.md", "w", **kwargs) as f:
# Generate root "index.md" file
with open(content_path / "index.md", "w", **wopen_kwargs) as f: # type: ignore
f.write(
f"# OpenBB Terminal Features\n\n{generate_index_markdown('', terminal_ref, 2)}"
)
@@ -252,7 +303,7 @@ def main() -> bool:
def gen_category_json(fname: str, path: Path):
"""Generate category json"""
fname = subnames[fname.lower()] if fname.lower() in subnames else fname.title()
with open(path / "_category_.json", "w", **kwargs) as f:
with open(path / "_category_.json", "w", **wopen_kwargs) as f: # type: ignore
f.write(json.dumps({"label": fname}, indent=2))
def gen_category_recursive(nested_path: Path):
@@ -262,6 +313,7 @@ def main() -> bool:
gen_category_json(folder.name, folder)
gen_category_recursive(folder) # pylint: disable=cell-var-from-loop
# Generate modules/sub category json and index files
gen_category_recursive(content_path)
console.print(
@@ -271,7 +323,23 @@ def main() -> bool:
return True
def generate_index_markdown(markdown, d, level):
def generate_index_markdown(markdown: str, d: dict, level: int) -> str:
"""Generate index markdown
Parameters
----------
markdown : str
The markdown to add to
d : dict
The dictionary to recursively generate markdown from
level : int
The level of the markdown header
Returns
-------
str
Generated index file markdown string
"""
for key in d:
if isinstance(d[key], dict):
markdown += f"\n{'#' * level} {key}\n\n"