mirror of
https://github.com/OpenBB-finance/OpenBB.git
synced 2026-10-27 04:31:13 +00:00
7f49354884
* fix cboe's cache lock so that concurrent callers in separate processes don't hang * fix test coverage and ty check * more test failure * more test failure * more Windows Python 3.10 test failure * Move lint path filter into a job so required checks always report lint-openbb-platform.yml used `on.pull_request.paths` to skip PRs that only touch api/extensions/providers/obbject_extensions. A workflow skipped by a trigger filter never reports its checks, so the required "General Code Linting" and "Markdown Linting" checks stayed pending forever on those PRs. Detect relevant changes in a `changes` job instead and gate the lint jobs with a job-level `if:`. Skipped jobs report success, so required checks resolve. The gate fails closed: if detection fails, the lint jobs still run. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: dlee-bq <325002271+dlee-bq@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
74 lines
2.2 KiB
Python
74 lines
2.2 KiB
Python
"""One OS process making cached requests, for the tests that share a cache file.
|
|
|
|
Run as ``cache_worker.py URL CACHE_FILE ROUNDS REQUESTS_PER_ROUND THREADS``. Each
|
|
thread runs bursts of concurrent requests on an event loop of its own, over a few
|
|
dozen URLs the other processes ask for too, and the last line printed is a JSON
|
|
summary of how many succeeded and what the failures were.
|
|
"""
|
|
|
|
import asyncio
|
|
import collections
|
|
import json
|
|
import logging
|
|
import sys
|
|
import threading
|
|
import traceback
|
|
from pathlib import Path
|
|
|
|
from openbb_cboe.utils import helpers
|
|
|
|
URL, CACHE_FILE, ROUNDS, PER_ROUND, THREADS = sys.argv[1:6]
|
|
URLS = 40
|
|
# Requests in flight per event loop. A test server can only queue so many
|
|
# connections at once (128 on macOS); this keeps the workers' total under it, so
|
|
# what the test measures is the cache and not the server's accept queue.
|
|
IN_FLIGHT = 2
|
|
|
|
logging.disable(logging.CRITICAL)
|
|
helpers.cache_path = lambda: CACHE_FILE
|
|
succeeded = 0
|
|
failures: collections.Counter = collections.Counter()
|
|
|
|
|
|
async def request(number, turn):
|
|
"""Request one URL, recording how it went."""
|
|
global succeeded # noqa: PLW0603
|
|
|
|
try:
|
|
async with turn:
|
|
await asyncio.wait_for(
|
|
helpers.get_cboe_data(f"{URL}/{number % URLS}.json"), timeout=30
|
|
)
|
|
|
|
succeeded += 1
|
|
except Exception as error: # noqa: BLE001
|
|
frames = traceback.extract_tb(error.__traceback__)[-4:]
|
|
where = " <- ".join(
|
|
f"{Path(frame.filename).name}:{frame.lineno} {frame.name}"
|
|
for frame in reversed(frames)
|
|
)
|
|
failures[f"{type(error).__name__}: {error} (at {where})"] += 1
|
|
|
|
|
|
def work(thread):
|
|
"""Run the bursts of one thread, each on a new event loop."""
|
|
|
|
async def burst(first):
|
|
turn = asyncio.Semaphore(IN_FLIGHT)
|
|
|
|
await asyncio.gather(*(request(first + n, turn) for n in range(int(PER_ROUND))))
|
|
|
|
for round_number in range(int(ROUNDS)):
|
|
asyncio.run(burst(round_number * int(PER_ROUND) + thread * 7))
|
|
|
|
|
|
threads = [threading.Thread(target=work, args=(n,)) for n in range(int(THREADS))]
|
|
|
|
for thread in threads:
|
|
thread.start()
|
|
|
|
for thread in threads:
|
|
thread.join()
|
|
|
|
sys.stdout.write(json.dumps({"succeeded": succeeded, "failures": failures}) + "\n")
|