jerry/buzz
1
0
Fork 0
mirror of https://github.com/block/buzz.git synced 2026-10-27 04:30:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(cli): paginate channel directory queries (#2181)

Browse Source 
Signed-off-by: npub1jh9wn95s0472h86ahapupaf7m6kx4v9sx2n0atj2hltcfer8k06s5n3pyf <95cae996907d7cab9f5dbf43c0f53edeac6ab0b032a6feae4abfd784e467b3f5@sprout-oss.stage.blox.sqprod.co>
Co-authored-by: npub1jh9wn95s0472h86ahapupaf7m6kx4v9sx2n0atj2hltcfer8k06s5n3pyf <95cae996907d7cab9f5dbf43c0f53edeac6ab0b032a6feae4abfd784e467b3f5@sprout-oss.stage.blox.sqprod.co>
This commit is contained in:
Tyler
2026-07-20 19:22:11 -04:00
committed by GitHub
parent d538111b51
commit 03fe19d603
5 changed files with 234 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

101
crates/buzz-cli/src/client.rs
View File

@@ -335,6 +335,29 @@ mod media_download_tests {
}
}
const QUERY_PAGE_SIZE: u32 = 500;
fn advance_query_cursor(
filter: &mut serde_json::Value,
page: &[serde_json::Value],
) -> Result<(), CliError> {
let last = page
.last()
.expect("a full query page always has a last event");
let created_at = last
.get("created_at")
.and_then(serde_json::Value::as_u64)
.ok_or_else(|| CliError::Other("query event missing created_at".into()))?;
let id = last
.get("id")
.and_then(serde_json::Value::as_str)
.filter(|id| id.len() == 64 && id.chars().all(|c| c.is_ascii_hexdigit()))
.ok_or_else(|| CliError::Other("query event missing valid id".into()))?;
filter["until"] = serde_json::json!(created_at);
filter["before_id"] = serde_json::json!(id);
Ok(())
}
pub struct BuzzClient {
http: reqwest::Client,
relay_url: String, // base URL, no trailing slash, e.g. "https://relay.buzz.place"
@@ -428,6 +451,54 @@ impl BuzzClient {
}
}
async fn query_pages(
&self,
mut filter: serde_json::Value,
limit: Option<u32>,
) -> Result<Vec<serde_json::Value>, CliError> {
let mut events = Vec::new();
while limit.is_none_or(|limit| events.len() < limit as usize) {
let page_limit = limit
.map(|limit| (limit as usize - events.len()).min(QUERY_PAGE_SIZE as usize))
.unwrap_or(QUERY_PAGE_SIZE as usize);
filter["limit"] = serde_json::json!(page_limit);
let raw = self.query(&filter).await?;
let page: Vec<serde_json::Value> = serde_json::from_str(&raw)
.map_err(|e| CliError::Other(format!("failed to parse query response: {e}")))?;
let done = page.len() < page_limit;
if !done {
advance_query_cursor(&mut filter, &page)?;
}
events.extend(page);
if done {
break;
}
}
Ok(events)
}
/// Query up to `limit` historical events, following the relay bridge's
/// composite `(until, before_id)` cursor across bounded result pages.
pub async fn query_paginated(
&self,
filter: serde_json::Value,
limit: u32,
) -> Result<Vec<serde_json::Value>, CliError> {
self.query_pages(filter, Some(limit)).await
}
/// Query every historical event matching a filter across bounded pages.
pub async fn query_all(
&self,
filter: serde_json::Value,
) -> Result<Vec<serde_json::Value>, CliError> {
self.query_pages(filter, None).await
}
/// Execute a one-shot query via the HTTP bridge.
/// `filter` is a Nostr filter object (will be wrapped in an array).
/// Returns the raw JSON response (array of events).
@@ -849,7 +920,35 @@ pub fn normalize_write_response(raw: &str) -> String {
#[cfg(test)]
mod tests {
use super::{create_response_with_id, extract_relay_response_field};
use super::{advance_query_cursor, create_response_with_id, extract_relay_response_field};
#[test]
fn query_cursor_uses_last_events_composite_sort_key() {
let mut filter = serde_json::json!({"kinds": [39000], "limit": 500});
let page = vec![
serde_json::json!({"id": "a".repeat(64), "created_at": 20}),
serde_json::json!({"id": "b".repeat(64), "created_at": 10}),
];
advance_query_cursor(&mut filter, &page).unwrap();
assert_eq!(filter["until"], serde_json::json!(10));
assert_eq!(filter["before_id"], serde_json::json!("b".repeat(64)));
}
#[test]
fn query_cursor_rejects_missing_or_malformed_sort_key() {
let mut filter = serde_json::json!({});
assert!(
advance_query_cursor(&mut filter, &[serde_json::json!({"id": "a".repeat(64)})])
.is_err()
);
assert!(advance_query_cursor(
&mut filter,
&[serde_json::json!({"id": "not-an-event-id", "created_at": 10})]
)
.is_err());
}
#[test]
fn extract_relay_response_field_reads_response_message_json() {

109
crates/buzz-cli/src/commands/channels.rs
View File

@@ -29,17 +29,16 @@ pub async fn cmd_list_channels(
format: &crate::OutputFormat,
) -> Result<(), CliError> {
let effective_limit = limit.unwrap_or(500);
let raw = if member == Some(true) {
let events = if member == Some(true) {
// Step 1: find channel IDs where we're a member (kind:39002)
let my_pk = client.keys().public_key().to_hex();
let member_filter = serde_json::json!({
"kinds": [39002],
"#p": [my_pk],
"limit": effective_limit
});
let member_resp = client.query(&member_filter).await?;
let member_events: Vec<serde_json::Value> =
serde_json::from_str(&member_resp).unwrap_or_default();
let member_events = client
.query_paginated(member_filter, effective_limit)
.await?;
let channel_ids: Vec<String> = member_events
.iter()
.map(extract_d_tag)
@@ -49,22 +48,21 @@ pub async fn cmd_list_channels(
println!("[]");
return Ok(());
}
// Step 2: fetch kind:39000 metadata for those channels
// Step 2: fetch kind:39000 metadata for those channels.
let metadata_filter = serde_json::json!({
"kinds": [39000],
"#d": channel_ids,
"limit": effective_limit
});
client.query(&metadata_filter).await?
client
.query_paginated(metadata_filter, effective_limit)
.await?
} else {
let filter = serde_json::json!({
"kinds": [39000],
"limit": effective_limit
});
client.query(&filter).await?
client.query_paginated(filter, effective_limit).await?
};
let events: Vec<serde_json::Value> = serde_json::from_str(&raw).unwrap_or_default();
let channels: Vec<serde_json::Value> = events
.iter()
.filter(|e| {
@@ -130,16 +128,8 @@ pub async fn cmd_search_channels(
let filter = serde_json::json!({
"kinds": [39000],
"limit": limit,
});
let raw = client.query(&filter).await?;
let events: serde_json::Value = serde_json::from_str(&raw)
.map_err(|e| CliError::Other(format!("failed to parse response: {e}")))?;
let Some(arr) = events.as_array() else {
println!("[]");
return Ok(());
};
let arr = client.query_paginated(filter, limit).await?;
let needle = query.to_ascii_lowercase();
let mut matches: Vec<ChannelSummary> = arr
@@ -337,14 +327,6 @@ pub async fn cmd_create_channel(
Ok(())
}
/// Relay-side cap on a single historical query (`buzz-relay/src/handlers/req.rs`).
/// Never request above this — a clamped response compared against a larger
/// requested size would falsely look like "no more pages."
const RELAY_MAX_HISTORICAL_LIMIT: u32 = 2000;
/// One page of a keyset-paginated kind:30177 scan.
const MANAGED_AGENT_PAGE_SIZE: u32 = RELAY_MAX_HISTORICAL_LIMIT;
/// A resolved live managed-agent instance backing a template persona slug.
#[derive(Debug, Clone, PartialEq, Eq)]
struct ResolvedAgent {
@@ -424,62 +406,29 @@ async fn scan_managed_agents_by_owner(
owner: &str,
slugs: &HashSet<&str>,
) -> Result<Vec<ResolvedAgent>, CliError> {
let filter = serde_json::json!({
"kinds": [KIND_MANAGED_AGENT],
"authors": [owner],
});
let events = client.query_all(filter).await?;
let mut found: Vec<ResolvedAgent> = Vec::new();
let mut seen_event_ids: HashSet<String> = HashSet::new();
let mut cursor: Option<(i64, String)> = None;
loop {
let mut filter = serde_json::json!({
"kinds": [KIND_MANAGED_AGENT],
"authors": [owner],
"limit": MANAGED_AGENT_PAGE_SIZE,
});
if let Some((until, ref before_id)) = cursor {
filter["until"] = serde_json::json!(until);
filter["before_id"] = serde_json::json!(before_id);
for event in &events {
let pubkey = extract_d_tag(event);
if pubkey.is_empty() {
continue;
}
let raw = client.query(&filter).await?;
let events: Vec<serde_json::Value> = serde_json::from_str(&raw).map_err(|e| {
CliError::Other(format!("failed to parse managed-agent query response: {e}"))
})?;
let page_len = events.len();
for event in &events {
let Some(event_id) = event.get("id").and_then(|v| v.as_str()) else {
continue;
};
if !seen_event_ids.insert(event_id.to_string()) {
continue;
}
let pubkey = extract_d_tag(event);
if pubkey.is_empty() {
continue;
}
let content: ManagedAgentContent = event
.get("content")
.and_then(|c| c.as_str())
.and_then(|s| serde_json::from_str(s).ok())
.unwrap_or(ManagedAgentContent { persona_id: None });
let Some(persona_id) = content.persona_id else {
continue;
};
if slugs.contains(persona_id.as_str()) {
found.push(ResolvedAgent { persona_id, pubkey });
}
}
if (page_len as u32) < MANAGED_AGENT_PAGE_SIZE {
break;
}
let Some(last) = events.last() else { break };
let Some(created_at) = last.get("created_at").and_then(|v| v.as_i64()) else {
break;
let content: ManagedAgentContent = event
.get("content")
.and_then(|c| c.as_str())
.and_then(|s| serde_json::from_str(s).ok())
.unwrap_or(ManagedAgentContent { persona_id: None });
let Some(persona_id) = content.persona_id else {
continue;
};
let Some(last_id) = last.get("id").and_then(|v| v.as_str()) else {
break;
};
cursor = Some((created_at, last_id.to_string()));
if slugs.contains(persona_id.as_str()) {
found.push(ResolvedAgent { persona_id, pubkey });
}
}
Ok(found)