diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index c4adb7fc63..b8ab2b2c12 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -579,6 +579,12 @@ multi-community mode every query filter includes `community_id`, so the shared `events` table is infrastructure, not a cross-community result space; the relay re-authorizes every candidate hit before returning it. +A database built from `migrations/0008_fresh_install_search_allowlist.sql` indexes +only an allowlist of kinds, and so does one converted by +`scripts/maintenance/nip_rs_search_allowlist.sql`. Such a database does not index +kind 30180 (custom emoji entries) until an operator runs the updated maintenance +script, which must happen before the emoji migration rollout. + **Key behaviors:** - `SearchService::new(pool)` wraps a `PgPool`; `search(&SearchQuery)` runs a parameterized FTS query against the `events.search_tsv` GIN index and returns diff --git a/crates/buzz-core/src/kind.rs b/crates/buzz-core/src/kind.rs index ae7a6221eb..2df27567da 100644 --- a/crates/buzz-core/src/kind.rs +++ b/crates/buzz-core/src/kind.rs @@ -119,6 +119,15 @@ pub const KIND_PUSH_LEASE: u32 = 30350; /// the owner's key to itself and contains the runnable identity/configuration /// plus exact public projection bindings. See `docs/nips/NIP-PMA.md`. pub const KIND_PRIVATE_MANAGED_AGENT: u32 = 30179; +/// Custom emoji entry (parameterized replaceable) — one event per emoji. +/// +/// User-owned, keyed by `(pubkey, kind, d_tag)` where `d` is the normalized +/// shortcode. `content` is the shortcode with `_` and `-` turned into spaces (the +/// search text); one `["emoji", shortcode, url]` tag carries the image. Unlike +/// the kind:30030 set, the catalog scales without any client loading it whole: +/// clients resolve names with `#d` filters and search with NIP-50 prefix mode. +/// See `docs/nips/NIP-CE.md`. +pub const KIND_EMOJI_ENTRY: u32 = 30180; /// Kinds whose stored events are readable only by their author. /// @@ -698,6 +707,7 @@ pub const ALL_KINDS: &[u32] = &[ KIND_FOLLOW_SET, KIND_BOOKMARK_SET, KIND_EMOJI_SET, + KIND_EMOJI_ENTRY, KIND_CHANNEL_METADATA, KIND_DELETION, KIND_REACTION, @@ -924,6 +934,7 @@ const _: () = assert!(is_parameterized_replaceable(KIND_TEAM)); // 30176 ∈ 300 const _: () = assert!(is_parameterized_replaceable(KIND_MANAGED_AGENT)); // 30177 ∈ 30000–39999 const _: () = assert!(is_parameterized_replaceable(KIND_TEAM_CATALOG)); // 30178 ∈ 30000–39999 const _: () = assert!(is_parameterized_replaceable(KIND_PRIVATE_MANAGED_AGENT)); // 30179 ∈ 30000–39999 +const _: () = assert!(is_parameterized_replaceable(KIND_EMOJI_ENTRY)); // 30180 ∈ 30000–39999 const _: () = assert!(is_parameterized_replaceable(KIND_WORKFLOW_DEF)); // 30620 ∈ 30000–39999 const _: () = assert!(is_parameterized_replaceable(KIND_EVENT_REMINDER)); // 30300 ∈ 30000–39999 const _: () = assert!(is_parameterized_replaceable(KIND_DM_VISIBILITY)); // 30622 ∈ 30000–39999 diff --git a/crates/buzz-relay/src/handlers/ingest.rs b/crates/buzz-relay/src/handlers/ingest.rs index f9b6f4b58d..eac849d48d 100644 --- a/crates/buzz-relay/src/handlers/ingest.rs +++ b/crates/buzz-relay/src/handlers/ingest.rs @@ -15,26 +15,26 @@ use buzz_core::kind::{ is_relay_admin_kind, KIND_AGENT_ENGRAM, KIND_AGENT_PROFILE, KIND_AGENT_TURN_METRIC, KIND_APPROVAL_DENY, KIND_APPROVAL_GRANT, KIND_AUTH, KIND_BOOKMARK_LIST, KIND_BOOKMARK_SET, KIND_CANVAS, KIND_CONTACT_LIST, KIND_DELETION, KIND_DM_ADD_MEMBER, KIND_DM_HIDE, KIND_DM_OPEN, - KIND_EMOJI_LIST, KIND_EMOJI_SET, KIND_EVENT_REMINDER, KIND_FOLLOW_SET, KIND_FORUM_COMMENT, - KIND_FORUM_POST, KIND_FORUM_VOTE, KIND_GIFT_WRAP, KIND_GIT_ISSUE, KIND_GIT_PATCH, - KIND_GIT_PR_UPDATE, KIND_GIT_PULL_REQUEST, KIND_GIT_REPO_ANNOUNCEMENT, KIND_GIT_REPO_STATE, - KIND_GIT_STATUS_CLOSED, KIND_GIT_STATUS_DRAFT, KIND_GIT_STATUS_MERGED, KIND_GIT_STATUS_OPEN, - KIND_HUDDLE_ENDED, KIND_HUDDLE_GUIDELINES, KIND_HUDDLE_PARTICIPANT_JOINED, - KIND_HUDDLE_PARTICIPANT_LEFT, KIND_HUDDLE_STARTED, KIND_IA_ARCHIVE_REQUEST, - KIND_IA_UNARCHIVE_REQUEST, KIND_LONG_FORM, KIND_MANAGED_AGENT, KIND_MEMBER_ADDED_NOTIFICATION, - KIND_MEMBER_REMOVED_NOTIFICATION, KIND_MODERATION_BAN, KIND_MODERATION_RESOLVE_REPORT, - KIND_MODERATION_TIMEOUT, KIND_MODERATION_UNBAN, KIND_MODERATION_UNTIMEOUT, KIND_MUTE_LIST, - KIND_NIP29_CREATE_GROUP, KIND_NIP29_DELETE_EVENT, KIND_NIP29_DELETE_GROUP, - KIND_NIP29_EDIT_METADATA, KIND_NIP29_JOIN_REQUEST, KIND_NIP29_LEAVE_REQUEST, - KIND_NIP29_PUT_USER, KIND_NIP29_REMOVE_USER, KIND_NIP43_LEAVE_REQUEST, - KIND_NIP65_RELAY_LIST_METADATA, KIND_PERSONA, KIND_PIN_LIST, KIND_PRESENCE_UPDATE, - KIND_PRIVATE_MANAGED_AGENT, KIND_PRODUCT_FEEDBACK, KIND_PROFILE, KIND_PROJECT, KIND_REACTION, - KIND_READ_STATE, KIND_REPORT, KIND_STREAM_MESSAGE, KIND_STREAM_MESSAGE_BOOKMARKED, - KIND_STREAM_MESSAGE_DIFF, KIND_STREAM_MESSAGE_EDIT, KIND_STREAM_MESSAGE_PINNED, - KIND_STREAM_MESSAGE_SCHEDULED, KIND_STREAM_MESSAGE_V2, KIND_STREAM_REMINDER, KIND_TEAM, - KIND_TEAM_CATALOG, KIND_TEXT_NOTE, KIND_USER_STATUS, KIND_WORKFLOW_DEF, KIND_WORKFLOW_TRIGGER, - RELAY_ADMIN_ADD_MEMBER, RELAY_ADMIN_CHANGE_ROLE, RELAY_ADMIN_REMOVE_MEMBER, - RELAY_ADMIN_SET_WORKSPACE_PROFILE, + KIND_EMOJI_ENTRY, KIND_EMOJI_LIST, KIND_EMOJI_SET, KIND_EVENT_REMINDER, KIND_FOLLOW_SET, + KIND_FORUM_COMMENT, KIND_FORUM_POST, KIND_FORUM_VOTE, KIND_GIFT_WRAP, KIND_GIT_ISSUE, + KIND_GIT_PATCH, KIND_GIT_PR_UPDATE, KIND_GIT_PULL_REQUEST, KIND_GIT_REPO_ANNOUNCEMENT, + KIND_GIT_REPO_STATE, KIND_GIT_STATUS_CLOSED, KIND_GIT_STATUS_DRAFT, KIND_GIT_STATUS_MERGED, + KIND_GIT_STATUS_OPEN, KIND_HUDDLE_ENDED, KIND_HUDDLE_GUIDELINES, + KIND_HUDDLE_PARTICIPANT_JOINED, KIND_HUDDLE_PARTICIPANT_LEFT, KIND_HUDDLE_STARTED, + KIND_IA_ARCHIVE_REQUEST, KIND_IA_UNARCHIVE_REQUEST, KIND_LONG_FORM, KIND_MANAGED_AGENT, + KIND_MEMBER_ADDED_NOTIFICATION, KIND_MEMBER_REMOVED_NOTIFICATION, KIND_MODERATION_BAN, + KIND_MODERATION_RESOLVE_REPORT, KIND_MODERATION_TIMEOUT, KIND_MODERATION_UNBAN, + KIND_MODERATION_UNTIMEOUT, KIND_MUTE_LIST, KIND_NIP29_CREATE_GROUP, KIND_NIP29_DELETE_EVENT, + KIND_NIP29_DELETE_GROUP, KIND_NIP29_EDIT_METADATA, KIND_NIP29_JOIN_REQUEST, + KIND_NIP29_LEAVE_REQUEST, KIND_NIP29_PUT_USER, KIND_NIP29_REMOVE_USER, + KIND_NIP43_LEAVE_REQUEST, KIND_NIP65_RELAY_LIST_METADATA, KIND_PERSONA, KIND_PIN_LIST, + KIND_PRESENCE_UPDATE, KIND_PRIVATE_MANAGED_AGENT, KIND_PRODUCT_FEEDBACK, KIND_PROFILE, + KIND_PROJECT, KIND_REACTION, KIND_READ_STATE, KIND_REPORT, KIND_STREAM_MESSAGE, + KIND_STREAM_MESSAGE_BOOKMARKED, KIND_STREAM_MESSAGE_DIFF, KIND_STREAM_MESSAGE_EDIT, + KIND_STREAM_MESSAGE_PINNED, KIND_STREAM_MESSAGE_SCHEDULED, KIND_STREAM_MESSAGE_V2, + KIND_STREAM_REMINDER, KIND_TEAM, KIND_TEAM_CATALOG, KIND_TEXT_NOTE, KIND_USER_STATUS, + KIND_WORKFLOW_DEF, KIND_WORKFLOW_TRIGGER, RELAY_ADMIN_ADD_MEMBER, RELAY_ADMIN_CHANGE_ROLE, + RELAY_ADMIN_REMOVE_MEMBER, RELAY_ADMIN_SET_WORKSPACE_PROFILE, }; use buzz_core::tenant::TenantContext; use buzz_core::verification::verify_event; @@ -157,6 +157,51 @@ fn validate_custom_emoji_tags(event: &Event) -> Result<(), IngestError> { Ok(()) } +/// Validate a kind:30180 custom emoji entry (see `docs/nips/NIP-CE.md`). +/// +/// Requires exactly one `d` tag, exactly one `emoji` tag, no `h` tag, a `d` equal +/// to the normalized shortcode in the `emoji` tag, an `http(s)` image URL, and +/// `content` equal to [`buzz_sdk::emoji_entry_search_text`] of that shortcode. +fn validate_emoji_entry(event: &Event) -> Result<(), IngestError> { + let reject = |message: &str| IngestError::Rejected(format!("invalid: {message}")); + let mut d_tags = Vec::new(); + let mut emoji_tags = Vec::new(); + for tag in event.tags.iter() { + match tag.as_slice().first().map(String::as_str) { + Some("h") => return Err(reject("emoji entry must not carry an h tag")), + Some("d") => d_tags.push(tag.as_slice()), + Some("emoji") => emoji_tags.push(tag.as_slice()), + _ => {} + } + } + let [d_tag] = d_tags.as_slice() else { + return Err(reject("emoji entry must have exactly one d tag")); + }; + let [emoji_tag] = emoji_tags.as_slice() else { + return Err(reject("emoji entry must have exactly one emoji tag")); + }; + let (Some(d), Some(shortcode), Some(url)) = (d_tag.get(1), emoji_tag.get(1), emoji_tag.get(2)) + else { + return Err(reject( + "emoji entry emoji tag must be [emoji, shortcode, url]", + )); + }; + let normalized = buzz_sdk::normalize_custom_emoji_shortcode(shortcode) + .map_err(|err| reject(&err.to_string()))?; + if *d != normalized || *shortcode != normalized { + return Err(reject( + "emoji entry d tag and emoji shortcode must equal the normalized shortcode", + )); + } + buzz_sdk::check_custom_emoji_url(url).map_err(|err| reject(&err.to_string()))?; + if event.content != buzz_sdk::emoji_entry_search_text(&normalized) { + return Err(reject( + "emoji entry content must equal the shortcode with _ and - replaced by spaces", + )); + } + Ok(()) +} + fn validate_reaction_emoji(event: &Event, emoji: &str) -> Result<(), IngestError> { let emoji_char_count = emoji.chars().count(); if emoji_char_count <= 64 { @@ -581,6 +626,8 @@ fn required_scope_for_kind(kind: u32, event: &Event) -> Result Ok(Scope::UsersWrite), KIND_DELETION | KIND_REACTION @@ -770,6 +817,8 @@ pub(crate) fn is_global_only_kind(kind: u32) -> bool { // keyed by (pubkey, kind[, d_tag]). A stray `h` tag must not channel-scope them. | KIND_EMOJI_SET | KIND_EMOJI_LIST + // Custom emoji entry (30180): same ownership shape, never channel-scoped. + | KIND_EMOJI_ENTRY // NIP-AE agent engrams are addressed by (pubkey_a, kind, d_tag); never channel-scoped. | KIND_AGENT_ENGRAM // NIP-ER event reminders are addressed by (pubkey, kind, d_tag); never channel-scoped. @@ -3401,6 +3450,9 @@ async fn ingest_event_inner( if kind_u32 == KIND_EMOJI_SET || kind_u32 == KIND_EMOJI_LIST { validate_custom_emoji_tags(&event)?; } + if kind_u32 == KIND_EMOJI_ENTRY { + validate_emoji_entry(&event)?; + } // Resolve the target reference, then use one DB transaction to upsert the // reaction row (dedup via ON CONFLICT) with reaction_event_id already set and @@ -3994,6 +4046,350 @@ mod postgres_tests { )); } + fn emoji_entry_event(tags: &[&[&str]], content: &str) -> nostr::Event { + EventBuilder::new(Kind::Custom(KIND_EMOJI_ENTRY as u16), content) + .tags( + tags.iter() + .map(|t| nostr::Tag::parse(t.iter().copied()).expect("tag")), + ) + .sign_with_keys(&nostr::Keys::generate()) + .expect("sign emoji entry") + } + + #[test] + fn emoji_entry_validation_table() { + const URL: &str = "https://example.com/p.gif"; + let ok_tags: &[&[&str]] = &[&["d", "party_parrot"], &["emoji", "party_parrot", URL]]; + assert!(validate_emoji_entry(&emoji_entry_event(ok_tags, "party parrot")).is_ok()); + let dashed_tags: &[&[&str]] = + &[&["d", "party-parrot-2"], &["emoji", "party-parrot-2", URL]]; + assert!(validate_emoji_entry(&emoji_entry_event(dashed_tags, "party parrot 2")).is_ok()); + let max_shortcode = "a".repeat(buzz_sdk::MAX_CUSTOM_EMOJI_SHORTCODE_LEN); + let max_tags: &[&[&str]] = &[&["d", &max_shortcode], &["emoji", &max_shortcode, URL]]; + assert!(validate_emoji_entry(&emoji_entry_event(max_tags, &max_shortcode)).is_ok()); + + let cases: Vec<(&str, Vec<&[&str]>, &str, &str)> = vec![ + ("no d", vec![&["emoji", "a", URL]], "", "exactly one d tag"), + ( + "two d", + vec![&["d", "a"], &["d", "a"], &["emoji", "a", URL]], + "", + "exactly one d tag", + ), + ("no emoji", vec![&["d", "a"]], "", "exactly one emoji tag"), + ( + "two emoji", + vec![&["d", "a"], &["emoji", "a", URL], &["emoji", "b", URL]], + "", + "exactly one emoji tag", + ), + ( + "h tag", + vec![&["d", "a"], &["emoji", "a", URL], &["h", "chan"]], + "", + "must not carry an h tag", + ), + ( + "emoji tag without url", + vec![&["d", "a"], &["emoji", "a"]], + "", + "[emoji, shortcode, url]", + ), + ( + "bad shortcode", + vec![&["d", "bad name"], &["emoji", "bad name", URL]], + "", + "may only contain", + ), + ( + "d differs from shortcode", + vec![&["d", "other"], &["emoji", "a", URL]], + "", + "must equal the normalized shortcode", + ), + ( + "uppercase shortcode", + vec![&["d", "a"], &["emoji", "A", URL]], + "", + "must equal the normalized shortcode", + ), + ( + "non-http url", + vec![&["d", "a"], &["emoji", "a", "ftp://example.com/a.gif"]], + "", + "http:// or https://", + ), + ( + "arbitrary content", + vec![&["d", "a"], &["emoji", "a", URL]], + "x", + "content must equal", + ), + ( + "empty content", + vec![&["d", "party_parrot"], &["emoji", "party_parrot", URL]], + "", + "content must equal", + ), + ( + "keyword-stuffed content", + vec![&["d", "party_parrot"], &["emoji", "party_parrot", URL]], + "party parrot cat dog", + "content must equal", + ), + ( + "content keeps underscores", + vec![&["d", "party_parrot"], &["emoji", "party_parrot", URL]], + "party_parrot", + "content must equal", + ), + ]; + for (name, tags, content, needle) in cases { + match validate_emoji_entry(&emoji_entry_event(&tags, content)) { + Err(IngestError::Rejected(msg)) => assert!( + msg.starts_with("invalid: ") && msg.contains(needle), + "{name}: unexpected rejection {msg:?}" + ), + other => panic!("{name}: expected rejection, got {other:?}"), + } + } + } + + #[test] + fn emoji_entry_is_a_global_users_write_kind() { + assert!(is_global_only_kind(KIND_EMOJI_ENTRY)); + assert!(!requires_h_channel_scope(KIND_EMOJI_ENTRY)); + let event = emoji_entry_event(&[], ""); + assert_eq!( + required_scope_for_kind(KIND_EMOJI_ENTRY, &event).ok(), + Some(Scope::UsersWrite) + ); + } + + async fn publish_emoji_entry( + state: &Arc, + tenant: &TenantContext, + keys: &nostr::Keys, + tags: &[&[&str]], + content: &str, + created_at: u64, + ) -> Result { + let event = EventBuilder::new(Kind::Custom(KIND_EMOJI_ENTRY as u16), content) + .tags( + tags.iter() + .map(|t| nostr::Tag::parse(t.iter().copied()).expect("tag")), + ) + .custom_created_at(nostr::Timestamp::from(created_at)) + .sign_with_keys(keys) + .expect("sign emoji entry"); + let auth = IngestAuth::Http { + pubkey: keys.public_key(), + scopes: vec![Scope::UsersWrite], + auth_method: HttpAuthMethod::Nip98, + }; + ingest_event(state, tenant, event, auth).await + } + + async fn publish_built_emoji_entry( + state: &Arc, + tenant: &TenantContext, + keys: &nostr::Keys, + shortcode: &str, + url: &str, + created_at: u64, + ) -> Result { + let event = buzz_sdk::build_emoji_entry(shortcode, url) + .expect("build emoji entry") + .custom_created_at(nostr::Timestamp::from(created_at)) + .sign_with_keys(keys) + .expect("sign emoji entry"); + let auth = IngestAuth::Http { + pubkey: keys.public_key(), + scopes: vec![Scope::UsersWrite], + auth_method: HttpAuthMethod::Nip98, + }; + ingest_event(state, tenant, event, auth).await + } + + async fn emoji_tenant( + state: &Arc, + keys: &[&nostr::Keys], + ) -> TenantContext { + let host = format!("emoji-entry-{}.test", Uuid::new_v4().simple()); + let community = state + .db + .ensure_configured_community(&host) + .await + .expect("community") + .id; + for k in keys { + state + .db + .ensure_user(community, k.public_key().as_bytes()) + .await + .expect("ensure user"); + } + TenantContext::resolved(community, &host) + } + + /// Production-path regression for kind:30180: invalid entries are refused by + /// `ingest_event` and never stored; a valid entry is stored; a second publish + /// under the same `d` leaves one live event (the newer one). + /// Mutations: drop the `validate_emoji_entry` call, the `KIND_EMOJI_ENTRY` + /// scope arm, or the replace path → RED. + #[tokio::test] + #[ignore = "requires Postgres"] + async fn emoji_entry_ingest_validates_and_replaces() { + let state = ingest_state().await; + let author = nostr::Keys::generate(); + let tenant = emoji_tenant(&state, &[&author]).await; + let now = chrono::Utc::now().timestamp() as u64; + const URL: &str = "https://example.com/p.gif"; + let url = URL; + let pool = sqlx::PgPool::connect(&crate::test_support::database_url()) + .await + .expect("connect test Postgres"); + let live_count = |d: &'static str| { + let pool = pool.clone(); + let community = *tenant.community().as_uuid(); + async move { + sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM events WHERE community_id = $1 AND kind = $2 \ + AND d_tag = $3 AND deleted_at IS NULL", + ) + .bind(community) + .bind(KIND_EMOJI_ENTRY as i32) + .bind(d) + .fetch_one(&pool) + .await + .expect("count") + } + }; + + let rejected: Vec<(&str, Vec<&[&str]>)> = vec![ + ( + "bad shortcode", + vec![&["d", "bad name"], &["emoji", "bad name", URL]], + ), + ( + "d != shortcode", + vec![&["d", "other"], &["emoji", "party_parrot", URL]], + ), + ( + "h tag", + vec![ + &["d", "party_parrot"], + &["emoji", "party_parrot", URL], + &["h", "11111111-1111-1111-1111-111111111111"], + ], + ), + ( + "non-http url", + vec![ + &["d", "party_parrot"], + &["emoji", "party_parrot", "javascript:alert(1)"], + ], + ), + ]; + for (name, tags) in rejected { + match publish_emoji_entry(&state, &tenant, &author, &tags, "party parrot", now).await { + Err(IngestError::Rejected(msg)) => { + assert!(msg.starts_with("invalid: "), "{name}: {msg}") + } + Err(other) => panic!("{name}: expected Rejected, got {other:?}"), + Ok(_) => panic!("{name}: must be rejected"), + } + } + assert_eq!( + live_count("party_parrot").await, + 0, + "rejected entries are not stored" + ); + + let first = + publish_built_emoji_entry(&state, &tenant, &author, "party_parrot", url, now - 10) + .await + .expect("first publish"); + assert!(first.accepted, "{}", first.message); + assert_eq!(live_count("party_parrot").await, 1); + + let url2 = "https://example.com/p2.gif"; + let second = publish_built_emoji_entry(&state, &tenant, &author, "Party_Parrot", url2, now) + .await + .expect("second publish"); + assert!(second.accepted, "{}", second.message); + assert_eq!( + live_count("party_parrot").await, + 1, + "same d keeps one live event" + ); + let surviving: Vec = sqlx::query_scalar( + "SELECT id FROM events WHERE community_id = $1 AND kind = $2 \ + AND d_tag = 'party_parrot' AND deleted_at IS NULL", + ) + .bind(tenant.community().as_uuid()) + .bind(KIND_EMOJI_ENTRY as i32) + .fetch_one(&pool) + .await + .expect("surviving id"); + assert_eq!(hex::encode(surviving), second.event_id); + } + + /// Prefix search over kind:30180 content finds "party parrot" for "parr", and + /// matches words at any position. + #[tokio::test] + #[ignore = "requires Postgres"] + async fn emoji_entry_prefix_search_finds_by_word_prefix() { + let state = ingest_state().await; + let author = nostr::Keys::generate(); + let tenant = emoji_tenant(&state, &[&author]).await; + let now = chrono::Utc::now().timestamp() as u64; + let url = "https://example.com/p.gif"; + for shortcode in ["party_parrot", "sad-parrot", "bufo_joy", "cloud_nine"] { + publish_built_emoji_entry(&state, &tenant, &author, shortcode, url, now) + .await + .unwrap_or_else(|e| panic!("publish {shortcode}: {e:?}")); + } + let pool = sqlx::PgPool::connect(&crate::test_support::database_url()) + .await + .expect("connect test Postgres"); + let indexed: i64 = sqlx::query_scalar( + "SELECT count(*) FROM events WHERE community_id = $1 AND kind = $2 \ + AND search_tsv IS NOT NULL", + ) + .bind(tenant.community().as_uuid()) + .bind(KIND_EMOJI_ENTRY as i32) + .fetch_one(&pool) + .await + .expect("count indexed entries"); + assert_eq!( + indexed, 4, + "events.search_tsv does not index kind 30180: this database uses the \ + positive FTS allowlist (migration 0008); run against the desired-state lane" + ); + let search = |q: &str| { + let service = buzz_search::SearchService::new(pool.clone()); + let query = buzz_search::SearchQuery { + community: tenant.community(), + q: q.to_owned(), + channel_scope: buzz_search::ChannelScope::Any, + kinds: Some(vec![KIND_EMOJI_ENTRY as i32]), + authors: None, + since: None, + until: None, + page: 1, + per_page: 50, + mode: buzz_search::SearchMode::Prefix, + sort: buzz_search::SearchSort::Relevance, + }; + async move { service.search(&query).await.expect("search").hits.len() } + }; + assert_eq!(search("parr").await, 2, "word-prefix finds both parrots"); + assert_eq!(search("party par").await, 1, "earlier words are exact"); + assert_eq!(search("joy").await, 1, "any word position matches"); + assert_eq!(search("zzz").await, 0); + } + /// A banned relay admin must be refused with the same wire prefix and /// transport status as every other durable-restriction refusal: /// `blocked:` and (via `bridge.rs`'s `AuthFailed` arm) HTTP 403 — never @@ -4487,6 +4883,7 @@ mod postgres_tests { KIND_BOOKMARK_SET, KIND_EMOJI_SET, KIND_EMOJI_LIST, + KIND_EMOJI_ENTRY, KIND_AGENT_ENGRAM, KIND_AGENT_PROFILE, KIND_PERSONA, diff --git a/crates/buzz-sdk/src/builders.rs b/crates/buzz-sdk/src/builders.rs index 10287491fd..3d0d1072d5 100644 --- a/crates/buzz-sdk/src/builders.rs +++ b/crates/buzz-sdk/src/builders.rs @@ -6,8 +6,8 @@ use buzz_core::{ kind::{ KIND_AGENT_OBSERVER_FRAME, KIND_APPROVAL_DENY, KIND_APPROVAL_GRANT, KIND_DELETION, - KIND_DM_ADD_MEMBER, KIND_DM_OPEN, KIND_EMOJI_SET, KIND_GIT_ISSUE, KIND_GIT_PATCH, - KIND_GIT_PR_UPDATE, KIND_GIT_PULL_REQUEST, KIND_GIT_REPO_ANNOUNCEMENT, + KIND_DM_ADD_MEMBER, KIND_DM_OPEN, KIND_EMOJI_ENTRY, KIND_EMOJI_SET, KIND_GIT_ISSUE, + KIND_GIT_PATCH, KIND_GIT_PR_UPDATE, KIND_GIT_PULL_REQUEST, KIND_GIT_REPO_ANNOUNCEMENT, KIND_GIT_STATUS_CLOSED, KIND_GIT_STATUS_DRAFT, KIND_GIT_STATUS_MERGED, KIND_GIT_STATUS_OPEN, KIND_IA_ARCHIVE_REQUEST, KIND_IA_UNARCHIVE_REQUEST, KIND_MODERATION_BAN, KIND_MODERATION_RESOLVE_REPORT, KIND_MODERATION_TIMEOUT, @@ -154,7 +154,8 @@ pub fn normalize_custom_emoji_shortcode(shortcode: &str) -> Result Result<(), SdkError> { +/// Validate a custom emoji image URL: non-empty, at most 2048 bytes, `http(s)://`. +pub fn check_custom_emoji_url(url: &str) -> Result<(), SdkError> { if url.is_empty() { return Err(SdkError::InvalidInput( "emoji image URL must not be empty".into(), @@ -574,6 +575,29 @@ pub fn build_custom_emoji_set(emojis: &[CustomEmoji]) -> Result String { + normalized_shortcode.replace(['_', '-'], " ") +} + +/// Build one custom emoji entry event (kind:30180). +/// +/// User-signed and parameterized-replaceable, keyed by `(pubkey, 30180, d)` +/// where `d` is the normalized shortcode. Publishing the same shortcode again +/// replaces the caller's earlier entry. `content` is the search text from +/// [`emoji_entry_search_text`]. +pub fn build_emoji_entry(shortcode: &str, url: &str) -> Result { + let shortcode = normalize_custom_emoji_shortcode(shortcode)?; + check_custom_emoji_url(url)?; + let tags = vec![tag(&["d", &shortcode])?, tag(&["emoji", &shortcode, url])?]; + Ok(EventBuilder::new( + Kind::Custom(KIND_EMOJI_ENTRY as u16), + emoji_entry_search_text(&shortcode), + ) + .tags(tags)) +} + /// Build a canvas update event (kind 40100). /// /// When `expected_revision` is set, an `["expected-revision", …]` tag is @@ -3414,6 +3438,51 @@ mod tests { assert!(has_tag(&ev, "emoji", "party")); } + #[test] + fn emoji_entry_happy_path() { + let ev = sign(build_emoji_entry(":Party_Parrot-2:", "https://example.com/p.gif").unwrap()); + assert_eq!(ev.kind.as_u16(), 30180); + assert_eq!(ev.content, "party parrot 2"); + assert!(has_tag(&ev, "d", "party_parrot-2")); + let emoji: Vec<_> = ev + .tags + .iter() + .filter(|t| t.as_slice().first().is_some_and(|k| k == "emoji")) + .map(|t| t.as_slice().to_vec()) + .collect(); + assert_eq!( + emoji, + vec![vec![ + "emoji".to_string(), + "party_parrot-2".to_string(), + "https://example.com/p.gif".to_string() + ]] + ); + assert!(!ev + .tags + .iter() + .any(|t| t.as_slice().first().is_some_and(|k| k == "h"))); + } + + #[test] + fn emoji_entry_rejects_bad_input() { + for (shortcode, url) in [ + ("", "https://example.com/p.gif"), + ("bad name", "https://example.com/p.gif"), + ("bad!", "https://example.com/p.gif"), + ("ok", ""), + ("ok", "ftp://example.com/p.gif"), + ("ok", "javascript:alert(1)"), + ] { + assert!( + build_emoji_entry(shortcode, url).is_err(), + "{shortcode:?} {url:?} must be rejected" + ); + } + let long = "a".repeat(MAX_CUSTOM_EMOJI_SHORTCODE_LEN + 1); + assert!(build_emoji_entry(&long, "https://example.com/p.gif").is_err()); + } + #[test] fn remove_reaction_happy_path() { let eid = event_id(); diff --git a/docs/nips/NIP-CE.md b/docs/nips/NIP-CE.md new file mode 100644 index 0000000000..f124a93171 --- /dev/null +++ b/docs/nips/NIP-CE.md @@ -0,0 +1,117 @@ +NIP-CE +====== + +Custom Emoji Entries +-------------------- + +`draft` `optional` `relay` + +This NIP defines `kind:30180`, an addressable event that holds one custom emoji. A community can hold many thousands of emoji without any client loading the whole catalog. + +[NIP-30](30.md) and [NIP-51](51.md) put many emoji in one `kind:30030` set. A set is one event, so every client must read and parse the whole set. This NIP uses one event for each emoji. Clients ask the relay for the names they need. + +## Event + +`kind:30180` is an addressable event keyed by `(pubkey, kind, d)` as defined in [NIP-01](01.md). + +```jsonc +{ + "kind": 30180, + "content": "party parrot", + "tags": [ + ["d", "party_parrot"], + ["emoji", "party_parrot", "https://example.com/media/9f2c.gif"] + ] +} +``` + +### Tags + +An entry MUST have exactly one `d` tag and exactly one `emoji` tag. + +The `emoji` tag has the form `["emoji", , ]`, as in [NIP-30](30.md). + +The shortcode MUST be 1 to 64 bytes of ASCII letters, digits, hyphens (`-`), and underscores (`_`). It MUST be lowercase. The `d` value MUST equal the shortcode in the `emoji` tag. + +The URL MUST start with `http://` or `https://`. It MUST NOT be longer than 2048 bytes. + +An entry MUST NOT have an `h` tag. Entries are never scoped to a channel. + +### Content + +`content` is the search text. It is the shortcode with each `_` and `-` replaced by a space. For `party_parrot-2` the content is `party parrot 2`. + +Full-text search splits on spaces. The replacement lets a search for `parrot` find `party_parrot`. + +`content` MUST equal this search text exactly. A relay MUST reject an entry with any other `content`, including empty content. Because the content comes from the shortcode, it is never longer than 64 bytes. + +The rule keeps the full-text index in step with the shortcode. An author cannot add keywords that the shortcode does not contain. + +## Replace semantics + +A new entry with the same `(pubkey, d)` replaces the old entry. The relay keeps the entry with the newest `created_at`. When two entries have the same `created_at`, the relay keeps the entry with the lowest event ID. This is the standard [NIP-01](01.md) rule. + +A member can only replace their own entries. Another member who publishes the same `d` creates a second, separate entry. The relay stores both. + +When clients see several entries for one `d`, they MUST pick the entry with the newest `created_at` across all authors. Clients SHOULD report the conflict to the user. Clients MUST apply the same tie-break as the relay. + +To remove an entry, the author sends a [NIP-09](09.md) deletion request. + +## Reading + +### Exact lookup + +To resolve names, send a filter with `kinds` and `#d`: + +```jsonc +{ "kinds": [30180], "#d": ["party_parrot", "sad_parrot"] } +``` + +A filter SHOULD NOT carry more than 50 names. A client that knows the author SHOULD add `authors`. Without `authors`, a relay can scan every entry of the kind to answer. + +### Prefix search + +To search, use [NIP-50](50.md) search over HTTP `POST /query` with prefix mode: + +```jsonc +{ "kinds": [30180], "search": "parr", "search_mode": "prefix", "limit": 50 } +``` + +Prefix mode treats each word except the last as a whole word. It treats the last word as a prefix. A search for `party par` finds `party parrot`. A search for `joy` finds `bufo joy`. + +The relay returns at most 500 hits for each request. Clients SHOULD ask for 50 or fewer. + +The relay reads `search_mode` on `POST /query` only. A WebSocket `REQ` with `search` uses full-word matching. + +Search needs the relay to index `content` of `kind:30180` in its full-text index. A relay that indexes only an allowlist of kinds MUST add `30180` to the allowlist before clients rely on search. Exact lookup does not depend on the full-text index. + +In Buzz, a database that migration `0008_fresh_install_search_allowlist.sql` built, or that an operator converted with `scripts/maintenance/nip_rs_search_allowlist.sql`, indexes only an allowlist of kinds. Such a database does not index `kind:30180` until an operator runs the updated maintenance script. The operator MUST run it before the emoji migration rollout. Without it, entries are stored and exact lookup works, but prefix search returns no results. + +## Relay behavior + +A relay that supports this NIP: + +- MUST accept `kind:30180` from any community member with the same permission as `kind:30030`. +- MUST reject an invalid entry with an `invalid:` message. +- MUST store entries as global events, not channel events. + +## Relationship to kind 30030 + +This NIP does not change `kind:30030`. Sets with `d` equal to `buzz:custom-emoji` keep their meaning. A client MAY read both. A client that reads both SHOULD prefer the newest event when a shortcode appears in both. + +The `emoji` tags on messages and reactions do not change. A message carries the `emoji` tags for the emoji it uses, so history renders without any lookup. + +Clients that do not know `kind:30180` ignore these events. + +## Limits summary + +| Item | Limit | +|------|-------| +| Shortcode | 1 to 64 bytes, `[A-Za-z0-9_-]`, lowercase in `d` and `emoji` | +| URL | 2048 bytes, `http://` or `https://` | +| Content | 256 bytes | +| `d` tags | exactly 1 | +| `emoji` tags | exactly 1 | +| `h` tags | 0 | +| Names per `#d` filter | 50 (SHOULD) | +| Search hits per request | 500 (relay maximum) | diff --git a/scripts/maintenance/nip_rs_search_allowlist.sql b/scripts/maintenance/nip_rs_search_allowlist.sql index 1dc0a667fc..38a87b1261 100644 --- a/scripts/maintenance/nip_rs_search_allowlist.sql +++ b/scripts/maintenance/nip_rs_search_allowlist.sql @@ -5,12 +5,26 @@ -- replacement heap/TOAST/index files plus WAL. ALTER TABLE takes ACCESS -- EXCLUSIVE, so event reads and writes block until this transaction commits. -- Consider combining this with the planned partition repack/reclaim operation. +-- +-- The allowlist includes kind 30180 (custom emoji entries, docs/nips/NIP-CE.md). +-- A database built from migrations/0008_fresh_install_search_allowlist.sql, or +-- converted by an earlier version of this script, does not index kind 30180: +-- those rows get a NULL search_tsv and emoji prefix search returns nothing. +-- Run this script on such a database BEFORE the emoji migration rollout. A +-- database that still has the original deny-list expression already indexes +-- kind 30180 and does not need it for emoji. +-- +-- Validate after COMMIT; the expected result is the single row `t`: +-- SELECT pg_get_expr(d.adbin, d.adrelid) LIKE '%30180%' +-- FROM pg_attrdef d +-- JOIN pg_attribute a ON a.attrelid = d.adrelid AND a.attnum = d.adnum +-- WHERE d.adrelid = 'events'::regclass AND a.attname = 'search_tsv'; BEGIN; SET LOCAL lock_timeout = '5s'; ALTER TABLE events DROP COLUMN search_tsv; ALTER TABLE events ADD COLUMN search_tsv TSVECTOR GENERATED ALWAYS AS ( - CASE WHEN kind IN (0, 9, 40002, 45001, 45003) + CASE WHEN kind IN (0, 9, 30180, 40002, 45001, 45003) THEN to_tsvector('simple', content) ELSE NULL::tsvector END