jerry/buzz
1
0
Fork 0
mirror of https://github.com/block/buzz.git synced 2026-10-27 04:30:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

NIP-33: d-tag keyed replacement for parameterized replaceable events (#246)

Browse Source
This commit is contained in:
tlongwell-block
2026-04-06 21:15:01 -04:00
committed by GitHub
parent d5bd78377e
commit 67706c93be
11 changed files with 897 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
crates/sprout-core/src/kind.rs
View File

@@ -21,6 +21,10 @@ pub const KIND_REACTION: u32 = 7;
pub const KIND_GIFT_WRAP: u32 = 1059;
/// NIP-94: File metadata attachment.
pub const KIND_FILE_METADATA: u32 = 1063;
/// NIP-23: Long-form content (articles, blog posts, RFCs).
/// Parameterized replaceable (NIP-33, 30000–39999 range) — keyed by `(pubkey, kind, d_tag)`.
/// Stored globally (channel_id = NULL); author-owned, not channel-scoped.
pub const KIND_LONG_FORM: u32 = 30023;
/// NIP-42 auth event — never stored (carries bearer tokens).
pub const KIND_AUTH: u32 = 22242;
@@ -62,6 +66,11 @@ pub const KIND_NIP29_GROUP_MEMBERS: u32 = 39002;
/// NIP-29: Addressable group roles definition.
pub const KIND_NIP29_GROUP_ROLES: u32 = 39003;
/// Lower bound of the NIP-33 parameterized replaceable range (30000–39999).
pub const PARAM_REPLACEABLE_KIND_MIN: u32 = 30000;
/// Upper bound of the NIP-33 parameterized replaceable range (30000–39999).
pub const PARAM_REPLACEABLE_KIND_MAX: u32 = 39999;
/// Lower bound of the ephemeral event range (20000–29999). Never stored.
pub const EPHEMERAL_KIND_MIN: u32 = 20000;
/// Upper bound of the ephemeral event range (20000–29999). Never stored.
@@ -271,6 +280,7 @@ pub const ALL_KINDS: &[u32] = &[
KIND_SUBSCRIPTION_RESUMED,
KIND_MEMBER_ADDED_NOTIFICATION,
KIND_MEMBER_REMOVED_NOTIFICATION,
KIND_LONG_FORM,
KIND_FORUM_POST,
KIND_FORUM_VOTE,
KIND_FORUM_COMMENT,
@@ -308,11 +318,18 @@ pub const fn is_ephemeral(kind: u32) -> bool {
/// Returns `true` if `kind` is replaceable (NIP-01: kinds 0, 3, 41, 10000–19999).
/// NIP-33 parameterized-replaceable kinds (30000–39999) use a different replacement
/// key (includes `d`-tag) and are handled separately via `replace_addressable_event`.
/// key (includes `d`-tag) and are handled separately via `replace_parameterized_event`.
pub const fn is_replaceable(kind: u32) -> bool {
matches!(kind, 0 | 3 | KIND_CHANNEL_METADATA | 10000..=19999)
}
/// Returns `true` if `kind` is in the NIP-33 parameterized replaceable range (30000–39999).
///
/// These events are keyed by `(pubkey, kind, d_tag)` — the latest `created_at` wins.
pub const fn is_parameterized_replaceable(kind: u32) -> bool {
kind >= PARAM_REPLACEABLE_KIND_MIN && kind <= PARAM_REPLACEABLE_KIND_MAX
}
/// Returns `true` if `kind` is a workflow execution event (46001–46012).
/// These must not trigger workflows (prevents infinite loops).
pub const fn is_workflow_execution_kind(kind: u32) -> bool {
@@ -348,4 +365,24 @@ mod tests {
assert!(seen.insert(k), "duplicate kind value: {k}");
}
}
#[test]
fn parameterized_replaceable_range() {
assert!(!is_parameterized_replaceable(29999));
assert!(is_parameterized_replaceable(30000));
assert!(is_parameterized_replaceable(30023)); // NIP-23 long-form
assert!(is_parameterized_replaceable(39000)); // NIP-29 group metadata
assert!(is_parameterized_replaceable(39999));
assert!(!is_parameterized_replaceable(40000));
}
#[test]
fn replaceable_and_parameterized_are_disjoint() {
for kind in 0..=65535u32 {
assert!(
!(is_replaceable(kind) && is_parameterized_replaceable(kind)),
"kind {kind} is both replaceable and parameterized replaceable"
);
}
}
}

155
crates/sprout-db/src/event.rs
View File

@@ -9,7 +9,7 @@ use nostr::Event;
use sqlx::{PgPool, QueryBuilder, Row};
use uuid::Uuid;
use sprout_core::kind::{event_kind_i32, is_ephemeral, KIND_AUTH};
use sprout_core::kind::{event_kind_i32, is_ephemeral, is_parameterized_replaceable, KIND_AUTH};
use sprout_core::StoredEvent;
use crate::error::{DbError, Result};
@@ -34,6 +34,38 @@ pub struct EventQuery {
/// Restrict to events with a `p` tag mentioning this hex pubkey.
/// Joins against `event_mentions` table (indexed).
pub p_tag_hex: Option<String>,
/// Restrict to events with this exact `d_tag` value (NIP-33).
/// Pushed into SQL via the `idx_events_parameterized` index.
pub d_tag: Option<String>,
}
/// Maximum length for a `d_tag` value (bytes). NIP-33 d-tags are short identifiers;
/// anything beyond this is either a bug or abuse.
pub const D_TAG_MAX_LEN: usize = 1024;
/// Extract the `d_tag` value for storage.
///
/// For NIP-33 parameterized replaceable events (kind 30000–39999): returns the first
/// `d` tag's value, or `""` if no `d` tag is present (per NIP-33 spec).
/// For all other events: returns `None` (column stays NULL).
pub fn extract_d_tag(event: &Event) -> Option<String> {
let kind_u32 = event.kind.as_u16() as u32;
if !is_parameterized_replaceable(kind_u32) {
return None;
}
let val = event
.tags
.iter()
.find_map(|tag| {
let parts = tag.as_slice();
if parts.len() >= 2 && parts[0] == "d" {
Some(parts[1].to_string())
} else {
None
}
})
.unwrap_or_default(); // Missing d tag → empty string per NIP-33
Some(val)
}
/// Insert a Nostr event. Rejects AUTH and ephemeral kinds.
@@ -64,10 +96,11 @@ pub async fn insert_event(
let created_at = DateTime::from_timestamp(created_at_secs, 0)
.ok_or(DbError::InvalidTimestamp(created_at_secs))?;
let received_at = Utc::now();
let d_tag = extract_d_tag(event);
let result = sqlx::query(
r#"
INSERT INTO events (id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
INSERT INTO events (id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id, d_tag)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
ON CONFLICT DO NOTHING
"#,
)
@@ -80,6 +113,7 @@ pub async fn insert_event(
.bind(sig_bytes.as_slice())
.bind(received_at)
.bind(channel_id)
.bind(d_tag.as_deref())
.execute(pool)
.await?;
@@ -152,6 +186,11 @@ pub async fn query_events(pool: &PgPool, q: &EventQuery) -> Result<Vec<StoredEve
.push_bind(u);
}
if let Some(ref d) = q.d_tag {
qb.push(format!(" AND {col_prefix}d_tag = "))
.push_bind(d.clone());
}
qb.push(format!(" ORDER BY {col_prefix}created_at DESC LIMIT "))
.push_bind(limit_val);
qb.push(" OFFSET ").push_bind(offset_val);
@@ -457,13 +496,14 @@ pub async fn insert_event_with_thread_metadata(
let created_at = DateTime::from_timestamp(created_at_secs, 0)
.ok_or(DbError::InvalidTimestamp(created_at_secs))?;
let received_at = Utc::now();
let d_tag = extract_d_tag(event);
let mut tx = pool.begin().await?;
// ── Insert event ──────────────────────────────────────────────────────────
let result = sqlx::query(
r#"
INSERT INTO events (id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
INSERT INTO events (id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id, d_tag)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
ON CONFLICT DO NOTHING
"#,
)
@@ -476,6 +516,7 @@ pub async fn insert_event_with_thread_metadata(
.bind(sig_bytes.as_slice())
.bind(received_at)
.bind(channel_id)
.bind(d_tag.as_deref())
.execute(&mut *tx)
.await?;
@@ -594,3 +635,107 @@ pub async fn insert_event_with_thread_metadata(
was_inserted,
))
}
#[cfg(test)]
mod tests {
use super::*;
use nostr::{EventBuilder, Keys, Kind, Tag};
fn make_event_with_kind_and_tags(kind: u16, tags: Vec<Tag>) -> nostr::Event {
let keys = Keys::generate();
EventBuilder::new(Kind::Custom(kind), "test", tags)
.sign_with_keys(&keys)
.expect("sign")
}
#[test]
fn extract_d_tag_from_nip33_event() {
let event = make_event_with_kind_and_tags(
30023,
vec![Tag::parse(&["d", "my-article-slug"]).unwrap()],
);
assert_eq!(extract_d_tag(&event), Some("my-article-slug".to_string()));
}
#[test]
fn extract_d_tag_first_d_wins() {
let event = make_event_with_kind_and_tags(
30023,
vec![
Tag::parse(&["d", "first"]).unwrap(),
Tag::parse(&["d", "second"]).unwrap(),
],
);
assert_eq!(extract_d_tag(&event), Some("first".to_string()));
}
#[test]
fn extract_d_tag_missing_becomes_empty_string() {
// NIP-33: "if there is no d tag, the d tag is considered to be ''"
let event =
make_event_with_kind_and_tags(30023, vec![Tag::parse(&["p", "abc123"]).unwrap()]);
assert_eq!(extract_d_tag(&event), Some(String::new()));
}
#[test]
fn extract_d_tag_empty_value_preserved() {
let event = make_event_with_kind_and_tags(30023, vec![Tag::parse(&["d", ""]).unwrap()]);
assert_eq!(extract_d_tag(&event), Some(String::new()));
}
#[test]
fn extract_d_tag_non_nip33_returns_none() {
// kind:1 (text note) — not parameterized replaceable
let event = make_event_with_kind_and_tags(
1,
vec![Tag::parse(&["d", "should-be-ignored"]).unwrap()],
);
assert_eq!(extract_d_tag(&event), None);
}
#[test]
fn extract_d_tag_nip29_group_metadata() {
// kind:39000 is in the 30000–39999 range — d_tag should be extracted
let event =
make_event_with_kind_and_tags(39000, vec![Tag::parse(&["d", "group-id"]).unwrap()]);
assert_eq!(extract_d_tag(&event), Some("group-id".to_string()));
}
#[test]
fn extract_d_tag_boundary_kinds() {
// kind:29999 — just below range
let below = make_event_with_kind_and_tags(29999, vec![Tag::parse(&["d", "val"]).unwrap()]);
assert_eq!(extract_d_tag(&below), None);
// kind:30000 — lower bound
let lower = make_event_with_kind_and_tags(30000, vec![Tag::parse(&["d", "val"]).unwrap()]);
assert_eq!(extract_d_tag(&lower), Some("val".to_string()));
// kind:39999 — upper bound
let upper = make_event_with_kind_and_tags(39999, vec![Tag::parse(&["d", "val"]).unwrap()]);
assert_eq!(extract_d_tag(&upper), Some("val".to_string()));
// kind:40000 — just above range
let above = make_event_with_kind_and_tags(40000, vec![Tag::parse(&["d", "val"]).unwrap()]);
assert_eq!(extract_d_tag(&above), None);
}
#[test]
fn extract_d_tag_single_element_d_tag_ignored() {
// A d tag with only one element (no value) should not match — parts.len() < 2
let event = make_event_with_kind_and_tags(30023, vec![Tag::parse(&["d"]).unwrap()]);
// No d tag with a value → empty string per NIP-33
assert_eq!(extract_d_tag(&event), Some(String::new()));
}
#[test]
fn extract_d_tag_preserves_full_value() {
// extract_d_tag returns the full value — length enforcement is at the ingest layer.
let long_val = "x".repeat(2048);
let event =
make_event_with_kind_and_tags(30023, vec![Tag::parse(&["d", &long_val]).unwrap()]);
let result = extract_d_tag(&event).unwrap();
assert_eq!(result.len(), 2048);
assert_eq!(result, long_val);
}
}

163
crates/sprout-db/src/lib.rs
View File

@@ -1179,6 +1179,25 @@ impl Db {
partition::ensure_future_partitions(&self.pool, months_ahead).await
}
/// Backfill `d_tag` for existing NIP-33 events (kind 30000–39999) that have `d_tag IS NULL`.
///
/// Idempotent — safe to call on every startup. No-ops when all rows are already populated.
/// Runs a single UPDATE touching only NIP-33 rows with NULL d_tag.
pub async fn backfill_d_tags(&self) -> Result<u64> {
let result = sqlx::query(
"UPDATE events \
SET d_tag = COALESCE( \
(SELECT elem->>1 FROM jsonb_array_elements(tags) AS elem \
WHERE elem->>0 = 'd' LIMIT 1), \
'' \
) \
WHERE kind BETWEEN 30000 AND 39999 AND d_tag IS NULL",
)
.execute(&self.pool)
.await?;
Ok(result.rows_affected())
}
// ── Pubkey Allowlist ─────────────────────────────────────────────────────
/// Check if a pubkey is in the allowlist.
@@ -1367,10 +1386,11 @@ impl Db {
let sig_bytes = event.sig.serialize();
let tags_json = serde_json::to_value(&event.tags)?;
let received_at = chrono::Utc::now();
let d_tag = crate::event::extract_d_tag(event);
let insert_result = sqlx::query(
"INSERT INTO events (id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id) \
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) \
"INSERT INTO events (id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id, d_tag) \
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) \
ON CONFLICT DO NOTHING",
)
.bind(event.id.as_bytes().as_slice())
@@ -1382,6 +1402,7 @@ impl Db {
.bind(sig_bytes.as_slice())
.bind(received_at)
.bind(channel_id)
.bind(d_tag.as_deref())
.execute(&mut *tx)
.await?;
@@ -1409,6 +1430,144 @@ impl Db {
true,
))
}
/// Atomically replace a NIP-33 parameterized replaceable event (kind 30000–39999).
///
/// Keeps only the event with the highest `created_at` per `(kind, pubkey, d_tag)`.
/// Same-second ties are broken by lowest event `id` (deterministic ordering).
/// The entire check → soft-delete → insert runs in a single transaction with
/// an advisory lock to prevent concurrent-insert races.
///
/// **Channel policy:** NIP-33 replacement keys on `(kind, pubkey, d_tag)` globally —
/// `channel_id` is NOT part of the replacement key. This matches the Nostr spec:
/// an author's parameterized replaceable event is a single global resource identified
/// by its d-tag, regardless of which channel it was submitted to. The `channel_id`
/// parameter is stored on the new row for query scoping but does not affect replacement.
///
/// Note: `replace_addressable_event()` keys on `channel_id` because it serves
/// relay-signed NIP-29 group metadata (kind 39000–39002) where the relay is the
/// author and channel_id distinguishes groups. User-submitted NIP-33 events use
/// this function instead, where the author's pubkey + d-tag is the natural key.
pub async fn replace_parameterized_event(
&self,
event: &nostr::Event,
d_tag: &str,
channel_id: Option<Uuid>,
) -> Result<(StoredEvent, bool)> {
let kind_i32 = sprout_core::kind::event_kind_i32(event);
let pubkey_bytes = event.pubkey.to_bytes();
let created_at_secs = event.created_at.as_u64() as i64;
let created_at = chrono::DateTime::from_timestamp(created_at_secs, 0)
.ok_or(DbError::InvalidTimestamp(created_at_secs))?;
// Stable advisory-lock key: FNV-1a over (kind, pubkey, d_tag).
// Same algorithm as replace_addressable_event — deterministic across processes.
let lock_key = {
let mut h: u64 = 0xcbf29ce484222325; // FNV offset basis
for b in kind_i32.to_le_bytes() {
h ^= b as u64;
h = h.wrapping_mul(0x100000001b3);
}
for b in pubkey_bytes.as_slice() {
h ^= *b as u64;
h = h.wrapping_mul(0x100000001b3);
}
for b in d_tag.as_bytes() {
h ^= *b as u64;
h = h.wrapping_mul(0x100000001b3);
}
h as i64
};
let mut tx = self.pool.begin().await?;
sqlx::query("SELECT pg_advisory_xact_lock($1)")
.bind(lock_key)
.execute(&mut *tx)
.await?;
// Check for existing event with same (kind, pubkey, d_tag).
let existing: Option<(chrono::DateTime<chrono::Utc>, Vec<u8>)> = sqlx::query_as(
"SELECT created_at, id FROM events \
WHERE kind = $1 AND pubkey = $2 AND d_tag = $3 AND deleted_at IS NULL \
ORDER BY created_at DESC, id ASC LIMIT 1",
)
.bind(kind_i32)
.bind(pubkey_bytes.as_slice())
.bind(d_tag)
.fetch_optional(&mut *tx)
.await?;
// Stale-write protection: reject if incoming is not newer.
let incoming_id = event.id.as_bytes().as_slice();
if let Some((existing_ts, existing_id)) = existing {
let dominated = created_at < existing_ts
|| (created_at == existing_ts && incoming_id >= existing_id.as_slice());
if dominated {
tx.rollback().await?;
let received_at = chrono::Utc::now();
return Ok((
StoredEvent::with_received_at(event.clone(), received_at, channel_id, false),
false,
));
}
// Soft-delete the older event(s).
sqlx::query(
"UPDATE events SET deleted_at = NOW() \
WHERE kind = $1 AND pubkey = $2 AND d_tag = $3 AND deleted_at IS NULL",
)
.bind(kind_i32)
.bind(pubkey_bytes.as_slice())
.bind(d_tag)
.execute(&mut *tx)
.await?;
}
// Insert the new event inside the transaction.
let sig_bytes = event.sig.serialize();
let tags_json = serde_json::to_value(&event.tags)?;
let received_at = chrono::Utc::now();
let insert_result = sqlx::query(
"INSERT INTO events (id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id, d_tag) \
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) \
ON CONFLICT DO NOTHING",
)
.bind(event.id.as_bytes().as_slice())
.bind(pubkey_bytes.as_slice())
.bind(created_at)
.bind(kind_i32)
.bind(&tags_json)
.bind(&event.content)
.bind(sig_bytes.as_slice())
.bind(received_at)
.bind(channel_id)
.bind(d_tag)
.execute(&mut *tx)
.await?;
let was_inserted = insert_result.rows_affected() > 0;
if !was_inserted {
tx.rollback().await?;
return Ok((
StoredEvent::with_received_at(event.clone(), received_at, channel_id, false),
false,
));
}
tx.commit().await?;
// Mentions are a denormalized index — safe outside the transaction.
if let Err(e) = crate::insert_mentions(&self.pool, event, channel_id).await {
tracing::warn!(event_id = %event.id, "Failed to insert mentions: {e}");
}
Ok((
StoredEvent::with_received_at(event.clone(), received_at, channel_id, true),
true,
))
}
}
/// A full API token record.

104
crates/sprout-relay/src/handlers/ingest.rs
View File

@@ -12,14 +12,15 @@ use uuid::Uuid;
use nostr::Event;
use sprout_auth::Scope;
use sprout_core::kind::{
event_kind_u32, KIND_AUTH, KIND_CANVAS, KIND_CONTACT_LIST, KIND_DELETION, KIND_FORUM_COMMENT,
KIND_FORUM_POST, KIND_FORUM_VOTE, KIND_GIFT_WRAP, KIND_MEMBER_ADDED_NOTIFICATION,
KIND_MEMBER_REMOVED_NOTIFICATION, KIND_NIP29_CREATE_GROUP, KIND_NIP29_DELETE_EVENT,
KIND_NIP29_DELETE_GROUP, KIND_NIP29_EDIT_METADATA, KIND_NIP29_JOIN_REQUEST,
KIND_NIP29_LEAVE_REQUEST, KIND_NIP29_PUT_USER, KIND_NIP29_REMOVE_USER, KIND_PRESENCE_UPDATE,
KIND_PROFILE, KIND_REACTION, KIND_STREAM_MESSAGE, KIND_STREAM_MESSAGE_BOOKMARKED,
KIND_STREAM_MESSAGE_DIFF, KIND_STREAM_MESSAGE_EDIT, KIND_STREAM_MESSAGE_PINNED,
KIND_STREAM_MESSAGE_SCHEDULED, KIND_STREAM_MESSAGE_V2, KIND_STREAM_REMINDER, KIND_TEXT_NOTE,
event_kind_u32, is_parameterized_replaceable, KIND_AUTH, KIND_CANVAS, KIND_CONTACT_LIST,
KIND_DELETION, KIND_FORUM_COMMENT, KIND_FORUM_POST, KIND_FORUM_VOTE, KIND_GIFT_WRAP,
KIND_LONG_FORM, KIND_MEMBER_ADDED_NOTIFICATION, KIND_MEMBER_REMOVED_NOTIFICATION,
KIND_NIP29_CREATE_GROUP, KIND_NIP29_DELETE_EVENT, KIND_NIP29_DELETE_GROUP,
KIND_NIP29_EDIT_METADATA, KIND_NIP29_JOIN_REQUEST, KIND_NIP29_LEAVE_REQUEST,
KIND_NIP29_PUT_USER, KIND_NIP29_REMOVE_USER, KIND_PRESENCE_UPDATE, KIND_PROFILE, KIND_REACTION,
KIND_STREAM_MESSAGE, KIND_STREAM_MESSAGE_BOOKMARKED, KIND_STREAM_MESSAGE_DIFF,
KIND_STREAM_MESSAGE_EDIT, KIND_STREAM_MESSAGE_PINNED, KIND_STREAM_MESSAGE_SCHEDULED,
KIND_STREAM_MESSAGE_V2, KIND_STREAM_REMINDER, KIND_TEXT_NOTE,
};
use sprout_core::verification::verify_event;
@@ -141,7 +142,7 @@ pub enum IngestError {
fn required_scope_for_kind(kind: u32, event: &Event) -> Result<Scope, &'static str> {
match kind {
KIND_PROFILE => Ok(Scope::UsersWrite),
KIND_TEXT_NOTE => Ok(Scope::MessagesWrite),
KIND_TEXT_NOTE | KIND_LONG_FORM => Ok(Scope::MessagesWrite),
KIND_CONTACT_LIST => Ok(Scope::UsersWrite),
KIND_DELETION
| KIND_REACTION
@@ -241,6 +242,24 @@ pub(crate) async fn derive_reaction_channel(
}
}
/// Kinds that are always global (`channel_id = NULL`).
///
/// If a client includes a stray `h` tag on these kinds, the ingest pipeline
/// sets `channel_id = None` — these events are never channel-scoped.
///
/// Note: the raw `h` tag remains on the stored event (Nostr events are signed,
/// so tags cannot be stripped without invalidating the signature). The read-path
/// filter matching in `filter.rs` treats explicit `h` tags as authoritative,
/// which means a stray `h` tag can still match `#h` queries. This is a known
/// limitation affecting all global-only kinds and should be addressed in the
/// filter layer as a follow-up.
pub(crate) fn is_global_only_kind(kind: u32) -> bool {
matches!(
kind,
KIND_PROFILE | KIND_TEXT_NOTE | KIND_CONTACT_LIST | KIND_LONG_FORM
)
}
/// Kinds that require an `h` tag for channel scoping.
pub(crate) fn requires_h_channel_scope(kind: u32) -> bool {
matches!(
@@ -843,9 +862,7 @@ pub async fn ingest_event(
};
// ── 5b. Global-only kinds ignore h-tags ─────────────────────────────
// kind:0 (profile), kind:1 (text note), kind:3 (contact list) are always global.
// If a client includes a stray h-tag, ignore it — these kinds are never channel-scoped.
if matches!(kind_u32, KIND_PROFILE | KIND_TEXT_NOTE | KIND_CONTACT_LIST) {
if is_global_only_kind(kind_u32) {
channel_id = None;
}
@@ -1238,6 +1255,21 @@ pub async fn ingest_event(
.replace_addressable_event(&event, channel_id)
.await
.map_err(|e| IngestError::Internal(format!("error: {e}")))?
} else if is_parameterized_replaceable(kind_u32) {
// NIP-33 parameterized replaceable — keyed by (kind, pubkey, d_tag).
let d_tag = sprout_db::event::extract_d_tag(&event).unwrap_or_default();
if d_tag.len() > sprout_db::event::D_TAG_MAX_LEN {
return Err(IngestError::Rejected(format!(
"invalid: d tag too long ({} bytes, max {})",
d_tag.len(),
sprout_db::event::D_TAG_MAX_LEN,
)));
}
state
.db
.replace_parameterized_event(&event, &d_tag, channel_id)
.await
.map_err(|e| IngestError::Internal(format!("error: {e}")))?
} else {
let thread_params = thread_meta.as_ref().map(|m| m.as_params());
match state
@@ -1298,8 +1330,8 @@ pub async fn ingest_event(
mod tests {
use super::*;
use sprout_core::kind::{
KIND_CANVAS, KIND_FORUM_COMMENT, KIND_FORUM_POST, KIND_FORUM_VOTE, KIND_PRESENCE_UPDATE,
KIND_STREAM_MESSAGE, KIND_STREAM_MESSAGE_DIFF,
KIND_CANVAS, KIND_FORUM_COMMENT, KIND_FORUM_POST, KIND_FORUM_VOTE, KIND_LONG_FORM,
KIND_PRESENCE_UPDATE, KIND_STREAM_MESSAGE, KIND_STREAM_MESSAGE_DIFF,
};
#[test]
@@ -1355,13 +1387,54 @@ mod tests {
assert!(!requires_h_channel_scope(KIND_REACTION));
}
#[test]
fn long_form_is_in_scope_allowlist() {
let dummy = make_dummy_event();
assert!(
required_scope_for_kind(KIND_LONG_FORM, &dummy).is_ok(),
"KIND_LONG_FORM (30023) should be accepted"
);
}
#[test]
fn long_form_requires_messages_write_scope() {
let dummy = make_dummy_event();
assert_eq!(
required_scope_for_kind(KIND_LONG_FORM, &dummy).unwrap(),
Scope::MessagesWrite,
);
}
#[test]
fn long_form_does_not_require_h_tag() {
// kind:30023 is global (author-owned, not channel-scoped)
assert!(!requires_h_channel_scope(KIND_LONG_FORM));
}
#[test]
fn long_form_is_global_only() {
// kind:30023 is always global — ingest nulls channel_id even if an h-tag is present
assert!(is_global_only_kind(KIND_LONG_FORM));
}
#[test]
fn global_only_and_channel_scoped_are_disjoint() {
// A kind cannot be both global-only and channel-scoped
for kind in 0..=65535u32 {
assert!(
!(is_global_only_kind(kind) && requires_h_channel_scope(kind)),
"kind {kind} is both global-only and channel-scoped"
);
}
}
#[test]
fn ephemeral_kinds_not_in_scope_allowlist() {
assert!(required_scope_for_kind(KIND_PRESENCE_UPDATE, &make_dummy_event()).is_err());
}
#[test]
fn per_kind_scope_allowlist_covers_all_18_migrated_kinds() {
fn per_kind_scope_allowlist_covers_all_migrated_kinds() {
let dummy = make_dummy_event();
let migrated = [
KIND_PROFILE,
@@ -1382,6 +1455,7 @@ mod tests {
KIND_FORUM_POST,
KIND_FORUM_VOTE,
KIND_FORUM_COMMENT,
KIND_LONG_FORM,
];
for kind in migrated {
assert!(

66
crates/sprout-relay/src/handlers/req.rs
View File

@@ -469,6 +469,33 @@ fn filter_to_query_params(filter: &Filter, channel_id: Option<uuid::Uuid>) -> Ev
}
});
// Push single-value #d tag into SQL via the d_tag column (NIP-33).
// Critical for parameterized replaceable lookups (authors + kinds + #d)
// where many events from the same author would push the target past LIMIT.
//
// Only push when the filter exclusively targets NIP-33 kinds (30000–39999),
// because `d_tag` is only populated for those kinds. Non-NIP-33 events have
// `d_tag = NULL`, so pushing `AND d_tag = $N` for a mixed-kind or kindless
// filter would silently exclude non-NIP-33 rows that match via their tags.
let filter_is_nip33_only = kinds.as_ref().is_some_and(|ks| {
!ks.is_empty()
&& ks
.iter()
.all(|&k| sprout_core::kind::is_parameterized_replaceable(k as u32))
});
let d_tag_key = nostr::SingleLetterTag::lowercase(nostr::Alphabet::D);
let d_tag = if filter_is_nip33_only {
filter.generic_tags.get(&d_tag_key).and_then(|values| {
if values.len() == 1 {
values.iter().next().map(|v| v.to_string())
} else {
None
}
})
} else {
None
};
EventQuery {
channel_id,
kinds,
@@ -477,6 +504,7 @@ fn filter_to_query_params(filter: &Filter, channel_id: Option<uuid::Uuid>) -> Ev
until,
limit: Some(limit),
p_tag_hex,
d_tag,
..Default::default()
}
}
@@ -601,4 +629,42 @@ mod tests {
assert!(has_search);
assert!(!has_non_search, "all-search filters should not be mixed");
}
#[test]
fn d_tag_pushdown_only_for_nip33_kinds() {
let d_tag = SingleLetterTag::lowercase(Alphabet::D);
// NIP-33 kind with #d → pushdown active
let nip33_filter = Filter::new()
.kind(nostr::Kind::Custom(30023))
.custom_tag(d_tag, ["my-slug"]);
let q = filter_to_query_params(&nip33_filter, None);
assert_eq!(q.d_tag, Some("my-slug".to_string()));
// Non-NIP-33 kind with #d → pushdown NOT active (would miss rows with d_tag=NULL)
let non_nip33_filter = Filter::new()
.kind(nostr::Kind::Custom(1))
.custom_tag(d_tag, ["some-value"]);
let q2 = filter_to_query_params(&non_nip33_filter, None);
assert_eq!(q2.d_tag, None);
// Mixed kinds (one NIP-33, one not) → pushdown NOT active
let mixed_filter = Filter::new()
.kinds([nostr::Kind::Custom(30023), nostr::Kind::Custom(1)])
.custom_tag(d_tag, ["slug"]);
let q3 = filter_to_query_params(&mixed_filter, None);
assert_eq!(q3.d_tag, None);
// No kinds specified → pushdown NOT active
let no_kinds_filter = Filter::new().custom_tag(d_tag, ["slug"]);
let q4 = filter_to_query_params(&no_kinds_filter, None);
assert_eq!(q4.d_tag, None);
// Multi-value #d → pushdown NOT active (can't push OR into single column match)
let multi_d_filter = Filter::new()
.kind(nostr::Kind::Custom(30023))
.custom_tag(d_tag, ["slug-a", "slug-b"]);
let q5 = filter_to_query_params(&multi_d_filter, None);
assert_eq!(q5.d_tag, None);
}
}

8
crates/sprout-relay/src/main.rs
View File

@@ -59,6 +59,14 @@ async fn main() -> anyhow::Result<()> {
error!("Failed to ensure partitions: {e}");
}
// NIP-33: backfill d_tag for any existing parameterized replaceable events
// that predate the column addition. Idempotent — no-ops when fully populated.
match db.backfill_d_tags().await {
Ok(0) => {}
Ok(n) => info!("Backfilled d_tag for {n} NIP-33 events"),
Err(e) => error!("Failed to backfill d_tags: {e}"),
}
let audit_pool = sqlx::PgPool::connect(&config.database_url)
.await
.map_err(|e| anyhow::anyhow!("Audit DB connection failed: {e}"))?;

37
crates/sprout-relay/src/nip11.rs
View File

@@ -4,6 +4,11 @@ use serde::{Deserialize, Serialize};
use crate::connection::MAX_FRAME_BYTES;
/// NIPs supported by this relay, advertised in the NIP-11 document.
/// Kept as a module-level constant so tests can verify it without constructing
/// a full `Config` (which reads env vars and races with config.rs tests).
pub(crate) const SUPPORTED_NIPS: &[u32] = &[1, 2, 10, 11, 16, 17, 23, 25, 29, 33, 42, 50];
/// Relay information document served at `GET /` with `Accept: application/nostr+json`.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RelayInfo {
@@ -56,7 +61,7 @@ impl RelayInfo {
description: "Sprout — private team communication relay".to_string(),
pubkey: None,
contact: None,
supported_nips: vec![1, 2, 10, 11, 16, 17, 25, 29, 42, 50],
supported_nips: SUPPORTED_NIPS.to_vec(),
software: "https://github.com/sprout-rs/sprout".to_string(),
version: env!("CARGO_PKG_VERSION").to_string(),
limitation: Some(RelayLimitation {
@@ -80,3 +85,33 @@ pub async fn relay_info_handler(
) -> axum::response::Json<RelayInfo> {
axum::response::Json(RelayInfo::from_config(&state.config))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn supported_nips_includes_nip23_and_nip33() {
// Tests the production SUPPORTED_NIPS constant directly — no Config::from_env()
// needed, avoiding the env-var race with config.rs tests.
assert!(
SUPPORTED_NIPS.contains(&23),
"NIP-23 (long-form content) must be advertised"
);
assert!(
SUPPORTED_NIPS.contains(&33),
"NIP-33 (parameterized replaceable) must be advertised"
);
}
#[test]
fn supported_nips_are_sorted() {
let mut sorted = SUPPORTED_NIPS.to_vec();
sorted.sort();
assert_eq!(
SUPPORTED_NIPS,
&sorted[..],
"supported_nips should be sorted"
);
}
}

319
crates/sprout-test-client/tests/e2e_long_form.rs Normal file
View File

@@ -0,0 +1,319 @@
//! End-to-end tests for NIP-23 long-form content (kind:30023).
//!
//! These tests require a running relay instance. By default they are marked
//! `#[ignore]` so that `cargo test` does not fail in CI when the relay is not
//! available.
//!
//! # Running
//!
//! Start the relay, then run:
//!
//! ```text
//! cargo test --test e2e_long_form -- --ignored
//! ```
//!
//! Override the relay URL with the `RELAY_URL` environment variable:
//!
//! ```text
//! RELAY_URL=ws://relay.example.com cargo test --test e2e_long_form -- --ignored
//! ```
use std::time::Duration;
use nostr::{Alphabet, EventBuilder, Filter, Keys, Kind, SingleLetterTag, Tag, Timestamp};
use sprout_test_client::SproutTestClient;
const KIND_LONG_FORM: u16 = 30023;
fn relay_url() -> String {
std::env::var("RELAY_URL").unwrap_or_else(|_| "ws://localhost:3000".to_string())
}
fn sub_id(name: &str) -> String {
format!("e2e-{name}-{}", uuid::Uuid::new_v4())
}
/// Build a kind:30023 event with standard NIP-23 tags.
fn build_long_form_event(
keys: &Keys,
d_tag: &str,
title: &str,
content: &str,
extra_tags: Vec<Tag>,
) -> nostr::Event {
let mut tags = vec![
Tag::parse(&["d", d_tag]).unwrap(),
Tag::parse(&["title", title]).unwrap(),
];
tags.extend(extra_tags);
EventBuilder::new(Kind::Custom(KIND_LONG_FORM), content, tags)
.sign_with_keys(keys)
.unwrap()
}
// ── Tests ─────────────────────────────────────────────────────────────────────
/// kind:30023 events are accepted by the relay.
#[tokio::test]
#[ignore]
async fn test_long_form_accepted() {
let url = relay_url();
let keys = Keys::generate();
let mut client = SproutTestClient::connect(&url, &keys)
.await
.expect("connect");
let event = build_long_form_event(
&keys,
"test-article-accept",
"Test Article",
"# Hello\n\nThis is a test article.",
vec![],
);
let ok = client.send_event(event).await.expect("send event");
assert!(
ok.accepted,
"relay should accept kind:30023: {}",
ok.message
);
client.disconnect().await.expect("disconnect");
}
/// kind:30023 events are retrievable via REQ with kinds filter.
#[tokio::test]
#[ignore]
async fn test_long_form_retrievable() {
let url = relay_url();
let keys = Keys::generate();
let mut client = SproutTestClient::connect(&url, &keys)
.await
.expect("connect");
let d_tag = format!("retrieve-{}", uuid::Uuid::new_v4().simple());
let event = build_long_form_event(
&keys,
&d_tag,
"Retrievable Article",
"# Retrievable\n\nBody text.",
vec![],
);
let event_id = event.id;
let ok = client.send_event(event).await.expect("send event");
assert!(ok.accepted, "relay should accept: {}", ok.message);
// Query back by kind + author
let sid = sub_id("retrieve");
let filter = Filter::new()
.kind(Kind::Custom(KIND_LONG_FORM))
.author(keys.public_key());
client
.subscribe(&sid, vec![filter])
.await
.expect("subscribe");
let events = client
.collect_until_eose(&sid, Duration::from_secs(5))
.await
.expect("collect");
assert!(
events.iter().any(|e| e.id == event_id),
"should find the published article in query results"
);
client.disconnect().await.expect("disconnect");
}
/// kind:30023 is stored globally (channel_id = NULL) — stray h-tags are ignored.
/// An event with a stray h-tag should still be retrievable via a global query
/// (no h-tag filter), proving it was stored as global.
#[tokio::test]
#[ignore]
async fn test_long_form_stray_h_tag_ignored() {
let url = relay_url();
let keys = Keys::generate();
let mut client = SproutTestClient::connect(&url, &keys)
.await
.expect("connect");
// Publish with a stray h-tag (a UUID that doesn't correspond to any channel).
let fake_channel = uuid::Uuid::new_v4().to_string();
let d_tag = format!("stray-h-{}", uuid::Uuid::new_v4().simple());
let event = build_long_form_event(
&keys,
&d_tag,
"Stray H-Tag Article",
"Should be stored globally despite h-tag.",
vec![Tag::parse(&["h", &fake_channel]).unwrap()],
);
let event_id = event.id;
let ok = client.send_event(event).await.expect("send event");
assert!(ok.accepted, "relay should accept: {}", ok.message);
// Query globally (no h-tag filter) — should find the article.
let sid = sub_id("stray-h");
let filter = Filter::new()
.kind(Kind::Custom(KIND_LONG_FORM))
.author(keys.public_key());
client
.subscribe(&sid, vec![filter])
.await
.expect("subscribe");
let events = client
.collect_until_eose(&sid, Duration::from_secs(5))
.await
.expect("collect");
assert!(
events.iter().any(|e| e.id == event_id),
"article with stray h-tag should be retrievable via global query"
);
// NOTE: Ideally, querying with #h=<fake_channel> should NOT return the
// article since it's global. However, the raw h-tag remains on the stored
// event (Nostr events are signed — tags can't be stripped without breaking
// the signature), and the read-path filter matching in filter.rs treats
// explicit h-tags as authoritative. This is a pre-existing limitation
// affecting all global-only kinds (0, 1, 3, 30023) and should be fixed
// in the filter layer as a follow-up.
client.disconnect().await.expect("disconnect");
}
/// NIP-33 replacement: publishing a newer kind:30023 with the same d-tag replaces the old one.
#[tokio::test]
#[ignore]
async fn test_long_form_nip33_replacement() {
let url = relay_url();
let keys = Keys::generate();
let mut client = SproutTestClient::connect(&url, &keys)
.await
.expect("connect");
let d_tag = format!("replace-{}", uuid::Uuid::new_v4().simple());
// Publish v1
let v1 = build_long_form_event(&keys, &d_tag, "Article v1", "Version 1 content.", vec![]);
let ok1 = client.send_event(v1).await.expect("send v1");
assert!(ok1.accepted, "v1 should be accepted: {}", ok1.message);
// Small delay to ensure different created_at timestamps
tokio::time::sleep(Duration::from_secs(1)).await;
// Publish v2 with the same d-tag
let v2 = build_long_form_event(
&keys,
&d_tag,
"Article v2",
"Version 2 content — updated.",
vec![],
);
let v2_id = v2.id;
let ok2 = client.send_event(v2).await.expect("send v2");
assert!(ok2.accepted, "v2 should be accepted: {}", ok2.message);
// Query — should only get v2 (v1 replaced)
let sid = sub_id("replace");
let filter = Filter::new()
.kind(Kind::Custom(KIND_LONG_FORM))
.author(keys.public_key())
.custom_tag(SingleLetterTag::lowercase(Alphabet::D), [d_tag.as_str()]);
client
.subscribe(&sid, vec![filter])
.await
.expect("subscribe");
let events = client
.collect_until_eose(&sid, Duration::from_secs(5))
.await
.expect("collect");
assert_eq!(
events.len(),
1,
"should have exactly one event after replacement"
);
assert_eq!(events[0].id, v2_id, "surviving event should be v2");
assert!(
events[0].content.contains("Version 2"),
"content should be v2"
);
client.disconnect().await.expect("disconnect");
}
/// NIP-33 stale-write protection: an older event cannot replace a newer one.
#[tokio::test]
#[ignore]
async fn test_long_form_stale_write_rejected() {
let url = relay_url();
let keys = Keys::generate();
let mut client = SproutTestClient::connect(&url, &keys)
.await
.expect("connect");
let d_tag = format!("stale-{}", uuid::Uuid::new_v4().simple());
// Publish the "newer" event first (with a future-ish timestamp)
let newer = {
let tags = vec![
Tag::parse(&["d", &d_tag]).unwrap(),
Tag::parse(&["title", "Newer Article"]).unwrap(),
];
EventBuilder::new(Kind::Custom(KIND_LONG_FORM), "Newer content.", tags)
.custom_created_at(Timestamp::from(nostr::Timestamp::now().as_u64() + 100))
.sign_with_keys(&keys)
.unwrap()
};
let newer_id = newer.id;
let ok1 = client.send_event(newer).await.expect("send newer");
assert!(ok1.accepted, "newer should be accepted: {}", ok1.message);
// Now try to publish an "older" event with the same d-tag but earlier timestamp
let older = {
let tags = vec![
Tag::parse(&["d", &d_tag]).unwrap(),
Tag::parse(&["title", "Older Article"]).unwrap(),
];
EventBuilder::new(Kind::Custom(KIND_LONG_FORM), "Older content.", tags)
.custom_created_at(Timestamp::from(nostr::Timestamp::now().as_u64() - 100))
.sign_with_keys(&keys)
.unwrap()
};
let _ok2 = client.send_event(older).await.expect("send older");
// Stale write may be rejected or accepted-as-duplicate — either way,
// the older event must NOT replace the newer one.
// Query — should still have the newer event
let sid = sub_id("stale");
let filter = Filter::new()
.kind(Kind::Custom(KIND_LONG_FORM))
.author(keys.public_key())
.custom_tag(SingleLetterTag::lowercase(Alphabet::D), [d_tag.as_str()]);
client
.subscribe(&sid, vec![filter])
.await
.expect("subscribe");
let events = client
.collect_until_eose(&sid, Duration::from_secs(5))
.await
.expect("collect");
assert_eq!(events.len(), 1, "should have exactly one event");
assert_eq!(
events[0].id, newer_id,
"surviving event should be the newer one"
);
assert!(
events[0].content.contains("Newer"),
"content should be from the newer event"
);
client.disconnect().await.expect("disconnect");
}

2
schema/schema.sql
View File

@@ -94,6 +94,7 @@ CREATE TABLE events (
received_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
channel_id UUID,
deleted_at TIMESTAMPTZ,
d_tag TEXT,
PRIMARY KEY (created_at, id)
) PARTITION BY RANGE (created_at);
@@ -120,6 +121,7 @@ CREATE INDEX idx_events_kind_created ON events (kind, created_at);
CREATE INDEX idx_events_id ON events (id);
CREATE INDEX idx_events_deleted ON events (deleted_at);
CREATE INDEX idx_events_addressable ON events (kind, pubkey, channel_id, deleted_at);
CREATE INDEX idx_events_parameterized ON events (kind, pubkey, d_tag, deleted_at) WHERE d_tag IS NOT NULL;
-- ── Event mentions ────────────────────────────────────────────────────────────

17
scripts/backfill-d-tag.sql Normal file
View File

@@ -0,0 +1,17 @@
-- Backfill d_tag for existing NIP-33 range events (kind 30000–39999).
-- Idempotent: only updates rows where d_tag is still NULL.
-- Includes soft-deleted rows so the column is fully populated.
-- Run once after adding the d_tag column to the events table.
--
-- Usage: psql $DATABASE_URL -f scripts/backfill-d-tag.sql
UPDATE events
SET d_tag = COALESCE(
(SELECT elem->>1
FROM jsonb_array_elements(tags) AS elem
WHERE elem->>0 = 'd'
LIMIT 1),
''
)
WHERE kind BETWEEN 30000 AND 39999
AND d_tag IS NULL;

11
scripts/dev-setup.sh
View File

@@ -151,6 +151,17 @@ else
sleep 2
done
success "Migrations applied via pgschema"
# Run data backfills (idempotent — safe to re-run).
BACKFILL_DIR="${REPO_ROOT}/scripts"
if [[ -f "${BACKFILL_DIR}/backfill-d-tag.sql" ]]; then
log "Running d_tag backfill for NIP-33 events..."
if psql "${DATABASE_URL}" -f "${BACKFILL_DIR}/backfill-d-tag.sql" 2>/dev/null; then
success "d_tag backfill complete"
else
warn "d_tag backfill failed (relay startup will retry automatically)"
fi
fi
else
error "pgschema not found at ${PGSCHEMA}. Run: ./bin/hermit install pgschema"
exit 1