fix(media): simplify object-key path rollout

Signed-off-by: Brad Seiler <seiler@squareup.com>
This commit is contained in:
Brad Seiler
2026-08-06 22:55:37 -04:00
committed by coder
parent 811ad71a7b
commit 45adf429ad
10 changed files with 67 additions and 56 deletions

View File

@@ -96,10 +96,10 @@ BUZZ_S3_REGION=us-east-1
BUZZ_S3_ADDRESSING_STYLE=path
# Media payload layout. Unset defaults to legacy-only, preserving the exact read/write
# behavior of existing deployments. Existing deployments migrate explicitly through:
# legacy-only -> dual-read-legacy-write -> dual-read-dual-write -> sharded-only.
# New deployments should set sharded-only before accepting their first upload so
# they never need a backfill. See deploy/charts/buzz/README.md for the full procedure.
# behavior of existing deployments. Existing deployments use legacy-only ->
# dual-read-and-write -> backfill -> sharded-only. New deployments should set
# sharded-only before accepting their first upload so they never need a backfill.
# See deploy/charts/buzz/README.md for the full procedure.
# BUZZ_MEDIA_MIGRATION_PHASE=legacy-only
# -----------------------------------------------------------------------------

View File

@@ -40,14 +40,10 @@ pub enum MediaMigrationPhase {
#[default]
#[serde(rename = "legacy-only")]
LegacyOnly,
/// Compatibility phase: prefer sharded reads with legacy fallback, but
/// write only legacy keys.
#[serde(rename = "dual-read-legacy-write")]
DualReadLegacyWrite,
/// Migration phase: prefer sharded reads with legacy fallback and write
/// both layouts (sharded first, then legacy).
#[serde(rename = "dual-read-dual-write")]
DualReadDualWrite,
#[serde(rename = "dual-read-and-write")]
DualReadAndWrite,
/// Final phase: read and write only hash-sharded keys.
#[serde(rename = "sharded-only")]
ShardedOnly,
@@ -66,7 +62,7 @@ impl MediaMigrationPhase {
/// Whether writes include the sharded key.
pub const fn writes_sharded(self) -> bool {
matches!(self, Self::DualReadDualWrite | Self::ShardedOnly)
matches!(self, Self::DualReadAndWrite | Self::ShardedOnly)
}
/// Whether writes include the legacy key.
@@ -81,13 +77,11 @@ impl FromStr for MediaMigrationPhase {
fn from_str(value: &str) -> Result<Self, Self::Err> {
match value {
"legacy-only" => Ok(Self::LegacyOnly),
"dual-read-legacy-write" => Ok(Self::DualReadLegacyWrite),
"dual-read-dual-write" => Ok(Self::DualReadDualWrite),
"dual-read-and-write" => Ok(Self::DualReadAndWrite),
"sharded-only" => Ok(Self::ShardedOnly),
_ => Err(format!(
"BUZZ_MEDIA_MIGRATION_PHASE must be 'legacy-only', \
'dual-read-legacy-write', 'dual-read-dual-write', or 'sharded-only', \
got {value:?}"
'dual-read-and-write', or 'sharded-only', got {value:?}"
)),
}
}
@@ -283,15 +277,18 @@ mod tests {
);
assert_eq!("legacy-only".parse(), Ok(MediaMigrationPhase::LegacyOnly));
assert_eq!(
"dual-read-legacy-write".parse(),
Ok(MediaMigrationPhase::DualReadLegacyWrite)
);
assert_eq!(
"dual-read-dual-write".parse(),
Ok(MediaMigrationPhase::DualReadDualWrite)
"dual-read-and-write".parse(),
Ok(MediaMigrationPhase::DualReadAndWrite)
);
assert_eq!("sharded-only".parse(), Ok(MediaMigrationPhase::ShardedOnly));
for invalid in ["legacy", "dual", "sharded", "new"] {
for invalid in [
"legacy",
"dual",
"dual-read-legacy-write",
"dual-read-dual-write",
"sharded",
"new",
] {
assert!(invalid.parse::<MediaMigrationPhase>().is_err());
}
}
@@ -301,14 +298,7 @@ mod tests {
let layouts = [
(MediaMigrationPhase::LegacyOnly, false, true, false, true),
(
MediaMigrationPhase::DualReadLegacyWrite,
true,
true,
false,
true,
),
(
MediaMigrationPhase::DualReadDualWrite,
MediaMigrationPhase::DualReadAndWrite,
true,
true,
true,

View File

@@ -293,9 +293,10 @@ impl MediaStorage {
/// Resolve a media payload according to the configured read layout.
///
/// Dual-read phases prefer the sharded key and fall back to legacy only on
/// an actual not-found result. Authorization, transport, throttling, and
/// service errors are returned so fallback cannot mask an unhealthy store.
/// The dual-read-and-write phase prefers the sharded key and falls back to
/// legacy only on an actual not-found result. Authorization, transport,
/// throttling, and service errors are returned so fallback cannot mask an
/// unhealthy store.
pub async fn resolve_read_key(
&self,
ctx: &TenantContext,

View File

@@ -1203,17 +1203,17 @@ mod tests {
}
#[test]
fn media_migration_phase_env_accepts_sharded_only_and_rejects_invalid_values() {
fn media_migration_phase_env_accepts_dual_read_and_write_and_rejects_removed_phase() {
let _guard = ENV_MUTEX.lock().unwrap();
let previous = std::env::var_os("BUZZ_MEDIA_MIGRATION_PHASE");
std::env::set_var("BUZZ_MEDIA_MIGRATION_PHASE", "sharded-only");
std::env::set_var("BUZZ_MEDIA_MIGRATION_PHASE", "dual-read-and-write");
let configured = Config::from_env()
.expect("sharded-only media config")
.expect("dual-read-and-write media config")
.media
.migration_phase;
std::env::set_var("BUZZ_MEDIA_MIGRATION_PHASE", "auto");
std::env::set_var("BUZZ_MEDIA_MIGRATION_PHASE", "dual-read-legacy-write");
let invalid = Config::from_env();
if let Some(value) = previous {
@@ -1222,7 +1222,10 @@ mod tests {
std::env::remove_var("BUZZ_MEDIA_MIGRATION_PHASE");
}
assert_eq!(configured, buzz_media::MediaMigrationPhase::ShardedOnly);
assert_eq!(
configured,
buzz_media::MediaMigrationPhase::DualReadAndWrite
);
assert!(matches!(
invalid,
Err(ConfigError::InvalidValue(ref message))

View File

@@ -107,8 +107,7 @@ The supported modes are:
| Mode | Reads | Writes | Use |
|---|---|---|---|
| `legacy-only` (default) | Legacy only | Legacy only | Behavior-preserving upgrades and deployments that are not migrating yet |
| `dual-read-legacy-write` | Sharded first, legacy fallback | Legacy only | Deploy compatibility readers before changing the write layout |
| `dual-read-dual-write` | Sharded first, legacy fallback | Sharded and legacy | Migration and rollback window |
| `dual-read-and-write` | Sharded first, legacy fallback | Sharded and legacy | Migration, backfill, and rollback window |
| `sharded-only` | Sharded only | Sharded only | New deployments and completed migrations |
**New deployments should start with `sharded-only` before accepting the first
@@ -118,7 +117,7 @@ eliminates the later backfill and cleanup migration.
Existing deployments should keep the `legacy-only` default until an operator
intentionally starts the staged migration. Do not roll `sharded-only` writers
back to a Buzz version that predates sharded reads. Returning from
`sharded-only` to `dual-read-dual-write` does not retroactively recreate legacy
`sharded-only` to `dual-read-and-write` does not retroactively recreate legacy
copies; run and verify the backfill before relying on old-version rollback.
## Relay Pod extensions
@@ -307,16 +306,15 @@ For an **existing deployment**:
1. Upgrade with the default `legacy-only`. This is behavior-preserving: the
relay continues reading and writing only flat legacy keys.
2. Select `dual-read-legacy-write`; confirm every relay can read both layouts.
Writes remain legacy-only, so this step creates no duplicate objects.
3. Select `dual-read-dual-write` and observe successful writes and storage
telemetry through a rollback window.
4. Run the backfill Job. Re-run from any logged checkpoint as needed, then run
2. Select `dual-read-and-write`; confirm every relay can read both layouts, and
observe successful dual writes and storage telemetry through a rollback
window.
3. Run the backfill Job. Re-run from any logged checkpoint as needed, then run
it again to a clean `copied=0` result. Reconcile storage sweep unknown keys,
duplicate gauges, migration failures, and legacy fallback traffic.
5. Select `sharded-only` only after every supported rollback version understands
4. Select `sharded-only` only after every supported rollback version understands
sharded keys and reconciliation finds no missing sharded destination.
6. Run the deletion Job with its default `dry-run=true`, review the output, and
5. Run the deletion Job with its default `dry-run=true`, review the output, and
retain a recovery window/S3 versions. Only then set `dry-run=false` and
`BUZZ_MEDIA_DELETE_CONFIRM=delete-verified-legacy-media`. The tool checks the
corresponding sharded object immediately before every deletion.

View File

@@ -65,6 +65,24 @@ tests:
value: "true"
template: templates/deployment.yaml
- it: renders the dual-read-and-write migration phase
set:
relayUrl: wss://buzz.example.com
ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000"
externalPostgresql.url: postgres://u:p@h:5432/d
externalRedis.url: redis://h:6379
s3.endpoint: http://minio:9000
s3.accessKey: a
s3.secretKey: s
relay.mediaMigrationPhase: dual-read-and-write
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: BUZZ_MEDIA_MIGRATION_PHASE
value: "dual-read-and-write"
template: templates/deployment.yaml
- it: lets a new deployment opt directly into sharded-only media
set:
relayUrl: wss://buzz.example.com

View File

@@ -63,7 +63,7 @@
"requireAuthToken": { "type": "boolean" },
"requireRelayMembership": { "type": "boolean" },
"requireMediaGetAuth": { "type": "boolean" },
"mediaMigrationPhase": { "type": "string", "enum": ["legacy-only", "dual-read-legacy-write", "dual-read-dual-write", "sharded-only"] },
"mediaMigrationPhase": { "type": "string", "enum": ["legacy-only", "dual-read-and-write", "sharded-only"] },
"allowNipOaAuth": { "type": "boolean" },
"huddleAudioAvailable": {
"type": ["boolean", "null"],

View File

@@ -123,9 +123,9 @@ relay:
# local development or fully public communities — desktop, mobile, and CLI
# clients all attach read auth.
requireMediaGetAuth: true
# Behavior-preserving upgrade default: legacy-only reads and writes. Existing deployments migrate through
# legacy-only -> dual-read-legacy-write -> dual-read-dual-write -> sharded-only. New deployments should override
# this to sharded-only before accepting their first upload so they never need a backfill.
# Behavior-preserving upgrade default: legacy-only reads and writes. Existing deployments use
# legacy-only -> dual-read-and-write -> backfill -> sharded-only. New deployments should override this to
# sharded-only before accepting their first upload so they never need a backfill.
mediaMigrationPhase: legacy-only
allowNipOaAuth: true
pubkeyAllowlist: false

View File

@@ -36,9 +36,9 @@ BUZZ_S3_BUCKET=buzz-media
# Bundled MinIO uses path-style URLs; deploy/compose/compose.yml pins this.
BUZZ_S3_ADDRESSING_STYLE=path
# Behavior-preserving upgrade default: read and write only legacy keys. Existing
# deployments migrate explicitly through legacy-only -> dual-read-legacy-write ->
# dual-read-dual-write -> sharded-only. New deployments should select sharded-only
# before their first upload so they never need a backfill.
# deployments use legacy-only -> dual-read-and-write -> backfill -> sharded-only.
# New deployments should select sharded-only before their first upload so they
# never need a backfill.
BUZZ_MEDIA_MIGRATION_PHASE=legacy-only
# Optional host ports. Base compose publishes the relay directly on BUZZ_HTTP_PORT.

View File

@@ -1,4 +1,5 @@
# Copy this example, pin the same image tag as the relay, and supply its S3
# Run this while relays are in dual-read-and-write, before switching them to
# sharded-only. Pin the same image tag as the relay and supply its S3
# environment/IRSA. Re-running is safe: existing sharded objects are skipped.
apiVersion: batch/v1
kind: Job