mirror of
https://github.com/block/buzz.git
synced 2026-11-16 06:11:34 +00:00
fix(media): simplify object-key path rollout
Signed-off-by: Brad Seiler <seiler@squareup.com>
This commit is contained in:
@@ -96,10 +96,10 @@ BUZZ_S3_REGION=us-east-1
|
||||
BUZZ_S3_ADDRESSING_STYLE=path
|
||||
|
||||
# Media payload layout. Unset defaults to legacy-only, preserving the exact read/write
|
||||
# behavior of existing deployments. Existing deployments migrate explicitly through:
|
||||
# legacy-only -> dual-read-legacy-write -> dual-read-dual-write -> sharded-only.
|
||||
# New deployments should set sharded-only before accepting their first upload so
|
||||
# they never need a backfill. See deploy/charts/buzz/README.md for the full procedure.
|
||||
# behavior of existing deployments. Existing deployments use legacy-only ->
|
||||
# dual-read-and-write -> backfill -> sharded-only. New deployments should set
|
||||
# sharded-only before accepting their first upload so they never need a backfill.
|
||||
# See deploy/charts/buzz/README.md for the full procedure.
|
||||
# BUZZ_MEDIA_MIGRATION_PHASE=legacy-only
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
@@ -40,14 +40,10 @@ pub enum MediaMigrationPhase {
|
||||
#[default]
|
||||
#[serde(rename = "legacy-only")]
|
||||
LegacyOnly,
|
||||
/// Compatibility phase: prefer sharded reads with legacy fallback, but
|
||||
/// write only legacy keys.
|
||||
#[serde(rename = "dual-read-legacy-write")]
|
||||
DualReadLegacyWrite,
|
||||
/// Migration phase: prefer sharded reads with legacy fallback and write
|
||||
/// both layouts (sharded first, then legacy).
|
||||
#[serde(rename = "dual-read-dual-write")]
|
||||
DualReadDualWrite,
|
||||
#[serde(rename = "dual-read-and-write")]
|
||||
DualReadAndWrite,
|
||||
/// Final phase: read and write only hash-sharded keys.
|
||||
#[serde(rename = "sharded-only")]
|
||||
ShardedOnly,
|
||||
@@ -66,7 +62,7 @@ impl MediaMigrationPhase {
|
||||
|
||||
/// Whether writes include the sharded key.
|
||||
pub const fn writes_sharded(self) -> bool {
|
||||
matches!(self, Self::DualReadDualWrite | Self::ShardedOnly)
|
||||
matches!(self, Self::DualReadAndWrite | Self::ShardedOnly)
|
||||
}
|
||||
|
||||
/// Whether writes include the legacy key.
|
||||
@@ -81,13 +77,11 @@ impl FromStr for MediaMigrationPhase {
|
||||
fn from_str(value: &str) -> Result<Self, Self::Err> {
|
||||
match value {
|
||||
"legacy-only" => Ok(Self::LegacyOnly),
|
||||
"dual-read-legacy-write" => Ok(Self::DualReadLegacyWrite),
|
||||
"dual-read-dual-write" => Ok(Self::DualReadDualWrite),
|
||||
"dual-read-and-write" => Ok(Self::DualReadAndWrite),
|
||||
"sharded-only" => Ok(Self::ShardedOnly),
|
||||
_ => Err(format!(
|
||||
"BUZZ_MEDIA_MIGRATION_PHASE must be 'legacy-only', \
|
||||
'dual-read-legacy-write', 'dual-read-dual-write', or 'sharded-only', \
|
||||
got {value:?}"
|
||||
'dual-read-and-write', or 'sharded-only', got {value:?}"
|
||||
)),
|
||||
}
|
||||
}
|
||||
@@ -283,15 +277,18 @@ mod tests {
|
||||
);
|
||||
assert_eq!("legacy-only".parse(), Ok(MediaMigrationPhase::LegacyOnly));
|
||||
assert_eq!(
|
||||
"dual-read-legacy-write".parse(),
|
||||
Ok(MediaMigrationPhase::DualReadLegacyWrite)
|
||||
);
|
||||
assert_eq!(
|
||||
"dual-read-dual-write".parse(),
|
||||
Ok(MediaMigrationPhase::DualReadDualWrite)
|
||||
"dual-read-and-write".parse(),
|
||||
Ok(MediaMigrationPhase::DualReadAndWrite)
|
||||
);
|
||||
assert_eq!("sharded-only".parse(), Ok(MediaMigrationPhase::ShardedOnly));
|
||||
for invalid in ["legacy", "dual", "sharded", "new"] {
|
||||
for invalid in [
|
||||
"legacy",
|
||||
"dual",
|
||||
"dual-read-legacy-write",
|
||||
"dual-read-dual-write",
|
||||
"sharded",
|
||||
"new",
|
||||
] {
|
||||
assert!(invalid.parse::<MediaMigrationPhase>().is_err());
|
||||
}
|
||||
}
|
||||
@@ -301,14 +298,7 @@ mod tests {
|
||||
let layouts = [
|
||||
(MediaMigrationPhase::LegacyOnly, false, true, false, true),
|
||||
(
|
||||
MediaMigrationPhase::DualReadLegacyWrite,
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
true,
|
||||
),
|
||||
(
|
||||
MediaMigrationPhase::DualReadDualWrite,
|
||||
MediaMigrationPhase::DualReadAndWrite,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
|
||||
@@ -293,9 +293,10 @@ impl MediaStorage {
|
||||
|
||||
/// Resolve a media payload according to the configured read layout.
|
||||
///
|
||||
/// Dual-read phases prefer the sharded key and fall back to legacy only on
|
||||
/// an actual not-found result. Authorization, transport, throttling, and
|
||||
/// service errors are returned so fallback cannot mask an unhealthy store.
|
||||
/// The dual-read-and-write phase prefers the sharded key and falls back to
|
||||
/// legacy only on an actual not-found result. Authorization, transport,
|
||||
/// throttling, and service errors are returned so fallback cannot mask an
|
||||
/// unhealthy store.
|
||||
pub async fn resolve_read_key(
|
||||
&self,
|
||||
ctx: &TenantContext,
|
||||
|
||||
@@ -1203,17 +1203,17 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn media_migration_phase_env_accepts_sharded_only_and_rejects_invalid_values() {
|
||||
fn media_migration_phase_env_accepts_dual_read_and_write_and_rejects_removed_phase() {
|
||||
let _guard = ENV_MUTEX.lock().unwrap();
|
||||
let previous = std::env::var_os("BUZZ_MEDIA_MIGRATION_PHASE");
|
||||
|
||||
std::env::set_var("BUZZ_MEDIA_MIGRATION_PHASE", "sharded-only");
|
||||
std::env::set_var("BUZZ_MEDIA_MIGRATION_PHASE", "dual-read-and-write");
|
||||
let configured = Config::from_env()
|
||||
.expect("sharded-only media config")
|
||||
.expect("dual-read-and-write media config")
|
||||
.media
|
||||
.migration_phase;
|
||||
|
||||
std::env::set_var("BUZZ_MEDIA_MIGRATION_PHASE", "auto");
|
||||
std::env::set_var("BUZZ_MEDIA_MIGRATION_PHASE", "dual-read-legacy-write");
|
||||
let invalid = Config::from_env();
|
||||
|
||||
if let Some(value) = previous {
|
||||
@@ -1222,7 +1222,10 @@ mod tests {
|
||||
std::env::remove_var("BUZZ_MEDIA_MIGRATION_PHASE");
|
||||
}
|
||||
|
||||
assert_eq!(configured, buzz_media::MediaMigrationPhase::ShardedOnly);
|
||||
assert_eq!(
|
||||
configured,
|
||||
buzz_media::MediaMigrationPhase::DualReadAndWrite
|
||||
);
|
||||
assert!(matches!(
|
||||
invalid,
|
||||
Err(ConfigError::InvalidValue(ref message))
|
||||
|
||||
@@ -107,8 +107,7 @@ The supported modes are:
|
||||
| Mode | Reads | Writes | Use |
|
||||
|---|---|---|---|
|
||||
| `legacy-only` (default) | Legacy only | Legacy only | Behavior-preserving upgrades and deployments that are not migrating yet |
|
||||
| `dual-read-legacy-write` | Sharded first, legacy fallback | Legacy only | Deploy compatibility readers before changing the write layout |
|
||||
| `dual-read-dual-write` | Sharded first, legacy fallback | Sharded and legacy | Migration and rollback window |
|
||||
| `dual-read-and-write` | Sharded first, legacy fallback | Sharded and legacy | Migration, backfill, and rollback window |
|
||||
| `sharded-only` | Sharded only | Sharded only | New deployments and completed migrations |
|
||||
|
||||
**New deployments should start with `sharded-only` before accepting the first
|
||||
@@ -118,7 +117,7 @@ eliminates the later backfill and cleanup migration.
|
||||
Existing deployments should keep the `legacy-only` default until an operator
|
||||
intentionally starts the staged migration. Do not roll `sharded-only` writers
|
||||
back to a Buzz version that predates sharded reads. Returning from
|
||||
`sharded-only` to `dual-read-dual-write` does not retroactively recreate legacy
|
||||
`sharded-only` to `dual-read-and-write` does not retroactively recreate legacy
|
||||
copies; run and verify the backfill before relying on old-version rollback.
|
||||
|
||||
## Relay Pod extensions
|
||||
@@ -307,16 +306,15 @@ For an **existing deployment**:
|
||||
|
||||
1. Upgrade with the default `legacy-only`. This is behavior-preserving: the
|
||||
relay continues reading and writing only flat legacy keys.
|
||||
2. Select `dual-read-legacy-write`; confirm every relay can read both layouts.
|
||||
Writes remain legacy-only, so this step creates no duplicate objects.
|
||||
3. Select `dual-read-dual-write` and observe successful writes and storage
|
||||
telemetry through a rollback window.
|
||||
4. Run the backfill Job. Re-run from any logged checkpoint as needed, then run
|
||||
2. Select `dual-read-and-write`; confirm every relay can read both layouts, and
|
||||
observe successful dual writes and storage telemetry through a rollback
|
||||
window.
|
||||
3. Run the backfill Job. Re-run from any logged checkpoint as needed, then run
|
||||
it again to a clean `copied=0` result. Reconcile storage sweep unknown keys,
|
||||
duplicate gauges, migration failures, and legacy fallback traffic.
|
||||
5. Select `sharded-only` only after every supported rollback version understands
|
||||
4. Select `sharded-only` only after every supported rollback version understands
|
||||
sharded keys and reconciliation finds no missing sharded destination.
|
||||
6. Run the deletion Job with its default `dry-run=true`, review the output, and
|
||||
5. Run the deletion Job with its default `dry-run=true`, review the output, and
|
||||
retain a recovery window/S3 versions. Only then set `dry-run=false` and
|
||||
`BUZZ_MEDIA_DELETE_CONFIRM=delete-verified-legacy-media`. The tool checks the
|
||||
corresponding sharded object immediately before every deletion.
|
||||
|
||||
@@ -65,6 +65,24 @@ tests:
|
||||
value: "true"
|
||||
template: templates/deployment.yaml
|
||||
|
||||
- it: renders the dual-read-and-write migration phase
|
||||
set:
|
||||
relayUrl: wss://buzz.example.com
|
||||
ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000"
|
||||
externalPostgresql.url: postgres://u:p@h:5432/d
|
||||
externalRedis.url: redis://h:6379
|
||||
s3.endpoint: http://minio:9000
|
||||
s3.accessKey: a
|
||||
s3.secretKey: s
|
||||
relay.mediaMigrationPhase: dual-read-and-write
|
||||
asserts:
|
||||
- contains:
|
||||
path: spec.template.spec.containers[0].env
|
||||
content:
|
||||
name: BUZZ_MEDIA_MIGRATION_PHASE
|
||||
value: "dual-read-and-write"
|
||||
template: templates/deployment.yaml
|
||||
|
||||
- it: lets a new deployment opt directly into sharded-only media
|
||||
set:
|
||||
relayUrl: wss://buzz.example.com
|
||||
|
||||
@@ -63,7 +63,7 @@
|
||||
"requireAuthToken": { "type": "boolean" },
|
||||
"requireRelayMembership": { "type": "boolean" },
|
||||
"requireMediaGetAuth": { "type": "boolean" },
|
||||
"mediaMigrationPhase": { "type": "string", "enum": ["legacy-only", "dual-read-legacy-write", "dual-read-dual-write", "sharded-only"] },
|
||||
"mediaMigrationPhase": { "type": "string", "enum": ["legacy-only", "dual-read-and-write", "sharded-only"] },
|
||||
"allowNipOaAuth": { "type": "boolean" },
|
||||
"huddleAudioAvailable": {
|
||||
"type": ["boolean", "null"],
|
||||
|
||||
@@ -123,9 +123,9 @@ relay:
|
||||
# local development or fully public communities — desktop, mobile, and CLI
|
||||
# clients all attach read auth.
|
||||
requireMediaGetAuth: true
|
||||
# Behavior-preserving upgrade default: legacy-only reads and writes. Existing deployments migrate through
|
||||
# legacy-only -> dual-read-legacy-write -> dual-read-dual-write -> sharded-only. New deployments should override
|
||||
# this to sharded-only before accepting their first upload so they never need a backfill.
|
||||
# Behavior-preserving upgrade default: legacy-only reads and writes. Existing deployments use
|
||||
# legacy-only -> dual-read-and-write -> backfill -> sharded-only. New deployments should override this to
|
||||
# sharded-only before accepting their first upload so they never need a backfill.
|
||||
mediaMigrationPhase: legacy-only
|
||||
allowNipOaAuth: true
|
||||
pubkeyAllowlist: false
|
||||
|
||||
@@ -36,9 +36,9 @@ BUZZ_S3_BUCKET=buzz-media
|
||||
# Bundled MinIO uses path-style URLs; deploy/compose/compose.yml pins this.
|
||||
BUZZ_S3_ADDRESSING_STYLE=path
|
||||
# Behavior-preserving upgrade default: read and write only legacy keys. Existing
|
||||
# deployments migrate explicitly through legacy-only -> dual-read-legacy-write ->
|
||||
# dual-read-dual-write -> sharded-only. New deployments should select sharded-only
|
||||
# before their first upload so they never need a backfill.
|
||||
# deployments use legacy-only -> dual-read-and-write -> backfill -> sharded-only.
|
||||
# New deployments should select sharded-only before their first upload so they
|
||||
# never need a backfill.
|
||||
BUZZ_MEDIA_MIGRATION_PHASE=legacy-only
|
||||
|
||||
# Optional host ports. Base compose publishes the relay directly on BUZZ_HTTP_PORT.
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
# Copy this example, pin the same image tag as the relay, and supply its S3
|
||||
# Run this while relays are in dual-read-and-write, before switching them to
|
||||
# sharded-only. Pin the same image tag as the relay and supply its S3
|
||||
# environment/IRSA. Re-running is safe: existing sharded objects are skipped.
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
|
||||
Reference in New Issue
Block a user