Files
buzz/crates/buzz-media/src/config.rs
Kalvin C 7012d86d52 feat: configure S3 URL addressing style (#3400)
## Summary

- add one strict `BUZZ_S3_ADDRESSING_STYLE=path|virtual` setting shared
by media and Git/CAS storage
- preserve path-style defaults for bundled Compose/Helm MinIO while
supporting Railway's virtual-hosted bucket contract
- fail startup on invalid or non-Unicode values before dependency
connection, and validate the Helm value with the same two choices
- document operator mappings and why endpoint and bucket remain separate
for routing and SigV4 signing

## Best-practice rationale

AWS documents both URL forms and favors virtual-hosted addressing for
S3, while compatibility endpoints such as the bundled MinIO deployment
can require path style. `rust-s3` defaults to virtual/subdomain
addressing and provides `with_path_style()` for the explicit
compatibility case.

Some providers buckets only support as virtual-hosted bucket styles.
This PR therefore uses one explicit, provider-neutral switch rather than
endpoint heuristics or fallback behavior, while retaining `path` as
Buzz's backward-compatible default.

Sources:
-
https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html
- https://docs.rs/rust-s3/0.37.0/s3/bucket/struct.Bucket.html
- https://docs.railway.com/storage-buckets#url-style
-
https://github.com/minio/minio/blob/master/docs/config/README.md#domain

## Validation

- `cargo fmt --all`
- `cargo check --workspace --all-targets`
- targeted `buzz-media` and `buzz-relay` parsing/client-construction
tests for defaults, strict errors, and both URL styles
- Helm unittest: 45/45 passed
- Compose config/render validation passed
- local MinIO path-mode relay startup passed the Git A3 conformance
probe and became ready
- unreachable object storage failed startup and readiness never opened
- push hooks completed the broader Rust and desktop suites successfully

---------

Signed-off-by: npub122y0pqkertljmedu303rl0aqrj3w8pvu43t6jxm6875lzg6f2pwqegc3xc <5288f082d91aff2de5bc8be23fbfa01ca2e3859cac57a91b7a3fa9f12349505c@buzz.block.builderlab.xyz>
Signed-off-by: Kalvin Chau <kalvin@block.xyz>
Co-authored-by: npub122y0pqkertljmedu303rl0aqrj3w8pvu43t6jxm6875lzg6f2pwqegc3xc <5288f082d91aff2de5bc8be23fbfa01ca2e3859cac57a91b7a3fa9f12349505c@buzz.block.builderlab.xyz>
2026-07-29 17:00:41 -07:00

255 lines
9.2 KiB
Rust

//! Media storage configuration.
use std::str::FromStr;
/// S3 URL addressing style shared by media and Git/CAS storage.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum S3AddressingStyle {
/// Put the bucket in the request path (`https://endpoint/bucket/key`).
///
/// This preserves compatibility with the bundled MinIO deployments, whose
/// internal DNS only resolves the endpoint hostname.
#[default]
Path,
/// Put the bucket in the hostname (`https://bucket.endpoint/key`).
///
/// This is the standard S3 form and is required by providers such as new
/// Railway Storage Buckets.
Virtual,
}
impl FromStr for S3AddressingStyle {
type Err = String;
fn from_str(value: &str) -> Result<Self, Self::Err> {
match value {
"path" => Ok(Self::Path),
"virtual" => Ok(Self::Virtual),
_ => Err(format!(
"BUZZ_S3_ADDRESSING_STYLE must be 'path' or 'virtual', got {value:?}"
)),
}
}
}
fn default_max_video_bytes() -> u64 {
524_288_000 // 500 MB
}
fn default_max_file_bytes() -> u64 {
104_857_600 // 100 MB
}
fn default_s3_region() -> String {
"us-east-1".to_string()
}
/// Configuration for media storage (S3/MinIO).
#[derive(Debug, Clone, serde::Deserialize)]
pub struct MediaConfig {
/// S3-compatible endpoint URL (e.g. "http://localhost:9000").
pub s3_endpoint: String,
/// S3 access key.
pub s3_access_key: String,
/// S3 secret key.
pub s3_secret_key: String,
/// S3 bucket name.
pub s3_bucket: String,
/// AWS region for SigV4 request signing (e.g. "us-west-2").
///
/// Must match the region of `s3_endpoint` for real AWS S3, otherwise
/// requests are signed with the wrong credential scope and AWS rejects
/// them. Defaults to "us-east-1" to preserve MinIO/local behavior, where
/// the value is not meaningfully checked.
#[serde(default = "default_s3_region")]
pub s3_region: String,
/// S3 URL addressing style. Defaults to path style for MinIO compatibility.
#[serde(default)]
pub s3_addressing_style: S3AddressingStyle,
/// Maximum upload size for images (bytes). Default: 50 MB.
pub max_image_bytes: u64,
/// Maximum upload size for animated GIFs (bytes). Default: 10 MB.
pub max_gif_bytes: u64,
/// Maximum upload size for video files (bytes). Default: 500 MB.
#[serde(default = "default_max_video_bytes")]
pub max_video_bytes: u64,
/// Maximum upload size for generic (non-image, non-video) files (bytes). Default: 100 MB.
#[serde(default = "default_max_file_bytes")]
pub max_file_bytes: u64,
/// Public base URL for media URLs in BlobDescriptor (must include `/media` path).
pub public_base_url: String,
/// Whether to write per-upload-event records under `_uploads/`
/// (moderation side channel). Off by default; set via
/// `BUZZ_MEDIA_UPLOAD_RECORDS=true`.
#[serde(default)]
pub upload_records_enabled: bool,
/// Trusted edge header to read the uploader's public IP from (e.g.
/// `cf-connecting-ip`). Unset (default) → no IP is read or recorded.
/// Only consulted when `upload_records_enabled` is true; the value is
/// validated as a public IP and dropped otherwise (fail-empty).
#[serde(default)]
pub upload_ip_header: Option<String>,
/// Trusted edge header to read the uploader's source port from. Standard
/// edges don't emit one, so this is usually unset; a port is only
/// recorded alongside a valid IP.
#[serde(default)]
pub upload_port_header: Option<String>,
}
impl MediaConfig {
/// Validate configuration at startup. Returns an error on invalid config.
pub fn validate(&self) -> Result<(), String> {
if !self.public_base_url.ends_with("/media") {
return Err(format!(
"public_base_url must end with /media: got '{}'",
self.public_base_url
));
}
if self.public_base_url.ends_with('/') {
return Err(format!(
"public_base_url must not end with /: got '{}'",
self.public_base_url
));
}
if self.max_image_bytes == 0 {
return Err("max_image_bytes must be > 0".to_string());
}
if self.max_gif_bytes == 0 || self.max_gif_bytes > self.max_image_bytes {
return Err("max_gif_bytes must be > 0 and <= max_image_bytes".to_string());
}
if self.max_video_bytes == 0 {
return Err("max_video_bytes must be > 0".to_string());
}
if self.max_file_bytes == 0 {
return Err("max_file_bytes must be > 0".to_string());
}
// Fail startup on incoherent collection config instead of silently
// recording nothing — an operator who set an IP header believes they
// are meeting a reporting obligation.
if self.upload_ip_header.is_some() && !self.upload_records_enabled {
return Err(
"BUZZ_MEDIA_UPLOAD_IP_HEADER is set but BUZZ_MEDIA_UPLOAD_RECORDS is not \
enabled — the IP would never be recorded. Enable upload records or unset \
the header."
.to_string(),
);
}
if self.upload_port_header.is_some() && self.upload_ip_header.is_none() {
return Err(
"BUZZ_MEDIA_UPLOAD_PORT_HEADER is set without BUZZ_MEDIA_UPLOAD_IP_HEADER — \
a port is only recorded alongside an IP. Set the IP header or unset the \
port header."
.to_string(),
);
}
for (name, value) in [
("BUZZ_MEDIA_UPLOAD_IP_HEADER", &self.upload_ip_header),
("BUZZ_MEDIA_UPLOAD_PORT_HEADER", &self.upload_port_header),
] {
if let Some(h) = value {
if axum::http::HeaderName::from_bytes(h.as_bytes()).is_err() {
return Err(format!("{name} is not a valid header name: {h:?}"));
}
}
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::{MediaConfig, S3AddressingStyle};
use std::str::FromStr;
fn valid_config() -> MediaConfig {
MediaConfig {
s3_endpoint: "http://localhost:9000".to_string(),
s3_access_key: "k".to_string(),
s3_secret_key: "s".to_string(),
s3_bucket: "buzz-media".to_string(),
s3_region: "us-east-1".to_string(),
s3_addressing_style: S3AddressingStyle::Path,
max_image_bytes: 1,
max_gif_bytes: 1,
max_video_bytes: 1,
max_file_bytes: 1,
public_base_url: "http://localhost:3000/media".to_string(),
upload_records_enabled: false,
upload_ip_header: None,
upload_port_header: None,
}
}
#[test]
fn addressing_style_parses_supported_values() {
assert_eq!(
S3AddressingStyle::from_str("path"),
Ok(S3AddressingStyle::Path)
);
assert_eq!(
S3AddressingStyle::from_str("virtual"),
Ok(S3AddressingStyle::Virtual)
);
}
#[test]
fn addressing_style_defaults_to_path() {
assert_eq!(S3AddressingStyle::default(), S3AddressingStyle::Path);
}
#[test]
fn addressing_style_rejects_unknown_or_ambiguous_values() {
for invalid in ["", "auto", "PATH", "virtual-hosted"] {
let error =
S3AddressingStyle::from_str(invalid).expect_err("must reject invalid style");
assert!(
error.contains("BUZZ_S3_ADDRESSING_STYLE must be 'path' or 'virtual'"),
"unexpected error for {invalid:?}: {error}"
);
}
}
#[test]
fn upload_record_knobs_default_off_and_validate() {
assert!(valid_config().validate().is_ok());
let mut on = valid_config();
on.upload_records_enabled = true;
assert!(on.validate().is_ok());
on.upload_ip_header = Some("cf-connecting-ip".to_string());
assert!(on.validate().is_ok());
on.upload_port_header = Some("x-client-port".to_string());
assert!(on.validate().is_ok());
}
#[test]
fn ip_header_without_records_fails_startup() {
// An operator who set the header believes IPs are being recorded —
// fail loudly instead of silently collecting nothing.
let mut cfg = valid_config();
cfg.upload_ip_header = Some("cf-connecting-ip".to_string());
assert!(cfg.validate().is_err());
}
#[test]
fn port_header_without_ip_header_fails_startup() {
let mut cfg = valid_config();
cfg.upload_records_enabled = true;
cfg.upload_port_header = Some("x-client-port".to_string());
assert!(cfg.validate().is_err());
}
#[test]
fn malformed_header_names_fail_startup() {
let mut cfg = valid_config();
cfg.upload_records_enabled = true;
for bad in ["with space", "colon:name", "bad/header", "bad,header", ""] {
cfg.upload_ip_header = Some(bad.to_string());
assert!(cfg.validate().is_err(), "should reject header name {bad:?}");
}
}
}