mirror of
https://github.com/block/buzz.git
synced 2026-11-06 01:11:33 +00:00
## Summary - add one strict `BUZZ_S3_ADDRESSING_STYLE=path|virtual` setting shared by media and Git/CAS storage - preserve path-style defaults for bundled Compose/Helm MinIO while supporting Railway's virtual-hosted bucket contract - fail startup on invalid or non-Unicode values before dependency connection, and validate the Helm value with the same two choices - document operator mappings and why endpoint and bucket remain separate for routing and SigV4 signing ## Best-practice rationale AWS documents both URL forms and favors virtual-hosted addressing for S3, while compatibility endpoints such as the bundled MinIO deployment can require path style. `rust-s3` defaults to virtual/subdomain addressing and provides `with_path_style()` for the explicit compatibility case. Some providers buckets only support as virtual-hosted bucket styles. This PR therefore uses one explicit, provider-neutral switch rather than endpoint heuristics or fallback behavior, while retaining `path` as Buzz's backward-compatible default. Sources: - https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html - https://docs.rs/rust-s3/0.37.0/s3/bucket/struct.Bucket.html - https://docs.railway.com/storage-buckets#url-style - https://github.com/minio/minio/blob/master/docs/config/README.md#domain ## Validation - `cargo fmt --all` - `cargo check --workspace --all-targets` - targeted `buzz-media` and `buzz-relay` parsing/client-construction tests for defaults, strict errors, and both URL styles - Helm unittest: 45/45 passed - Compose config/render validation passed - local MinIO path-mode relay startup passed the Git A3 conformance probe and became ready - unreachable object storage failed startup and readiness never opened - push hooks completed the broader Rust and desktop suites successfully --------- Signed-off-by: npub122y0pqkertljmedu303rl0aqrj3w8pvu43t6jxm6875lzg6f2pwqegc3xc <5288f082d91aff2de5bc8be23fbfa01ca2e3859cac57a91b7a3fa9f12349505c@buzz.block.builderlab.xyz> Signed-off-by: Kalvin Chau <kalvin@block.xyz> Co-authored-by: npub122y0pqkertljmedu303rl0aqrj3w8pvu43t6jxm6875lzg6f2pwqegc3xc <5288f082d91aff2de5bc8be23fbfa01ca2e3859cac57a91b7a3fa9f12349505c@buzz.block.builderlab.xyz>
255 lines
9.2 KiB
Rust
255 lines
9.2 KiB
Rust
//! Media storage configuration.
|
|
|
|
use std::str::FromStr;
|
|
|
|
/// S3 URL addressing style shared by media and Git/CAS storage.
|
|
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Deserialize)]
|
|
#[serde(rename_all = "lowercase")]
|
|
pub enum S3AddressingStyle {
|
|
/// Put the bucket in the request path (`https://endpoint/bucket/key`).
|
|
///
|
|
/// This preserves compatibility with the bundled MinIO deployments, whose
|
|
/// internal DNS only resolves the endpoint hostname.
|
|
#[default]
|
|
Path,
|
|
/// Put the bucket in the hostname (`https://bucket.endpoint/key`).
|
|
///
|
|
/// This is the standard S3 form and is required by providers such as new
|
|
/// Railway Storage Buckets.
|
|
Virtual,
|
|
}
|
|
|
|
impl FromStr for S3AddressingStyle {
|
|
type Err = String;
|
|
|
|
fn from_str(value: &str) -> Result<Self, Self::Err> {
|
|
match value {
|
|
"path" => Ok(Self::Path),
|
|
"virtual" => Ok(Self::Virtual),
|
|
_ => Err(format!(
|
|
"BUZZ_S3_ADDRESSING_STYLE must be 'path' or 'virtual', got {value:?}"
|
|
)),
|
|
}
|
|
}
|
|
}
|
|
|
|
fn default_max_video_bytes() -> u64 {
|
|
524_288_000 // 500 MB
|
|
}
|
|
|
|
fn default_max_file_bytes() -> u64 {
|
|
104_857_600 // 100 MB
|
|
}
|
|
|
|
fn default_s3_region() -> String {
|
|
"us-east-1".to_string()
|
|
}
|
|
|
|
/// Configuration for media storage (S3/MinIO).
|
|
#[derive(Debug, Clone, serde::Deserialize)]
|
|
pub struct MediaConfig {
|
|
/// S3-compatible endpoint URL (e.g. "http://localhost:9000").
|
|
pub s3_endpoint: String,
|
|
/// S3 access key.
|
|
pub s3_access_key: String,
|
|
/// S3 secret key.
|
|
pub s3_secret_key: String,
|
|
/// S3 bucket name.
|
|
pub s3_bucket: String,
|
|
/// AWS region for SigV4 request signing (e.g. "us-west-2").
|
|
///
|
|
/// Must match the region of `s3_endpoint` for real AWS S3, otherwise
|
|
/// requests are signed with the wrong credential scope and AWS rejects
|
|
/// them. Defaults to "us-east-1" to preserve MinIO/local behavior, where
|
|
/// the value is not meaningfully checked.
|
|
#[serde(default = "default_s3_region")]
|
|
pub s3_region: String,
|
|
/// S3 URL addressing style. Defaults to path style for MinIO compatibility.
|
|
#[serde(default)]
|
|
pub s3_addressing_style: S3AddressingStyle,
|
|
/// Maximum upload size for images (bytes). Default: 50 MB.
|
|
pub max_image_bytes: u64,
|
|
/// Maximum upload size for animated GIFs (bytes). Default: 10 MB.
|
|
pub max_gif_bytes: u64,
|
|
/// Maximum upload size for video files (bytes). Default: 500 MB.
|
|
#[serde(default = "default_max_video_bytes")]
|
|
pub max_video_bytes: u64,
|
|
/// Maximum upload size for generic (non-image, non-video) files (bytes). Default: 100 MB.
|
|
#[serde(default = "default_max_file_bytes")]
|
|
pub max_file_bytes: u64,
|
|
/// Public base URL for media URLs in BlobDescriptor (must include `/media` path).
|
|
pub public_base_url: String,
|
|
/// Whether to write per-upload-event records under `_uploads/`
|
|
/// (moderation side channel). Off by default; set via
|
|
/// `BUZZ_MEDIA_UPLOAD_RECORDS=true`.
|
|
#[serde(default)]
|
|
pub upload_records_enabled: bool,
|
|
/// Trusted edge header to read the uploader's public IP from (e.g.
|
|
/// `cf-connecting-ip`). Unset (default) → no IP is read or recorded.
|
|
/// Only consulted when `upload_records_enabled` is true; the value is
|
|
/// validated as a public IP and dropped otherwise (fail-empty).
|
|
#[serde(default)]
|
|
pub upload_ip_header: Option<String>,
|
|
/// Trusted edge header to read the uploader's source port from. Standard
|
|
/// edges don't emit one, so this is usually unset; a port is only
|
|
/// recorded alongside a valid IP.
|
|
#[serde(default)]
|
|
pub upload_port_header: Option<String>,
|
|
}
|
|
|
|
impl MediaConfig {
|
|
/// Validate configuration at startup. Returns an error on invalid config.
|
|
pub fn validate(&self) -> Result<(), String> {
|
|
if !self.public_base_url.ends_with("/media") {
|
|
return Err(format!(
|
|
"public_base_url must end with /media: got '{}'",
|
|
self.public_base_url
|
|
));
|
|
}
|
|
if self.public_base_url.ends_with('/') {
|
|
return Err(format!(
|
|
"public_base_url must not end with /: got '{}'",
|
|
self.public_base_url
|
|
));
|
|
}
|
|
if self.max_image_bytes == 0 {
|
|
return Err("max_image_bytes must be > 0".to_string());
|
|
}
|
|
if self.max_gif_bytes == 0 || self.max_gif_bytes > self.max_image_bytes {
|
|
return Err("max_gif_bytes must be > 0 and <= max_image_bytes".to_string());
|
|
}
|
|
if self.max_video_bytes == 0 {
|
|
return Err("max_video_bytes must be > 0".to_string());
|
|
}
|
|
if self.max_file_bytes == 0 {
|
|
return Err("max_file_bytes must be > 0".to_string());
|
|
}
|
|
// Fail startup on incoherent collection config instead of silently
|
|
// recording nothing — an operator who set an IP header believes they
|
|
// are meeting a reporting obligation.
|
|
if self.upload_ip_header.is_some() && !self.upload_records_enabled {
|
|
return Err(
|
|
"BUZZ_MEDIA_UPLOAD_IP_HEADER is set but BUZZ_MEDIA_UPLOAD_RECORDS is not \
|
|
enabled — the IP would never be recorded. Enable upload records or unset \
|
|
the header."
|
|
.to_string(),
|
|
);
|
|
}
|
|
if self.upload_port_header.is_some() && self.upload_ip_header.is_none() {
|
|
return Err(
|
|
"BUZZ_MEDIA_UPLOAD_PORT_HEADER is set without BUZZ_MEDIA_UPLOAD_IP_HEADER — \
|
|
a port is only recorded alongside an IP. Set the IP header or unset the \
|
|
port header."
|
|
.to_string(),
|
|
);
|
|
}
|
|
for (name, value) in [
|
|
("BUZZ_MEDIA_UPLOAD_IP_HEADER", &self.upload_ip_header),
|
|
("BUZZ_MEDIA_UPLOAD_PORT_HEADER", &self.upload_port_header),
|
|
] {
|
|
if let Some(h) = value {
|
|
if axum::http::HeaderName::from_bytes(h.as_bytes()).is_err() {
|
|
return Err(format!("{name} is not a valid header name: {h:?}"));
|
|
}
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::{MediaConfig, S3AddressingStyle};
|
|
use std::str::FromStr;
|
|
|
|
fn valid_config() -> MediaConfig {
|
|
MediaConfig {
|
|
s3_endpoint: "http://localhost:9000".to_string(),
|
|
s3_access_key: "k".to_string(),
|
|
s3_secret_key: "s".to_string(),
|
|
s3_bucket: "buzz-media".to_string(),
|
|
s3_region: "us-east-1".to_string(),
|
|
s3_addressing_style: S3AddressingStyle::Path,
|
|
max_image_bytes: 1,
|
|
max_gif_bytes: 1,
|
|
max_video_bytes: 1,
|
|
max_file_bytes: 1,
|
|
public_base_url: "http://localhost:3000/media".to_string(),
|
|
upload_records_enabled: false,
|
|
upload_ip_header: None,
|
|
upload_port_header: None,
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn addressing_style_parses_supported_values() {
|
|
assert_eq!(
|
|
S3AddressingStyle::from_str("path"),
|
|
Ok(S3AddressingStyle::Path)
|
|
);
|
|
assert_eq!(
|
|
S3AddressingStyle::from_str("virtual"),
|
|
Ok(S3AddressingStyle::Virtual)
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn addressing_style_defaults_to_path() {
|
|
assert_eq!(S3AddressingStyle::default(), S3AddressingStyle::Path);
|
|
}
|
|
|
|
#[test]
|
|
fn addressing_style_rejects_unknown_or_ambiguous_values() {
|
|
for invalid in ["", "auto", "PATH", "virtual-hosted"] {
|
|
let error =
|
|
S3AddressingStyle::from_str(invalid).expect_err("must reject invalid style");
|
|
assert!(
|
|
error.contains("BUZZ_S3_ADDRESSING_STYLE must be 'path' or 'virtual'"),
|
|
"unexpected error for {invalid:?}: {error}"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn upload_record_knobs_default_off_and_validate() {
|
|
assert!(valid_config().validate().is_ok());
|
|
|
|
let mut on = valid_config();
|
|
on.upload_records_enabled = true;
|
|
assert!(on.validate().is_ok());
|
|
|
|
on.upload_ip_header = Some("cf-connecting-ip".to_string());
|
|
assert!(on.validate().is_ok());
|
|
|
|
on.upload_port_header = Some("x-client-port".to_string());
|
|
assert!(on.validate().is_ok());
|
|
}
|
|
|
|
#[test]
|
|
fn ip_header_without_records_fails_startup() {
|
|
// An operator who set the header believes IPs are being recorded —
|
|
// fail loudly instead of silently collecting nothing.
|
|
let mut cfg = valid_config();
|
|
cfg.upload_ip_header = Some("cf-connecting-ip".to_string());
|
|
assert!(cfg.validate().is_err());
|
|
}
|
|
|
|
#[test]
|
|
fn port_header_without_ip_header_fails_startup() {
|
|
let mut cfg = valid_config();
|
|
cfg.upload_records_enabled = true;
|
|
cfg.upload_port_header = Some("x-client-port".to_string());
|
|
assert!(cfg.validate().is_err());
|
|
}
|
|
|
|
#[test]
|
|
fn malformed_header_names_fail_startup() {
|
|
let mut cfg = valid_config();
|
|
cfg.upload_records_enabled = true;
|
|
for bad in ["with space", "colon:name", "bad/header", "bad,header", ""] {
|
|
cfg.upload_ip_header = Some(bad.to_string());
|
|
assert!(cfg.validate().is_err(), "should reject header name {bad:?}");
|
|
}
|
|
}
|
|
}
|