jerry/buzz
1
0
Fork 0
mirror of https://github.com/block/buzz.git synced 2026-10-27 12:40:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(media): transcode HEIC/HEIF to JPEG on desktop upload (#1257)

Browse Source 
Signed-off-by: Taylor Ho <taylorkmho@gmail.com>
Co-authored-by: npub1223z34hd7vtwc6qj4s7flsxkj644nlre2nthu7lrrmkumhu3xddsrx9r6w <52a228d6edf316ec6812ac3c9fc0d696ab59fc7954d77e7be31eedcddf91335b@sprout-oss.stage.blox.sqprod.co>
This commit is contained in:
Taylor Ho
2026-06-26 09:34:26 -07:00
committed by GitHub
parent 1843a05747
commit d32f3c0a58
2 changed files with 292 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
desktop/src-tauri/src/commands/media.rs
View File

@@ -10,7 +10,10 @@ use crate::relay::{
relay_error_message,
};
use super::media_transcode::{is_video_file, transcode_and_extract_poster};
use super::media_transcode::{
has_heic_extension, is_heic_file, is_video_file, transcode_and_extract_poster,
transcode_heic_path_to_jpeg_bytes,
};
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BlobDescriptor {
@@ -305,6 +308,12 @@ async fn process_picked_path(
// local attacker from swapping the file between dialog return and read.
let mut file = std::fs::File::open(&path).map_err(|e| e.to_string())?;
// Extension hint for HEIC detection — some HEIC files from non-Apple
// tooling carry brands outside HEIC_BRANDS, but the `.heic`/`.heif`
// extension still tells us the webview can't render them. Computed before
// the closure since `path` isn't moved in.
let heic_by_ext = has_heic_extension(&path);
// All sync I/O (sniff, transcode, read) runs off the async runtime to
// avoid blocking Tokio worker threads during long ffmpeg transcodes.
let (body, poster_bytes) =
@@ -326,6 +335,15 @@ async fn process_picked_path(
let result = transcode_and_extract_poster(&fd_path);
drop(file); // release fd only after ffmpeg is done
result
} else if heic_by_ext || is_heic_file(&header[..n]) {
// HEIC/HEIF still: Chromium/the webview can't decode it, so
// transcode to JPEG before upload (mirrors mobile). Resolve the
// fd's real path so ffmpeg reads the pinned inode, and keep
// `file` alive until the transcode finishes.
let fd_path = fd_real_path(&file)?;
let result = transcode_heic_path_to_jpeg_bytes(&fd_path).map(|jpeg| (jpeg, None));
drop(file); // release fd only after ffmpeg is done
result
} else {
// Image: read the rest from the already-open fd (TOCTOU-safe).
let mut bytes = header[..n].to_vec();
@@ -437,6 +455,24 @@ pub async fn upload_media_bytes(
})
.await
.map_err(|e| format!("transcode task failed: {e}"))??
} else if is_heic_file(&data) {
// HEIC/HEIF still pasted/dropped: no filename here, so detection is
// magic-bytes only. ffmpeg needs a path, so write to temp, transcode
// to JPEG, and clean up. (Mirrors mobile's pre-upload transcode.)
tokio::task::spawn_blocking(move || -> Result<(Vec<u8>, Option<Vec<u8>>), String> {
let tmp_input =
std::env::temp_dir().join(format!("buzz-drop-{}", uuid::Uuid::new_v4()));
// Cleanup guard: remove temp file on ALL exit paths (including write failure).
let result = (|| {
std::fs::write(&tmp_input, &data)
.map_err(|e| format!("failed to write temp file: {e}"))?;
transcode_heic_path_to_jpeg_bytes(&tmp_input).map(|jpeg| (jpeg, None))
})();
let _ = std::fs::remove_file(&tmp_input);
result
})
.await
.map_err(|e| format!("transcode task failed: {e}"))??
} else {
(data, None)
};

255
desktop/src-tauri/src/commands/media_transcode.rs
View File

@@ -40,6 +40,60 @@ pub(super) fn is_video_file(buf: &[u8]) -> bool {
infer::get(buf).is_some_and(|t| t.mime_type().starts_with("video/"))
}
/// HEIC/HEIF compatible-brand codes that mark an ISO-BMFF file as a still
/// HEIF image. Mirrors mobile's `_heicBrands` set in
/// `mobile/lib/shared/relay/media_upload.dart` so detection stays consistent
/// across platforms — deliberately broader than the `infer` crate, which only
/// recognizes `heic`/`heix` majors (or `mif1`/`msf1` with a `heic` compatible
/// brand) and would miss `hevc`/`hevx`/`heim`/`heis`.
const HEIC_BRANDS: &[&[u8; 4]] = &[
b"heic", b"heix", b"hevc", b"hevx", b"heim", b"heis", b"mif1", b"msf1",
];
/// Detect a HEIC/HEIF still image by magic bytes.
///
/// HEIC/HEIF is an ISO base media file (ISO-BMFF): a `ftyp` box at offset 4
/// followed by a major brand and a list of compatible brands. We scan the
/// major brand plus the compatible-brand list for any of `HEIC_BRANDS`.
///
/// Mirrors mobile's `_looksLikeHeicOrHeif`: requires the `ftyp` marker at
/// offset 4 and scans 4-byte brand codes at offsets 8, 12, 16, ... up to the
/// first 32 bytes. The Tauri webview / Chromium cannot decode HEIC, so any
/// match here is transcoded to JPEG before upload.
pub(super) fn is_heic_file(buf: &[u8]) -> bool {
// Need at least the 8-byte box header + 4-byte major brand.
if buf.len() < 12 || &buf[4..8] != b"ftyp" {
return false;
}
// Scan the major brand (offset 8) and each compatible brand, bounded to
// the first 32 bytes (matches mobile's window).
let upper = buf.len().min(32);
let mut offset = 8;
while offset + 4 <= upper {
let brand: &[u8; 4] = buf[offset..offset + 4].try_into().expect("4-byte slice");
if HEIC_BRANDS.contains(&brand) {
return true;
}
offset += 4;
}
false
}
/// True if a filename ends in `.heic` or `.heif` (case-insensitive).
///
/// Mirrors mobile's `_hasHeicFileExtension`. Used on the file-picker path as a
/// secondary signal — some HEIC files from non-Apple tooling carry brands not
/// in `HEIC_BRANDS`, but the extension still tells us the webview can't render
/// them. The byte-based path (paste/drag) has no filename and relies solely on
/// `is_heic_file`.
pub(super) fn has_heic_extension(path: &std::path::Path) -> bool {
path.extension()
.and_then(|e| e.to_str())
.is_some_and(|ext| ext.eq_ignore_ascii_case("heic") || ext.eq_ignore_ascii_case("heif"))
}
/// Maximum wall-clock time for an ffmpeg transcode before we kill it.
/// 10 minutes is generous for any reasonable video; pathological inputs
/// (crafted to cause exponential decode time) get killed instead of
@@ -154,6 +208,66 @@ pub(super) fn transcode_to_mp4(
Ok(output)
}
/// Transcode a HEIC/HEIF still image to JPEG via ffmpeg.
///
/// The Tauri webview / Chromium cannot decode HEIC, so iPhone photos uploaded
/// as-is render blank in the composer and are unviewable for everyone. This
/// normalizes them to JPEG (the same fix mobile applies before upload).
///
/// Uses `-frames:v 1` so multi-image HEIF containers (Live Photos, bursts)
/// yield a single still, and `-q:v 2` for high JPEG quality. Returns the path
/// to a temp file. Caller must clean up.
pub(super) fn transcode_heic_to_jpeg(
source: &std::path::Path,
ffmpeg: &std::path::Path,
) -> Result<std::path::PathBuf, String> {
// UUID-based temp path — unique across concurrent uploads.
let output = std::env::temp_dir().join(format!("buzz-heic-{}.jpg", uuid::Uuid::new_v4()));
// Single-frame image decode — 60s is generous even for large HEICs.
let heic_timeout = std::time::Duration::from_secs(60);
let result = run_ffmpeg_with_timeout(
std::process::Command::new(ffmpeg)
.args(["-y", "-loglevel", "error"]) // suppress progress spam — prevents stderr pipe deadlock
.arg("-i")
.arg(source) // OsStr — handles non-UTF-8 paths on Unix
.args(["-frames:v", "1", "-q:v", "2"])
.arg(&output)
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::piped()),
heic_timeout,
)?;
if !result.status.success() {
let _ = std::fs::remove_file(&output);
let stderr = String::from_utf8_lossy(&result.stderr);
let detail = stderr
.lines()
.rev()
.find(|l| !l.is_empty() && !l.starts_with(" "))
.unwrap_or("unknown error");
return Err(format!("HEIC conversion failed: {detail}"));
}
Ok(output)
}
/// Transcode a HEIC/HEIF still image (from a path) to JPEG bytes.
///
/// Resolves ffmpeg, transcodes, reads the JPEG bytes, and cleans up the temp
/// file. Mirrors `transcode_and_extract_poster` but for images (no poster).
pub(super) fn transcode_heic_path_to_jpeg_bytes(
source: &std::path::Path,
) -> Result<Vec<u8>, String> {
let ffmpeg_path = find_ffmpeg()?;
let jpeg_path = transcode_heic_to_jpeg(source, &ffmpeg_path)?;
let bytes =
std::fs::read(&jpeg_path).map_err(|e| format!("failed to read transcoded HEIC: {e}"));
let _ = std::fs::remove_file(&jpeg_path);
bytes
}
/// Extract a single JPEG poster frame from a transcoded MP4 via ffmpeg.
///
/// Seeks to 1 second (avoids black leader frames), falls back to first frame
@@ -280,4 +394,145 @@ mod tests {
// It may pass or fail depending on whether ffmpeg is installed.
let _ = find_ffmpeg();
}
/// Build a minimal ISO-BMFF `ftyp` box header with the given major brand
/// and optional compatible brands, suitable for `is_heic_file` testing.
fn ftyp_box(major: &[u8; 4], compatible: &[&[u8; 4]]) -> Vec<u8> {
let mut buf = vec![0x00, 0x00, 0x00, 0x00]; // box size (unused by detector)
buf.extend_from_slice(b"ftyp");
buf.extend_from_slice(major);
buf.extend_from_slice(&[0x00, 0x00, 0x00, 0x00]); // minor version
for brand in compatible {
buf.extend_from_slice(*brand);
}
buf
}
#[test]
fn test_is_heic_file_major_brands() {
// Every brand in HEIC_BRANDS should be detected as the major brand.
for brand in HEIC_BRANDS {
let buf = ftyp_box(brand, &[]);
assert!(is_heic_file(&buf), "major brand {brand:?} not detected");
}
}
#[test]
fn test_is_heic_file_variants_infer_misses() {
// These brands are detected by mobile but NOT by the `infer` crate's
// HEIC heuristic — the whole reason we mirror mobile's full set.
for brand in [b"hevc", b"hevx", b"heim", b"heis"] {
let buf = ftyp_box(brand, &[]);
assert!(is_heic_file(&buf), "variant brand {brand:?} not detected");
}
}
#[test]
fn test_is_heic_file_compatible_brand() {
// Major brand is generic (mif1), HEIC signaled via compatible brand.
let buf = ftyp_box(b"mif1", &[b"heic"]);
assert!(is_heic_file(&buf));
}
#[test]
fn test_is_heic_file_jpeg_is_not_heic() {
let jpeg = [
0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10, b'J', b'F', b'I', b'F', 0x00, 0x01,
];
assert!(!is_heic_file(&jpeg));
}
#[test]
fn test_is_heic_file_mp4_is_not_heic() {
// An MP4 ftyp box (isom) must not be misdetected as HEIC.
let mp4 = ftyp_box(b"isom", &[b"isom", b"iso2"]);
assert!(!is_heic_file(&mp4));
}
#[test]
fn test_is_heic_file_empty() {
assert!(!is_heic_file(&[]));
}
#[test]
fn test_is_heic_file_too_short() {
// Has `ftyp` marker but fewer than 12 bytes — below mobile's threshold.
let buf = [0x00, 0x00, 0x00, 0x00, b'f', b't', b'y', b'p'];
assert!(!is_heic_file(&buf));
}
#[test]
fn test_is_heic_file_no_ftyp_marker() {
// 12+ bytes containing a HEIC brand but no `ftyp` at offset 4.
let mut buf = vec![0u8; 16];
buf[8..12].copy_from_slice(b"heic");
assert!(!is_heic_file(&buf));
}
#[test]
fn test_is_heic_file_brand_past_window() {
// A HEIC brand sitting beyond the 32-byte scan window must not match,
// matching mobile's bounded scan. Use non-HEIC major + filler brands
// so the only HEIC brand present is the one pushed past offset 32.
let mut buf = ftyp_box(b"isom", &[b"iso2", b"iso4", b"avc1", b"mp41", b"mp42"]);
buf.extend_from_slice(b"heic"); // lands at offset 36, past the window
assert!(!is_heic_file(&buf));
}
#[test]
fn test_has_heic_extension() {
use std::path::Path;
assert!(has_heic_extension(Path::new("IMG_1234.HEIC")));
assert!(has_heic_extension(Path::new("photo.heic")));
assert!(has_heic_extension(Path::new("photo.heif")));
assert!(has_heic_extension(Path::new("photo.HEIF")));
assert!(!has_heic_extension(Path::new("photo.jpg")));
assert!(!has_heic_extension(Path::new("photo.png")));
assert!(!has_heic_extension(Path::new("noextension")));
}
/// Round-trip transcode test, gated on ffmpeg being present so CI without
/// ffmpeg doesn't fail. Generates a HEIC via ffmpeg, then transcodes it
/// back to JPEG and asserts the output is a valid JPEG.
#[test]
fn test_transcode_heic_round_trip() {
let Ok(ffmpeg) = find_ffmpeg() else {
eprintln!("skipping HEIC round-trip: ffmpeg not found");
return;
};
// Generate a small HEIC test image from a synthetic color source.
let heic_path =
std::env::temp_dir().join(format!("buzz-test-{}.heic", uuid::Uuid::new_v4()));
let gen = std::process::Command::new(&ffmpeg)
.args(["-y", "-loglevel", "error", "-f", "lavfi", "-i"])
.arg("color=c=red:s=64x64:d=1")
.args(["-frames:v", "1"])
.arg(&heic_path)
.output();
let gen = match gen {
Ok(o) if o.status.success() && heic_path.exists() => o,
other => {
// This ffmpeg build can't encode HEIC — skip rather than fail.
eprintln!("skipping HEIC round-trip: ffmpeg cannot encode HEIC: {other:?}");
let _ = std::fs::remove_file(&heic_path);
return;
}
};
drop(gen);
// Sanity: the generated file should be detected as HEIC.
let heic_bytes = std::fs::read(&heic_path).expect("read generated heic");
assert!(
is_heic_file(&heic_bytes),
"generated file not detected as HEIC"
);
// Transcode to JPEG bytes and verify the JPEG magic.
let jpeg = transcode_heic_path_to_jpeg_bytes(&heic_path).expect("transcode to jpeg");
let _ = std::fs::remove_file(&heic_path);
assert!(jpeg.len() > 2, "empty jpeg output");
assert_eq!(&jpeg[0..2], &[0xFF, 0xD8], "output is not a JPEG");
}
}