mirror of
https://github.com/block/buzz.git
synced 2026-10-27 12:40:30 +00:00
fix(link-preview): resolve YouTube videos through oEmbed (#5520)
**Category:** fix **User Impact:** YouTube video links now resolve into reliable previews instead of intermittently appearing as bare links. **Problem:** Buzz intentionally reads at most **256 KiB** of page HTML when building a generic link preview. The YouTube response that exposed this bug was roughly **1.3 MiB**, with its Open Graph metadata beginning around **686 KiB**—well beyond Buzz's bounded read—so extraction returned no usable preview. YouTube can move that metadata between responses, which explains why the same link may appear to work in one build or request and fail in another; raising the generic cap would increase bandwidth and allocation for every site while still scraping an unstable application document. **Solution:** Route recognized YouTube video URLs through YouTube's structured oEmbed endpoint instead of parsing raw watch-page HTML. The provider response is capped at **64 KiB** and retains Buzz's existing HTTPS validation, pinned DNS/SSRF protection, disabled redirects, timeouts, metadata bounds, and thumbnail sanitization. Provider failures return no preview rather than falling back to fragile HTML scraping, and embed URLs are canonicalized safely, including percent-encoded video IDs. <details> <summary>File changes</summary> **desktop/src-tauri/src/commands/link_preview.rs** Recognizes supported YouTube URL forms, fetches bounded JSON metadata from YouTube oEmbed, canonicalizes embed links, and adds response, URL-boundary, malformed-data, resource-limit, and encoded-ID regressions. **desktop/src-tauri/Cargo.toml** Declares percent decoding as a direct desktop dependency for safe embed-ID canonicalization. **desktop/src-tauri/Cargo.lock** Records the direct dependency in the desktop package lock entry. </details> ## Reproduction Steps 1. On the base branch, paste a YouTube URL whose Open Graph metadata falls beyond the first 256 KiB of the raw watch-page response and observe that no preview is produced. 2. Run this branch and paste a YouTube watch, mobile, music, `youtu.be`, Shorts, live, or embed URL into the composer. 3. Confirm the preview resolves with the video's title, creator, and sanitized thumbnail without downloading the full watch-page HTML. 4. Try an embed URL with a percent-encoded ID, such as `https://www.youtube.com/embed/%64Qw4w9WgXcQ`, and confirm it resolves to the same video. 5. Try a YouTube lookalike domain or an embed ID containing encoded separators and confirm it is not routed through the provider path. --------- Signed-off-by: Taylor Ho <taylorkmho@gmail.com>
This commit is contained in:
1
desktop/src-tauri/Cargo.lock
generated
1
desktop/src-tauri/Cargo.lock
generated
@@ -1108,6 +1108,7 @@ dependencies = [
|
||||
"objc2-foundation",
|
||||
"objc2-user-notifications",
|
||||
"opus",
|
||||
"percent-encoding",
|
||||
"plist",
|
||||
"png 0.18.1",
|
||||
"portable-pty",
|
||||
|
||||
@@ -98,6 +98,7 @@ nostr = { version = "0.44", features = ["nip44", "nip49"] }
|
||||
# transitive dependency; pinned here for direct use).
|
||||
getrandom = "0.2"
|
||||
zeroize = "1"
|
||||
percent-encoding = "2"
|
||||
reqwest = { version = "0.13", features = ["json", "query", "stream", "blocking"] }
|
||||
rustls = { version = "0.23", default-features = false, features = ["aws_lc_rs", "std"] }
|
||||
url = "2"
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
use std::{io::Cursor, net::IpAddr, time::Duration};
|
||||
|
||||
use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
|
||||
use image::ImageDecoder;
|
||||
|
||||
use futures_util::StreamExt;
|
||||
use image::ImageDecoder;
|
||||
use reqwest::{
|
||||
header::{ACCEPT, CONTENT_TYPE, LOCATION, USER_AGENT},
|
||||
redirect::Policy,
|
||||
@@ -13,6 +12,8 @@ use url::Url;
|
||||
|
||||
#[path = "link_preview_rate_limit.rs"]
|
||||
mod rate_limit;
|
||||
#[path = "link_preview_youtube.rs"]
|
||||
mod youtube;
|
||||
|
||||
use rate_limit::{image_host_cooldown_remaining, retry_after_duration, set_image_host_cooldown};
|
||||
|
||||
@@ -67,6 +68,10 @@ async fn fetch_link_preview_metadata_inner(
|
||||
let mut url = Url::parse(href.trim()).map_err(|error| format!("invalid URL: {error}"))?;
|
||||
validate_public_https_url(&url).await?;
|
||||
|
||||
if youtube::is_video_url(&url) {
|
||||
return youtube::fetch_oembed_metadata(&url).await;
|
||||
}
|
||||
|
||||
for redirect_count in 0..=MAX_REDIRECTS {
|
||||
let response = send_pinned_request(&url, "text/html,application/xhtml+xml;q=0.9").await?;
|
||||
|
||||
|
||||
361
desktop/src-tauri/src/commands/link_preview_youtube.rs
Normal file
361
desktop/src-tauri/src/commands/link_preview_youtube.rs
Normal file
@@ -0,0 +1,361 @@
|
||||
use percent_encoding::percent_decode_str;
|
||||
use reqwest::header::CONTENT_TYPE;
|
||||
use serde::Deserialize;
|
||||
use url::Url;
|
||||
|
||||
use super::{
|
||||
apply_image_result, fetch_sanitized_image, normalize_metadata_description,
|
||||
normalize_metadata_text, read_limited_bytes, send_pinned_request, ImageFetchError,
|
||||
LinkPreviewImageFetchState, LinkPreviewMetadata, PREVIEW_FETCH_TIMEOUT,
|
||||
};
|
||||
|
||||
const MAX_OEMBED_FETCH_BYTES: usize = 64 * 1024;
|
||||
const OEMBED_ENDPOINT: &str = "https://www.youtube.com/oembed";
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct OEmbedResponse {
|
||||
title: String,
|
||||
author_name: Option<String>,
|
||||
provider_name: Option<String>,
|
||||
thumbnail_url: Option<String>,
|
||||
}
|
||||
|
||||
pub(super) fn is_video_url(url: &Url) -> bool {
|
||||
let Some(host) = url.host_str().map(|host| host.to_ascii_lowercase()) else {
|
||||
return false;
|
||||
};
|
||||
match host.as_str() {
|
||||
"youtu.be" | "www.youtu.be" => url
|
||||
.path_segments()
|
||||
.and_then(|mut segments| segments.next())
|
||||
.is_some_and(|segment| !segment.is_empty()),
|
||||
"youtube.com" | "www.youtube.com" | "m.youtube.com" | "music.youtube.com" => {
|
||||
(url.path() == "/watch"
|
||||
&& url
|
||||
.query_pairs()
|
||||
.any(|(key, value)| key == "v" && !value.is_empty()))
|
||||
|| ["shorts", "live", "embed"].iter().any(|prefix| {
|
||||
url.path_segments().is_some_and(|mut segments| {
|
||||
segments.next() == Some(prefix)
|
||||
&& segments.next().is_some_and(|segment| !segment.is_empty())
|
||||
})
|
||||
})
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn fetch_oembed_metadata(
|
||||
video_url: &Url,
|
||||
) -> Result<Option<LinkPreviewMetadata>, String> {
|
||||
let oembed_url = oembed_url(video_url)?;
|
||||
let response = send_pinned_request(&oembed_url, "application/json").await?;
|
||||
let Some((mut metadata, thumbnail_url)) = parse_oembed_response(response).await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let image_result = match thumbnail_url {
|
||||
Some(thumbnail_url) => Some(
|
||||
tokio::time::timeout(
|
||||
PREVIEW_FETCH_TIMEOUT,
|
||||
fetch_sanitized_image(thumbnail_url, false),
|
||||
)
|
||||
.await
|
||||
.unwrap_or(Err(ImageFetchError::Transient { retry_after: None })),
|
||||
),
|
||||
None => None,
|
||||
};
|
||||
apply_image_result(&mut metadata, image_result);
|
||||
Ok(Some(metadata))
|
||||
}
|
||||
|
||||
fn oembed_url(video_url: &Url) -> Result<Url, String> {
|
||||
let mut canonical_video_url = video_url.clone();
|
||||
if matches!(
|
||||
video_url.host_str(),
|
||||
Some("youtube.com" | "www.youtube.com" | "m.youtube.com" | "music.youtube.com")
|
||||
) {
|
||||
let mut segments = video_url.path_segments();
|
||||
if segments.as_mut().and_then(|segments| segments.next()) == Some("embed") {
|
||||
let encoded_video_id = segments
|
||||
.and_then(|mut segments| segments.next())
|
||||
.ok_or_else(|| "YouTube embed URL has no video ID".to_string())?;
|
||||
let video_id = percent_decode_str(encoded_video_id)
|
||||
.decode_utf8()
|
||||
.map_err(|_| "YouTube embed URL has an invalid video ID".to_string())?;
|
||||
if !video_id.chars().all(|character| {
|
||||
character.is_ascii_alphanumeric() || matches!(character, '-' | '_')
|
||||
}) {
|
||||
return Err("YouTube embed URL has an invalid video ID".to_string());
|
||||
}
|
||||
canonical_video_url.set_path("/watch");
|
||||
canonical_video_url.set_query(None);
|
||||
canonical_video_url
|
||||
.query_pairs_mut()
|
||||
.append_pair("v", &video_id);
|
||||
canonical_video_url.set_fragment(None);
|
||||
}
|
||||
}
|
||||
|
||||
let mut oembed_url = Url::parse(OEMBED_ENDPOINT)
|
||||
.map_err(|error| format!("invalid YouTube oEmbed endpoint: {error}"))?;
|
||||
oembed_url
|
||||
.query_pairs_mut()
|
||||
.append_pair("format", "json")
|
||||
.append_pair("url", canonical_video_url.as_str());
|
||||
Ok(oembed_url)
|
||||
}
|
||||
|
||||
async fn parse_oembed_response(
|
||||
response: reqwest::Response,
|
||||
) -> Result<Option<(LinkPreviewMetadata, Option<Url>)>, String> {
|
||||
if !response.status().is_success() || !is_json_response(&response) {
|
||||
return Ok(None);
|
||||
}
|
||||
let body = read_limited_bytes(response, MAX_OEMBED_FETCH_BYTES).await?;
|
||||
let response: OEmbedResponse = match serde_json::from_slice(&body) {
|
||||
Ok(response) => response,
|
||||
Err(_) => return Ok(None),
|
||||
};
|
||||
Ok(oembed_metadata(response))
|
||||
}
|
||||
|
||||
fn oembed_metadata(response: OEmbedResponse) -> Option<(LinkPreviewMetadata, Option<Url>)> {
|
||||
let title = normalize_metadata_text(&response.title)?;
|
||||
let thumbnail_url = response
|
||||
.thumbnail_url
|
||||
.as_deref()
|
||||
.and_then(|thumbnail| Url::parse(thumbnail).ok());
|
||||
let metadata = LinkPreviewMetadata {
|
||||
title,
|
||||
site_name: response
|
||||
.provider_name
|
||||
.as_deref()
|
||||
.and_then(normalize_metadata_text)
|
||||
.or_else(|| Some("YouTube".to_string())),
|
||||
description: response
|
||||
.author_name
|
||||
.as_deref()
|
||||
.and_then(normalize_metadata_description),
|
||||
image_data_url: None,
|
||||
image_domain: None,
|
||||
image_fetch_state: LinkPreviewImageFetchState::None,
|
||||
image_retry_after_ms: None,
|
||||
favicon_data_url: None,
|
||||
};
|
||||
Some((metadata, thumbnail_url))
|
||||
}
|
||||
|
||||
fn is_json_response(response: &reqwest::Response) -> bool {
|
||||
response
|
||||
.headers()
|
||||
.get(CONTENT_TYPE)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(|value| {
|
||||
value
|
||||
.split(';')
|
||||
.next()
|
||||
.unwrap_or_default()
|
||||
.trim()
|
||||
.eq_ignore_ascii_case("application/json")
|
||||
})
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use axum::{body::Body, http::Response, routing::get, Router};
|
||||
|
||||
async fn test_response(router: Router, path: &str) -> reqwest::Response {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, router).await.unwrap();
|
||||
});
|
||||
reqwest::get(format!("http://{address}{path}"))
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recognizes_supported_video_urls_only() {
|
||||
for href in [
|
||||
"https://www.youtube.com/watch?v=hLFs9JtMaRg",
|
||||
"https://m.youtube.com/watch?v=hLFs9JtMaRg&feature=share",
|
||||
"https://music.youtube.com/watch?v=hLFs9JtMaRg",
|
||||
"https://youtu.be/hLFs9JtMaRg?t=10",
|
||||
"https://www.youtube.com/shorts/hLFs9JtMaRg",
|
||||
"https://www.youtube.com/live/hLFs9JtMaRg",
|
||||
"https://www.youtube.com/embed/hLFs9JtMaRg",
|
||||
] {
|
||||
assert!(is_video_url(&Url::parse(href).unwrap()), "{href}");
|
||||
}
|
||||
for href in [
|
||||
"https://www.youtube.com/",
|
||||
"https://www.youtube.com/watch",
|
||||
"https://www.youtube.com/watch?v=",
|
||||
"https://www.youtube.com/@buzz",
|
||||
"https://youtube.com.evil.example/watch?v=hLFs9JtMaRg",
|
||||
"https://notyoutube.com/watch?v=hLFs9JtMaRg",
|
||||
] {
|
||||
assert!(!is_video_url(&Url::parse(href).unwrap()), "{href}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonicalizes_embed_url_for_oembed() {
|
||||
for (video_url, expected_video_id) in [
|
||||
(
|
||||
"https://www.youtube.com/embed/dQw4w9WgXcQ?start=10#player",
|
||||
"dQw4w9WgXcQ",
|
||||
),
|
||||
("https://www.youtube.com/embed/%64Qw4w9WgXcQ", "dQw4w9WgXcQ"),
|
||||
("https://www.youtube.com/embed/dQw4w9WgX%63Q", "dQw4w9WgXcQ"),
|
||||
] {
|
||||
let oembed_url = oembed_url(&Url::parse(video_url).unwrap()).unwrap();
|
||||
let params = oembed_url
|
||||
.query_pairs()
|
||||
.collect::<std::collections::HashMap<_, _>>();
|
||||
assert_eq!(
|
||||
params.get("format").map(|value| value.as_ref()),
|
||||
Some("json")
|
||||
);
|
||||
let provider_video_url =
|
||||
Url::parse(params.get("url").expect("oEmbed URL parameter")).unwrap();
|
||||
assert_eq!(provider_video_url.path(), "/watch");
|
||||
assert_eq!(
|
||||
provider_video_url
|
||||
.query_pairs()
|
||||
.find(|(key, _)| key == "v")
|
||||
.map(|(_, value)| value.into_owned()),
|
||||
Some(expected_video_id.to_string())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_invalid_encoded_embed_video_ids() {
|
||||
for href in [
|
||||
"https://www.youtube.com/embed/video%2Fid",
|
||||
"https://www.youtube.com/embed/video%5Cid",
|
||||
"https://www.youtube.com/embed/video%00id",
|
||||
"https://www.youtube.com/embed/video%25id",
|
||||
"https://www.youtube.com/embed/video%252Fid",
|
||||
"https://www.youtube.com/embed/video%FFid",
|
||||
] {
|
||||
assert!(oembed_url(&Url::parse(href).unwrap()).is_err(), "{href}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn response_requires_successful_bounded_json() {
|
||||
let valid_json =
|
||||
r#"{"title":"Video title","author_name":"Creator","provider_name":"YouTube"}"#;
|
||||
let response = test_response(
|
||||
Router::new().route(
|
||||
"/valid",
|
||||
get(move || async move {
|
||||
Response::builder()
|
||||
.header("content-type", "application/json; charset=UTF-8")
|
||||
.body(Body::from(valid_json))
|
||||
.unwrap()
|
||||
}),
|
||||
),
|
||||
"/valid",
|
||||
)
|
||||
.await;
|
||||
let (metadata, _) = parse_oembed_response(response).await.unwrap().unwrap();
|
||||
assert_eq!(metadata.title, "Video title");
|
||||
|
||||
for response in [
|
||||
test_response(
|
||||
Router::new().route(
|
||||
"/not-found",
|
||||
get(|| async {
|
||||
Response::builder()
|
||||
.status(404)
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from("{}"))
|
||||
.unwrap()
|
||||
}),
|
||||
),
|
||||
"/not-found",
|
||||
)
|
||||
.await,
|
||||
test_response(
|
||||
Router::new().route(
|
||||
"/html",
|
||||
get(|| async {
|
||||
Response::builder()
|
||||
.header("content-type", "text/html")
|
||||
.body(Body::from("<title>Not JSON</title>"))
|
||||
.unwrap()
|
||||
}),
|
||||
),
|
||||
"/html",
|
||||
)
|
||||
.await,
|
||||
] {
|
||||
assert_eq!(parse_oembed_response(response).await.unwrap(), None);
|
||||
}
|
||||
|
||||
let oversized = vec![b' '; MAX_OEMBED_FETCH_BYTES + 1];
|
||||
let response = test_response(
|
||||
Router::new().route(
|
||||
"/oversized",
|
||||
get(move || {
|
||||
let oversized = oversized.clone();
|
||||
async move {
|
||||
Response::builder()
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(oversized))
|
||||
.unwrap()
|
||||
}
|
||||
}),
|
||||
),
|
||||
"/oversized",
|
||||
)
|
||||
.await;
|
||||
assert!(parse_oembed_response(response).await.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn converts_response_to_bounded_preview_metadata() {
|
||||
let (metadata, thumbnail_url) = oembed_metadata(OEmbedResponse {
|
||||
title: " Video title ".to_string(),
|
||||
author_name: Some("Buzz Creator".to_string()),
|
||||
provider_name: Some("YouTube".to_string()),
|
||||
thumbnail_url: Some("https://i.ytimg.com/vi/example/hqdefault.jpg".to_string()),
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(metadata.title, "Video title");
|
||||
assert_eq!(metadata.site_name.as_deref(), Some("YouTube"));
|
||||
assert_eq!(metadata.description.as_deref(), Some("Buzz Creator"));
|
||||
assert_eq!(metadata.image_fetch_state, LinkPreviewImageFetchState::None);
|
||||
assert_eq!(
|
||||
thumbnail_url.unwrap().as_str(),
|
||||
"https://i.ytimg.com/vi/example/hqdefault.jpg"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_titleless_response_and_ignores_invalid_thumbnail() {
|
||||
assert!(oembed_metadata(OEmbedResponse {
|
||||
title: " ".to_string(),
|
||||
author_name: None,
|
||||
provider_name: None,
|
||||
thumbnail_url: None,
|
||||
})
|
||||
.is_none());
|
||||
|
||||
let (metadata, thumbnail_url) = oembed_metadata(OEmbedResponse {
|
||||
title: "Video title".to_string(),
|
||||
author_name: None,
|
||||
provider_name: None,
|
||||
thumbnail_url: Some("not a URL".to_string()),
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(metadata.site_name.as_deref(), Some("YouTube"));
|
||||
assert_eq!(thumbnail_url, None);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user