From 31c2735c5cd663e301387dfb139905a3a52beddd Mon Sep 17 00:00:00 2001 From: Matt Toohey Date: Wed, 2 Sep 2026 12:21:15 +1000 Subject: [PATCH] fix(desktop): downscale Project Canvas avatars so real ones reach widgets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Widgets rendered initials for every person who actually had an avatar. Canvas frames run with `connect-src 'none'` and an `img-src` that excludes remote origins, so the host inlines avatars as `data:` URLs in the RPC payload — but it forwarded them at full resolution and then rejected anything over 48 KiB. A 512px gravatar, the size the app itself requests, is ~230 KB and ~307,000 characters once base64-encoded: 6.3x over the cap. `inlineAvatarDataUrl` fetched it, encoded it, discarded it, and cached the miss for the rest of the session, so `people.lookup` always answered `avatarDataUrl: null`. Re-encode instead of rejecting. `avatarDownscale` center-crops the decoded image to a square, redraws it at no more than 96px (canvas avatars render at 20-42px, so this stays crisp at 2x), and encodes WebP with a JPEG fallback for webviews that silently return a PNG from `toDataURL("image/webp")`. The same gravatar lands at ~4 KB. Decode and encode sit behind an injectable seam because neither exists under Node. Also bound the combined size, a second and independent failure: avatars were capped individually at 48 KiB, but a whole response shares the 64 KiB port-message ceiling, so two large avatars in one `people.lookup` trip `postSized` and the frame receives a `too-large` error instead of the result — losing every display name too, not just the pictures. `selectAvatarsWithinBudget` keeps avatars in order until 40 KiB is spent and degrades the rest to initials; the lookup path and the channel-snapshot path in `ProjectChannelHome` both go through it. `fetchAvatarBlob` splits out of `fetchAvatarDataUrl` so the canvas path can re-encode raw bytes while self-profile storage keeps caching the verbatim data URL it puts in localStorage. Both still share one fetch policy and the same 256 KB ceiling. Tests pin the downscaler on target size (drawing at full size fails it), center-crop geometry, the no-upscale rule, the WebP-to-JPEG fallback, degradation to null rather than a broken image, and bitmap release on both the success and failure paths; the budget is pinned on ordering, index alignment, and staying under PROJECT_CANVAS_MAX_PORT_MESSAGE_BYTES for a full 32-person lookup. Verified: desktop unit suite passes 5914, and biome, tsc, and the pubkey-truncation guard are clean. Co-Authored-By: Claude Opus 5 Signed-off-by: Matt Toohey --- .../profile/lib/selfProfileStorage.ts | 56 +++-- .../projects/ui/ProjectChannelHome.tsx | 26 ++- .../ui/project-canvas/canvasAvatars.test.mjs | 74 +++++++ .../ui/project-canvas/canvasAvatars.ts | 78 +++++++ .../project-canvas/useProjectCanvasBroker.ts | 33 +-- .../src/shared/lib/avatarDownscale.test.mjs | 199 ++++++++++++++++++ desktop/src/shared/lib/avatarDownscale.ts | 154 ++++++++++++++ 7 files changed, 581 insertions(+), 39 deletions(-) create mode 100644 desktop/src/features/projects/ui/project-canvas/canvasAvatars.test.mjs create mode 100644 desktop/src/features/projects/ui/project-canvas/canvasAvatars.ts create mode 100644 desktop/src/shared/lib/avatarDownscale.test.mjs create mode 100644 desktop/src/shared/lib/avatarDownscale.ts diff --git a/desktop/src/features/profile/lib/selfProfileStorage.ts b/desktop/src/features/profile/lib/selfProfileStorage.ts index eaff2546c2..cb92d6cb3a 100644 --- a/desktop/src/features/profile/lib/selfProfileStorage.ts +++ b/desktop/src/features/profile/lib/selfProfileStorage.ts @@ -365,19 +365,27 @@ export function resolveAvatarDataUrl( } /** - * Fetches an avatar from `avatarProxyUrl` and converts it to a base64 data URL. + * Upper bound on a fetched avatar's encoded bytes. Keeps localStorage usage + * bounded across communities and accounts, and bounds the work any single + * decode can be asked to do. + */ +const MAX_AVATAR_BLOB_BYTES = 256 * 1024; + +/** + * Fetches an avatar from `avatarProxyUrl` as an image blob. * * The caller should pass the `rewriteRelayUrl()`-proxied URL and invoke this * only immediately after a successful profile fetch — at that moment the relay * is known to be reachable, so the fetch has the best chance of succeeding. * - * The data URL is capped at 256 KB to keep localStorage usage bounded across - * communities and accounts. Returns null on ANY failure: network error, non-OK - * response, wrong content-type, blob too large, or FileReader error. + * Returns null on ANY failure: network error, non-OK response, wrong + * content-type, or blob too large. Callers that need the raw bytes (to + * re-encode at a smaller size, say) use this; callers that want the image + * verbatim use {@link fetchAvatarDataUrl}. */ -export async function fetchAvatarDataUrl( +export async function fetchAvatarBlob( avatarProxyUrl: string, -): Promise { +): Promise { try { const response = await fetch(avatarProxyUrl); if (!response.ok) return null; @@ -386,18 +394,34 @@ export async function fetchAvatarDataUrl( if (!contentType.startsWith("image/")) return null; const blob = await response.blob(); - if (blob.size > 256 * 1024) return null; + if (blob.size > MAX_AVATAR_BLOB_BYTES) return null; - return await new Promise((resolve) => { - const reader = new FileReader(); - reader.onload = () => { - const result = reader.result; - resolve(typeof result === "string" ? result : null); - }; - reader.onerror = () => resolve(null); - reader.readAsDataURL(blob); - }); + return blob; } catch { return null; } } + +/** + * Fetches an avatar from `avatarProxyUrl` and converts it to a base64 data URL. + * + * Shares {@link fetchAvatarBlob}'s fetch policy, so the same 256 KB ceiling and + * failure modes apply. Returns null on any failure, including a FileReader + * error. + */ +export async function fetchAvatarDataUrl( + avatarProxyUrl: string, +): Promise { + const blob = await fetchAvatarBlob(avatarProxyUrl); + if (!blob) return null; + + return await new Promise((resolve) => { + const reader = new FileReader(); + reader.onload = () => { + const result = reader.result; + resolve(typeof result === "string" ? result : null); + }; + reader.onerror = () => resolve(null); + reader.readAsDataURL(blob); + }); +} diff --git a/desktop/src/features/projects/ui/ProjectChannelHome.tsx b/desktop/src/features/projects/ui/ProjectChannelHome.tsx index cc3bb433b3..52863ce818 100644 --- a/desktop/src/features/projects/ui/ProjectChannelHome.tsx +++ b/desktop/src/features/projects/ui/ProjectChannelHome.tsx @@ -9,7 +9,6 @@ import { ChannelScreenLoadingFallback } from "@/features/channels/ui/ChannelScre import { ChannelViewOverrideProvider } from "@/features/channels/ui/ChannelViewOverrideContext"; import { useCommunities } from "@/features/communities/useCommunities"; import { useProfileQuery, useUsersBatchQuery } from "@/features/profile/hooks"; -import { fetchAvatarDataUrl } from "@/features/profile/lib/selfProfileStorage"; import { type Project, useProjectPullRequestsQuery, @@ -25,11 +24,14 @@ import { useLiveProjectWorkItems } from "@/features/projects/useLiveProjectWorkI import { useIdentityQuery } from "@/shared/api/hooks"; import type { Channel, RelayEvent } from "@/shared/api/types"; import { getAvatarSnapshotUrl } from "@/shared/lib/animatedAvatar"; -import { rewriteRelayUrl } from "@/shared/lib/mediaUrl"; import { Button } from "@/shared/ui/button"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/shared/ui/tooltip"; import { ViewLoadingFallback } from "@/shared/ui/ViewLoadingFallback"; import { ProjectChannelResourcesView } from "./ProjectChannelResourcesView"; +import { + fetchCanvasAvatarDataUrl, + selectAvatarsWithinBudget, +} from "./project-canvas/canvasAvatars"; import { ProjectCanvasSurface } from "./project-canvas/ProjectCanvasSurface"; import type { ProjectCanvasOpenTarget } from "./project-canvas/projectCanvasBroker"; import type { ProjectCanvasSnapshots } from "./project-canvas/projectCanvasProtocol"; @@ -54,7 +56,6 @@ const MAX_CANVAS_PEOPLE_PER_CHANNEL = 5; const MAX_CANVAS_REPOSITORIES = 64; const MAX_CANVAS_REVIEWS = 32; const MAX_CANVAS_AVATARS = 8; -const MAX_CANVAS_AVATAR_DATA_URL_LENGTH = 48 * 1_024; function boundedCanvasText(value: string, maxLength: number): string { return value.slice(0, maxLength); @@ -230,21 +231,24 @@ export function ProjectChannelHome({ const canvasAvatarQueries = useQueries({ queries: canvasAvatarCandidates.map(({ pubkey, snapshotUrl }) => ({ gcTime: 10 * 60_000, - queryFn: () => fetchAvatarDataUrl(rewriteRelayUrl(snapshotUrl)), + queryFn: () => fetchCanvasAvatarDataUrl(snapshotUrl), queryKey: ["project-canvas-avatar", pubkey, snapshotUrl], staleTime: 10 * 60_000, })), }); const canvasAvatarDataByPubkey = React.useMemo(() => { + // The channel snapshot ships these to the frame in one RPC message, so the + // combined ceiling applies here exactly as it does to a people lookup. + const budgeted = selectAvatarsWithinBudget( + canvasAvatarCandidates.map((_candidate, index) => { + const dataUrl = canvasAvatarQueries[index]?.data; + return dataUrl?.startsWith("data:image/") ? dataUrl : null; + }), + ); const avatars = new Map(); canvasAvatarCandidates.forEach((candidate, index) => { - const dataUrl = canvasAvatarQueries[index]?.data; - if ( - dataUrl?.startsWith("data:image/") && - dataUrl.length <= MAX_CANVAS_AVATAR_DATA_URL_LENGTH - ) { - avatars.set(candidate.pubkey, dataUrl); - } + const dataUrl = budgeted[index]; + if (dataUrl) avatars.set(candidate.pubkey, dataUrl); }); return avatars; }, [canvasAvatarCandidates, canvasAvatarQueries]); diff --git a/desktop/src/features/projects/ui/project-canvas/canvasAvatars.test.mjs b/desktop/src/features/projects/ui/project-canvas/canvasAvatars.test.mjs new file mode 100644 index 0000000000..e9144809a2 --- /dev/null +++ b/desktop/src/features/projects/ui/project-canvas/canvasAvatars.test.mjs @@ -0,0 +1,74 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + CANVAS_AVATAR_TOTAL_DATA_URL_BUDGET, + selectAvatarsWithinBudget, +} from "./canvasAvatars.ts"; +import { PROJECT_CANVAS_MAX_PORT_MESSAGE_BYTES } from "./projectCanvasProtocol.ts"; + +/** A data URL of exactly `length` characters. */ +function avatar(length) { + const prefix = "data:image/webp;base64,"; + return prefix + "A".repeat(Math.max(0, length - prefix.length)); +} + +function totalLength(dataUrls) { + return dataUrls.reduce((sum, value) => sum + (value?.length ?? 0), 0); +} + +test("the combined avatar budget leaves room inside one rpc message", () => { + // The whole response — avatars, names, pubkeys, envelope — must fit the port + // ceiling, so the avatar share has to stay strictly under it. + assert.ok( + CANVAS_AVATAR_TOTAL_DATA_URL_BUDGET < PROJECT_CANVAS_MAX_PORT_MESSAGE_BYTES, + ); +}); + +test("avatars are kept in order until the budget is spent", () => { + const rows = [avatar(10_000), avatar(10_000), avatar(10_000)]; + + const selected = selectAvatarsWithinBudget(rows, 25_000); + + assert.deepEqual( + selected.map((value) => value !== null), + [true, true, false], + ); + assert.ok(totalLength(selected) <= 25_000); +}); + +test("a lookup that would overrun the port ceiling is trimmed under it", () => { + // 32 people is the lookup maximum; at the per-avatar cap they would total + // 512 KiB and the frame would get `too-large` instead of the result. + const rows = Array.from({ length: 32 }, () => avatar(16 * 1_024)); + + const selected = selectAvatarsWithinBudget(rows); + + assert.ok(totalLength(selected) <= CANVAS_AVATAR_TOTAL_DATA_URL_BUDGET); + assert.ok(totalLength(selected) < PROJECT_CANVAS_MAX_PORT_MESSAGE_BYTES); + assert.ok(selected.some((value) => value !== null)); +}); + +test("a single avatar larger than the whole budget is dropped", () => { + const selected = selectAvatarsWithinBudget([avatar(50_000)], 40_000); + + assert.deepEqual(selected, [null]); +}); + +test("missing avatars pass through without consuming budget", () => { + const selected = selectAvatarsWithinBudget( + [null, avatar(30_000), null, avatar(9_000)], + 40_000, + ); + + assert.deepEqual( + selected.map((value) => value !== null), + [false, true, false, true], + ); +}); + +test("the result is index-aligned with its input so rows can be zipped back", () => { + const rows = [avatar(100), null, avatar(100)]; + + assert.equal(selectAvatarsWithinBudget(rows).length, rows.length); +}); diff --git a/desktop/src/features/projects/ui/project-canvas/canvasAvatars.ts b/desktop/src/features/projects/ui/project-canvas/canvasAvatars.ts new file mode 100644 index 0000000000..1d846f6959 --- /dev/null +++ b/desktop/src/features/projects/ui/project-canvas/canvasAvatars.ts @@ -0,0 +1,78 @@ +import { fetchAvatarBlob } from "@/features/profile/lib/selfProfileStorage"; +import { getAvatarSnapshotUrl } from "@/shared/lib/animatedAvatar"; +import { + DEFAULT_AVATAR_PIXEL_SIZE, + downscaleAvatarDataUrl, +} from "@/shared/lib/avatarDownscale"; +import { rewriteRelayUrl } from "@/shared/lib/mediaUrl"; + +/** + * Avatar delivery policy for sandboxed Project Canvas frames. + * + * A canvas frame cannot reach the network, so the host fetches avatars itself + * and hands them to widgets as `data:` URLs inside the RPC payload. Two + * ceilings apply and they are not independent: each image is re-encoded small + * enough to be worth sending, and the *sum* across a response has to stay + * inside `PROJECT_CANVAS_MAX_PORT_MESSAGE_BYTES` (64 KiB), or the host replies + * with a `too-large` error and the widget loses the whole result — display + * names included, not just the pictures. + */ + +/** + * Square size, in pixels, avatars are re-encoded to. Canvas avatars render + * between 20px and 42px, so this stays crisp on a 2x display. + */ +export const CANVAS_AVATAR_PIXEL_SIZE = DEFAULT_AVATAR_PIXEL_SIZE; + +/** Per-avatar ceiling. A 96px WebP/JPEG avatar lands far under this. */ +export const CANVAS_AVATAR_MAX_DATA_URL_LENGTH = 16 * 1_024; + +/** + * Combined ceiling across one response. Leaves ~24 KiB of the 64 KiB message + * budget for names, pubkeys, and the envelope — ample for the 32-person + * maximum a lookup can return. + */ +export const CANVAS_AVATAR_TOTAL_DATA_URL_BUDGET = 40 * 1_024; + +/** + * Fetches `avatarUrl` and re-encodes it small enough to embed in a canvas RPC + * payload. Returns null when there is no avatar, the fetch fails, or the image + * cannot be brought under {@link CANVAS_AVATAR_MAX_DATA_URL_LENGTH} — all of + * which mean "render initials". + * + * Animated avatars collapse to their poster frame, and relay-hosted URLs go + * through the media proxy, matching how the rest of the app resolves an avatar. + */ +export async function fetchCanvasAvatarDataUrl( + avatarUrl: string | null, +): Promise { + const snapshotUrl = getAvatarSnapshotUrl(avatarUrl); + if (!snapshotUrl) return null; + const blob = await fetchAvatarBlob(rewriteRelayUrl(snapshotUrl)); + if (!blob) return null; + return await downscaleAvatarDataUrl(blob, { + maxDataUrlLength: CANVAS_AVATAR_MAX_DATA_URL_LENGTH, + pixelSize: CANVAS_AVATAR_PIXEL_SIZE, + }); +} + +/** + * Drops avatars that would push a response past `totalBudget`, preserving + * order: earlier entries keep their image and later ones degrade to initials. + * + * Returns a same-length array so callers can zip it back onto their rows. A + * single avatar larger than the whole budget is dropped rather than allowed + * through, so the ceiling always holds. + */ +export function selectAvatarsWithinBudget( + dataUrls: ReadonlyArray, + totalBudget: number = CANVAS_AVATAR_TOTAL_DATA_URL_BUDGET, +): Array { + let spent = 0; + return dataUrls.map((dataUrl) => { + if (!dataUrl) return null; + if (spent + dataUrl.length > totalBudget) return null; + spent += dataUrl.length; + return dataUrl; + }); +} diff --git a/desktop/src/features/projects/ui/project-canvas/useProjectCanvasBroker.ts b/desktop/src/features/projects/ui/project-canvas/useProjectCanvasBroker.ts index 6d2c60fc84..e3e7c131fd 100644 --- a/desktop/src/features/projects/ui/project-canvas/useProjectCanvasBroker.ts +++ b/desktop/src/features/projects/ui/project-canvas/useProjectCanvasBroker.ts @@ -7,12 +7,14 @@ import { useUnassignProjectIssueMutation, } from "@/features/projects/issueAssignments"; import { useUpdateProjectIssueStatusMutation } from "@/features/projects/issueMutations"; -import { fetchAvatarDataUrl } from "@/features/profile/lib/selfProfileStorage"; import { sendChannelMessage } from "@/shared/api/tauriMessages"; import { getUsersBatch, searchUsers } from "@/shared/api/tauriProfiles"; import { getAvatarSnapshotUrl } from "@/shared/lib/animatedAvatar"; -import { rewriteRelayUrl } from "@/shared/lib/mediaUrl"; import { normalizePubkey } from "@/shared/lib/pubkey"; +import { + fetchCanvasAvatarDataUrl, + selectAvatarsWithinBudget, +} from "./canvasAvatars"; import type { Repository } from "@/features/projects/hooks"; import type { ProjectIssue } from "@/features/projects/projectIssues.mjs"; import type { ProjectPullRequest } from "@/features/projects/projectPullRequests.mjs"; @@ -28,7 +30,6 @@ import type { ProjectCanvasSnapshots } from "./projectCanvasProtocol"; const MAX_BROKER_ROWS = 50; const MAX_BROKER_TEXT = 256; const MAX_AVATAR_CACHE_ENTRIES = 32; -const MAX_AVATAR_DATA_URL_LENGTH = 48 * 1_024; function boundedText(value: string, maxLength = MAX_BROKER_TEXT): string { return value.slice(0, maxLength); @@ -36,8 +37,10 @@ function boundedText(value: string, maxLength = MAX_BROKER_TEXT): string { /** * Bounded avatar inliner: canvas frames run with `connect-src 'none'`, so - * people rows carry small data-URL avatars instead of relay URLs. The cache - * holds at most 32 entries of at most 48 KiB each; failures are cached as + * people rows carry re-encoded data-URL avatars instead of relay URLs. Each + * image is downscaled to a canvas-sized square before it is cached, so the + * cache holds at most 32 entries of at most + * {@link CANVAS_AVATAR_MAX_DATA_URL_LENGTH} each; failures are cached as * misses so a broken avatar cannot retrigger fetch loops. */ async function inlineAvatarDataUrl( @@ -54,15 +57,11 @@ async function inlineAvatarDataUrl( } let dataUrl: string | null = null; try { - dataUrl = await fetchAvatarDataUrl(rewriteRelayUrl(snapshotUrl)); + dataUrl = await fetchCanvasAvatarDataUrl(snapshotUrl); } catch { dataUrl = null; } - const accepted = - dataUrl?.startsWith("data:image/") && - dataUrl.length <= MAX_AVATAR_DATA_URL_LENGTH - ? dataUrl - : ""; + const accepted = dataUrl?.startsWith("data:image/") ? dataUrl : ""; cache.set(snapshotUrl, accepted); while (cache.size > MAX_AVATAR_CACHE_ENTRIES) { const oldest = cache.keys().next().value; @@ -152,7 +151,7 @@ export function useProjectCanvasBroker({ pubkeys: string[], ): Promise => { const batch = await getUsersBatch(pubkeys); - return Promise.all( + const rows = await Promise.all( pubkeys.map(async (pubkey) => { const profile = batch.profiles[pubkey]; return { @@ -169,6 +168,16 @@ export function useProjectCanvasBroker({ }; }), ); + // Every row travels in one RPC message. Without a combined ceiling a + // handful of avatars overruns it and the frame gets a `too-large` error + // instead of the lookup — losing the names too, not just the pictures. + const budgeted = selectAvatarsWithinBudget( + rows.map((row) => row.avatarDataUrl), + ); + return rows.map((row, index) => ({ + ...row, + avatarDataUrl: budgeted[index] ?? null, + })); }; brokerRef.current = new ProjectCanvasBroker({ lookupPeople: lookupProfiles, diff --git a/desktop/src/shared/lib/avatarDownscale.test.mjs b/desktop/src/shared/lib/avatarDownscale.test.mjs new file mode 100644 index 0000000000..3d77e689fa --- /dev/null +++ b/desktop/src/shared/lib/avatarDownscale.test.mjs @@ -0,0 +1,199 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + DEFAULT_AVATAR_PIXEL_SIZE, + downscaleAvatarDataUrl, +} from "./avatarDownscale.ts"; + +/** + * Builds the decode/encode seam with a recording target. `encode` maps a + * requested mime type to the data URL the surface would return, standing in for + * a webview's `toDataURL`. + */ +function fakeDeps({ source, encode, targetForSize = () => true }) { + const calls = { closed: 0, draws: [], targets: [] }; + const bitmap = { + ...source, + close: () => { + calls.closed += 1; + }, + }; + return { + calls, + deps: { + createTarget: (size) => { + calls.targets.push(size); + if (!targetForSize(size)) return null; + return { + drawImage: (...args) => { + calls.draws.push(args); + }, + toDataURL: (type, quality) => encode(type, quality, size), + }; + }, + decode: async () => (source ? bitmap : null), + }, + }; +} + +/** A data URL of `length` characters carrying `type`'s prefix. */ +function dataUrlOfLength(type, length) { + const prefix = `data:${type};base64,`; + return prefix + "A".repeat(Math.max(0, length - prefix.length)); +} + +const BLOB = /** @type {Blob} */ ({}); + +test("a large avatar is redrawn at the target size before it is encoded", async () => { + const { calls, deps } = fakeDeps({ + encode: (type) => dataUrlOfLength(type, 4_000), + source: { height: 512, width: 512 }, + }); + + const result = await downscaleAvatarDataUrl(BLOB, { deps }); + + assert.ok(result.startsWith("data:image/webp;base64,")); + // Binds the fix: the 512px source must reach the encoder as a + // DEFAULT_AVATAR_PIXEL_SIZE square. Encoding at full size fails here. + assert.deepEqual(calls.targets, [DEFAULT_AVATAR_PIXEL_SIZE]); + const [, sx, sy, sw, sh, dx, dy, dw, dh] = calls.draws[0]; + assert.deepEqual( + { dh, dw, dx, dy }, + { + dh: DEFAULT_AVATAR_PIXEL_SIZE, + dw: DEFAULT_AVATAR_PIXEL_SIZE, + dx: 0, + dy: 0, + }, + ); + assert.deepEqual({ sh, sw, sx, sy }, { sh: 512, sw: 512, sx: 0, sy: 0 }); +}); + +test("a non-square avatar is center-cropped to its shorter side", async () => { + const { calls, deps } = fakeDeps({ + encode: (type) => dataUrlOfLength(type, 100), + source: { height: 100, width: 300 }, + }); + + await downscaleAvatarDataUrl(BLOB, { deps, pixelSize: 64 }); + + const [, sx, sy, sw, sh] = calls.draws[0]; + assert.deepEqual({ sh, sw, sx, sy }, { sh: 100, sw: 100, sx: 100, sy: 0 }); +}); + +test("an avatar smaller than the target is not upscaled", async () => { + const { calls, deps } = fakeDeps({ + encode: (type) => dataUrlOfLength(type, 100), + source: { height: 32, width: 32 }, + }); + + await downscaleAvatarDataUrl(BLOB, { deps, pixelSize: 96 }); + + assert.deepEqual(calls.targets, [32]); +}); + +test("an unsupported webp encoder falls through to jpeg", async () => { + // Safari returns a PNG rather than failing when it cannot encode WebP. + const { deps } = fakeDeps({ + encode: (type) => + type === "image/webp" + ? dataUrlOfLength("image/png", 500) + : dataUrlOfLength(type, 500), + source: { height: 200, width: 200 }, + }); + + const result = await downscaleAvatarDataUrl(BLOB, { deps }); + + assert.ok(result.startsWith("data:image/jpeg;base64,")); +}); + +test("an encoding over the cap is rejected and the next one is tried", async () => { + const { deps } = fakeDeps({ + encode: (type) => + dataUrlOfLength(type, type === "image/webp" ? 9_000 : 1_000), + source: { height: 200, width: 200 }, + }); + + const result = await downscaleAvatarDataUrl(BLOB, { + deps, + maxDataUrlLength: 2_000, + }); + + assert.ok(result.startsWith("data:image/jpeg;base64,")); +}); + +test("an avatar that fits no encoding degrades to null rather than a broken image", async () => { + const { deps } = fakeDeps({ + encode: (type) => dataUrlOfLength(type, 50_000), + source: { height: 200, width: 200 }, + }); + + const result = await downscaleAvatarDataUrl(BLOB, { + deps, + maxDataUrlLength: 16 * 1_024, + }); + + assert.equal(result, null); +}); + +test("a throwing encoder is skipped instead of failing the avatar", async () => { + const { deps } = fakeDeps({ + encode: (type) => { + if (type === "image/webp") throw new Error("encoder unavailable"); + return dataUrlOfLength(type, 500); + }, + source: { height: 200, width: 200 }, + }); + + assert.ok( + (await downscaleAvatarDataUrl(BLOB, { deps })).startsWith( + "data:image/jpeg;base64,", + ), + ); +}); + +test("an undecodable blob, a zero-size source, and a missing surface all yield null", async () => { + const undecodable = fakeDeps({ + encode: (type) => dataUrlOfLength(type, 100), + source: null, + }); + assert.equal( + await downscaleAvatarDataUrl(BLOB, { deps: undecodable.deps }), + null, + ); + + const empty = fakeDeps({ + encode: (type) => dataUrlOfLength(type, 100), + source: { height: 0, width: 0 }, + }); + assert.equal(await downscaleAvatarDataUrl(BLOB, { deps: empty.deps }), null); + + const noSurface = fakeDeps({ + encode: (type) => dataUrlOfLength(type, 100), + source: { height: 64, width: 64 }, + targetForSize: () => false, + }); + assert.equal( + await downscaleAvatarDataUrl(BLOB, { deps: noSurface.deps }), + null, + ); +}); + +test("the decoded bitmap is released on both the success and failure paths", async () => { + const ok = fakeDeps({ + encode: (type) => dataUrlOfLength(type, 100), + source: { height: 64, width: 64 }, + }); + await downscaleAvatarDataUrl(BLOB, { deps: ok.deps }); + assert.equal(ok.calls.closed, 1); + + const failed = fakeDeps({ + encode: () => { + throw new Error("boom"); + }, + source: { height: 64, width: 64 }, + }); + await downscaleAvatarDataUrl(BLOB, { deps: failed.deps }); + assert.equal(failed.calls.closed, 1); +}); diff --git a/desktop/src/shared/lib/avatarDownscale.ts b/desktop/src/shared/lib/avatarDownscale.ts new file mode 100644 index 0000000000..11a6388ec7 --- /dev/null +++ b/desktop/src/shared/lib/avatarDownscale.ts @@ -0,0 +1,154 @@ +/** + * Re-encodes an avatar image to a small square data URL. + * + * Sandboxed Project Canvas frames run with `connect-src 'none'` and an + * `img-src` that excludes remote origins, so an avatar can only reach a widget + * as a `data:` URL carried inside the RPC payload. That payload has a hard + * 64 KiB per-message ceiling (`PROJECT_CANVAS_MAX_PORT_MESSAGE_BYTES`) shared + * by every row in the response, and base64 inflates bytes by a third — so a + * full-resolution avatar cannot be forwarded verbatim. A 512px gravatar is + * ~230 KB on the wire and ~307,000 characters once encoded, roughly six times + * the entire message budget on its own. + * + * Decoding it to a square no larger than the display size first turns that into + * a few thousand characters, which is what makes real avatars deliverable at + * all. Canvas avatars render between 20px and 42px, so the default 96px target + * still covers the largest of them on a 2x display. + */ + +/** Minimal shape of a decoded image this module can draw. */ +export type AvatarImageSource = { + readonly height: number; + readonly width: number; + close?: () => void; +}; + +/** Minimal square drawing surface the downscaler encodes through. */ +export type AvatarRenderTarget = { + drawImage( + source: AvatarImageSource, + sx: number, + sy: number, + sw: number, + sh: number, + dx: number, + dy: number, + dw: number, + dh: number, + ): void; + toDataURL(type: string, quality: number): string; +}; + +/** + * Decode/encode seam. Defaults to `createImageBitmap` plus a ``; + * tests substitute fakes because neither exists under Node. + */ +export type AvatarDownscaleDeps = { + createTarget: (size: number) => AvatarRenderTarget | null; + decode: (blob: Blob) => Promise; +}; + +export type AvatarDownscaleOptions = { + deps?: AvatarDownscaleDeps; + maxDataUrlLength?: number; + pixelSize?: number; +}; + +export const DEFAULT_AVATAR_PIXEL_SIZE = 96; + +/** + * Encoders tried in order. WebP keeps alpha and is the smallest, but a webview + * that cannot encode it silently returns a PNG from `toDataURL`, so each + * candidate is verified against the type it asked for and JPEG backs it up. + */ +const ENCODINGS: ReadonlyArray = [ + ["image/webp", 0.82], + ["image/jpeg", 0.82], +]; + +function defaultDeps(): AvatarDownscaleDeps { + return { + createTarget: (size) => { + if (typeof document === "undefined") return null; + const canvas = document.createElement("canvas"); + canvas.width = size; + canvas.height = size; + const context = canvas.getContext("2d"); + if (!context) return null; + return { + drawImage: (source, sx, sy, sw, sh, dx, dy, dw, dh) => { + context.drawImage( + source as unknown as CanvasImageSource, + sx, + sy, + sw, + sh, + dx, + dy, + dw, + dh, + ); + }, + toDataURL: (type, quality) => canvas.toDataURL(type, quality), + }; + }, + decode: async (blob) => { + if (typeof createImageBitmap !== "function") return null; + return await createImageBitmap(blob); + }, + }; +} + +/** + * Center-crops `source` to a square and re-encodes it at no more than + * `pixelSize` pixels a side, returning the first encoding that fits + * `maxDataUrlLength`. + * + * Returns null when the image cannot be decoded, the platform offers no + * drawing surface, or every encoding is still over budget. A null result means + * "render initials", never a broken image. + * + * Never upscales: a source smaller than `pixelSize` is re-encoded at its own + * size rather than inflated to the target. + */ +export async function downscaleAvatarDataUrl( + blob: Blob, + options: AvatarDownscaleOptions = {}, +): Promise { + const pixelSize = options.pixelSize ?? DEFAULT_AVATAR_PIXEL_SIZE; + const maxDataUrlLength = options.maxDataUrlLength ?? Number.POSITIVE_INFINITY; + const deps = options.deps ?? defaultDeps(); + + let source: AvatarImageSource | null = null; + try { + source = await deps.decode(blob); + if (!source || source.width < 1 || source.height < 1) return null; + + const side = Math.min(source.width, source.height); + const sx = (source.width - side) / 2; + const sy = (source.height - side) / 2; + const size = Math.max(1, Math.min(Math.floor(pixelSize), Math.floor(side))); + + const target = deps.createTarget(size); + if (!target) return null; + target.drawImage(source, sx, sy, side, side, 0, 0, size, size); + + for (const [type, quality] of ENCODINGS) { + let encoded: string; + try { + encoded = target.toDataURL(type, quality); + } catch { + continue; + } + // A webview without an encoder for `type` returns some other format + // instead of failing, so an unrequested prefix means "unsupported". + if (!encoded.startsWith(`data:${type};base64,`)) continue; + if (encoded.length <= maxDataUrlLength) return encoded; + } + return null; + } catch { + return null; + } finally { + source?.close?.(); + } +}