feat(desktop): fuzzy emoji shortcode autocomplete (#1945)

Signed-off-by: Torsten Rudolf <trudolf@squareup.com>

desktop/src/features/messages/lib/useEmojiAutocomplete.ts

@@ -4,6 +4,7 @@ import { init, SearchIndex } from "emoji-mart";
 import data from "@emoji-mart/data";
 
 import type { CustomEmoji } from "@/shared/lib/remarkCustomEmoji";
+import { fuzzyStandardEmoji, rankByShortcode } from "@/shared/lib/emojiSearch";
 import { rewriteRelayUrl } from "@/shared/lib/mediaUrl";
 import type { AutocompleteEdit } from "./useRichTextEditor";
 
@@ -75,17 +76,18 @@ export function useEmojiAutocomplete(customEmoji: CustomEmoji[] = []) {
 
     let cancelled = false;
 
-    // Custom emoji match by shortcode prefix/substring (case-insensitive).
-    const q = emojiQuery.toLowerCase();
-    const customMatches: EmojiSuggestion[] = customEmojiRef.current
-      .filter((e) => e.shortcode.toLowerCase().includes(q))
-      .slice(0, MAX_RESULTS)
-      .map((e) => ({
-        id: e.shortcode,
-        name: e.shortcode,
-        native: "",
-        url: rewriteRelayUrl(e.url),
-      }));
+    // Custom emoji match by shortcode, separator-insensitive and fuzzy.
+    const customMatches: EmojiSuggestion[] = rankByShortcode(
+      emojiQuery,
+      customEmojiRef.current,
+      (e) => e.shortcode,
+      MAX_RESULTS,
+    ).map((e) => ({
+      id: e.shortcode,
+      name: e.shortcode,
+      native: "",
+      url: rewriteRelayUrl(e.url),
+    }));
 
     SearchIndex.search(emojiQuery)
       .then(
@@ -104,8 +106,22 @@ export function useEmojiAutocomplete(customEmoji: CustomEmoji[] = []) {
               native: emoji.skins[0]?.native ?? "",
             }))
             .filter((e) => e.native !== "");
-          // Custom emoji first (community-specific), then standard, capped.
-          const merged = [...customMatches, ...standard].slice(0, MAX_RESULTS);
+          // Top up remaining slots with fuzzy shortcode matches emoji-mart
+          // missed — its token-prefix search can't cross `_` (so `pointup`
+          // finds nothing). Skip ids already shown to avoid duplicates.
+          const shown = new Set<string>(
+            [...customMatches, ...standard].map((e) => e.id),
+          );
+          const fuzzy: EmojiSuggestion[] = fuzzyStandardEmoji(
+            emojiQuery,
+            MAX_RESULTS - customMatches.length - standard.length,
+            shown,
+          ).map((e) => ({ id: e.id, name: e.name, native: e.native }));
+          // Custom emoji first (community-specific), then standard, then fuzzy.
+          const merged = [...customMatches, ...standard, ...fuzzy].slice(
+            0,
+            MAX_RESULTS,
+          );
           setSuggestions(merged);
           setEmojiSelectedIndex(0);
         },

desktop/src/shared/lib/emojiSearch.test.mjs

@@ -0,0 +1,94 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+  fuzzyStandardEmoji,
+  normalizeShortcode,
+  rankByShortcode,
+  scoreShortcodeMatch,
+} from "./emojiSearch.ts";
+
+test("normalizeShortcode lowercases and strips separators", () => {
+  assert.equal(normalizeShortcode(":Point_Up:"), "pointup");
+  assert.equal(normalizeShortcode("man-woman-boy"), "manwomanboy");
+  assert.equal(normalizeShortcode("thumbs up"), "thumbsup");
+});
+
+test("separator-only difference is an exact match (pointup == point_up)", () => {
+  const m = scoreShortcodeMatch("pointup", "point_up");
+  assert.ok(m);
+  assert.equal(m.tier, 0); // both normalize to "pointup"
+});
+
+test("interior substring across `_` matches (ntup -> point_up)", () => {
+  const m = scoreShortcodeMatch("ntup", "point_up");
+  assert.ok(m);
+  assert.equal(m.tier, 2); // substring, not a prefix
+});
+
+test("subsequence matches with omitted chars (pntup -> point_up)", () => {
+  const m = scoreShortcodeMatch("pntup", "point_up");
+  assert.ok(m);
+  assert.equal(m.tier, 3); // subsequence
+});
+
+test("leading + interior omission still matches via subsequence (ointp)", () => {
+  const m = scoreShortcodeMatch("ointp", "point_up");
+  assert.ok(m);
+  assert.equal(m.tier, 3);
+});
+
+test("exact and prefix rank above substring and subsequence", () => {
+  assert.equal(scoreShortcodeMatch("pointup", "point_up").tier, 0); // exact
+  assert.equal(scoreShortcodeMatch("point", "point_up").tier, 1); // prefix
+  assert.equal(scoreShortcodeMatch("ntup", "point_up").tier, 2); // substring
+  assert.equal(scoreShortcodeMatch("pntup", "point_up").tier, 3); // subsequence
+});
+
+test("no match returns null", () => {
+  assert.equal(scoreShortcodeMatch("xyz", "point_up"), null);
+  assert.equal(scoreShortcodeMatch("", "point_up"), null);
+});
+
+test("rankByShortcode orders exact > prefix > substring > subsequence", () => {
+  const items = [
+    { code: "point" }, // exact
+    { code: "point_up" }, // -> "pointup": prefix (query is a prefix of it)
+    { code: "endpoint" }, // -> "endpoint": substring
+    { code: "pot_in_time" }, // -> "potintime": subsequence with gaps
+    { code: "unrelated" }, // no match
+  ];
+  const ranked = rankByShortcode("point", items, (i) => i.code, 10).map(
+    (i) => i.code,
+  );
+  assert.deepEqual(ranked, [
+    "point", // exact
+    "point_up", // prefix
+    "endpoint", // substring
+    "pot_in_time", // subsequence
+  ]);
+});
+
+test("rankByShortcode respects the limit", () => {
+  const items = ["smile", "smiley", "smiling", "smirk"].map((code) => ({
+    code,
+  }));
+  const ranked = rankByShortcode("smi", items, (i) => i.code, 2);
+  assert.equal(ranked.length, 2);
+});
+
+test("fuzzyStandardEmoji surfaces point_up for `pointup`", () => {
+  const hits = fuzzyStandardEmoji("pointup", 8, new Set());
+  const ids = hits.map((e) => e.id);
+  assert.ok(ids.includes("point_up"), `expected point_up in ${ids.join(",")}`);
+  assert.ok(hits.every((e) => e.native !== ""));
+});
+
+test("fuzzyStandardEmoji excludes already-shown ids", () => {
+  const hits = fuzzyStandardEmoji("pointup", 8, new Set(["point_up"]));
+  assert.ok(!hits.some((e) => e.id === "point_up"));
+});
+
+test("fuzzyStandardEmoji returns nothing when limit is non-positive", () => {
+  assert.deepEqual(fuzzyStandardEmoji("pointup", 0, new Set()), []);
+});

desktop/src/shared/lib/emojiSearch.ts

@@ -0,0 +1,165 @@
+import data from "@emoji-mart/data";
+
+/**
+ * Shortcode fuzzy matching for the `:shortcode` emoji autocomplete.
+ *
+ * emoji-mart's own search is token-prefix based, so it can't cross the `_`
+ * separator in a shortcode — `pointup` never finds `point_up`. This module adds
+ * separator-insensitive matching in tiers so recall improves without reordering
+ * the good hits: exact > prefix > substring > subsequence. Tiers 1–3 keep normal
+ * queries identical to before; the subsequence tier only surfaces looser matches
+ * (`pntup` → `point_up`) and always ranks last.
+ */
+
+// Lower tier = stronger match.
+const TIER_EXACT = 0;
+const TIER_PREFIX = 1;
+const TIER_SUBSTRING = 2;
+const TIER_SUBSEQUENCE = 3;
+
+/** Lowercase and drop separators (`:`, `_`, `-`, whitespace) so matching is
+ *  insensitive to shortcode punctuation. `point_up` and `pointup` collapse to
+ *  the same normalized form. */
+export function normalizeShortcode(value: string): string {
+  return value.toLowerCase().replace(/[:_\s-]/g, "");
+}
+
+export interface ShortcodeMatch {
+  tier: number;
+  /** Lower is better within a tier. */
+  score: number;
+}
+
+/**
+ * If `query` is a subsequence of `target` (all chars in order, gaps allowed),
+ * return the span of the match (`lastIndex - firstIndex`) as a tightness score —
+ * smaller is tighter, hence better. Returns null when not a subsequence.
+ */
+function subsequenceSpan(query: string, target: string): number | null {
+  let first = -1;
+  let last = -1;
+  let qi = 0;
+  for (let ti = 0; ti < target.length && qi < query.length; ti++) {
+    if (target[ti] === query[qi]) {
+      if (first === -1) first = ti;
+      last = ti;
+      qi++;
+    }
+  }
+  if (qi < query.length) return null;
+  return last - first;
+}
+
+/**
+ * Score `query` against a single `shortcode`. Returns null when there is no
+ * match at any tier. Both sides are normalized first, so separators are ignored.
+ */
+export function scoreShortcodeMatch(
+  query: string,
+  shortcode: string,
+): ShortcodeMatch | null {
+  const q = normalizeShortcode(query);
+  if (q.length === 0) return null;
+  const target = normalizeShortcode(shortcode);
+  if (target.length === 0) return null;
+
+  if (q === target) return { tier: TIER_EXACT, score: 0 };
+  if (target.startsWith(q)) return { tier: TIER_PREFIX, score: 0 };
+  const idx = target.indexOf(q);
+  if (idx !== -1) return { tier: TIER_SUBSTRING, score: idx };
+  const span = subsequenceSpan(q, target);
+  if (span !== null) return { tier: TIER_SUBSEQUENCE, score: span };
+  return null;
+}
+
+/**
+ * Rank `items` by how well their shortcode matches `query`, best first, capped
+ * at `limit`. Ordering: tier, then in-tier score, then shorter shortcode (more
+ * specific), then alphabetical for a stable result.
+ */
+export function rankByShortcode<T>(
+  query: string,
+  items: readonly T[],
+  shortcodeOf: (item: T) => string,
+  limit: number,
+): T[] {
+  if (limit <= 0) return [];
+  const scored: Array<{
+    item: T;
+    tier: number;
+    score: number;
+    len: number;
+    code: string;
+  }> = [];
+  for (const item of items) {
+    const code = shortcodeOf(item);
+    const match = scoreShortcodeMatch(query, code);
+    if (match) {
+      scored.push({
+        item,
+        tier: match.tier,
+        score: match.score,
+        len: code.length,
+        code,
+      });
+    }
+  }
+  scored.sort(
+    (a, b) =>
+      a.tier - b.tier ||
+      a.score - b.score ||
+      a.len - b.len ||
+      (a.code < b.code ? -1 : a.code > b.code ? 1 : 0),
+  );
+  return scored.slice(0, limit).map((s) => s.item);
+}
+
+export interface StandardEmoji {
+  id: string;
+  name: string;
+  native: string;
+}
+
+type EmojiMartData = {
+  emojis?: Record<
+    string,
+    {
+      id?: string;
+      name?: string;
+      skins?: Array<{ native?: string }>;
+    }
+  >;
+};
+
+// Flat index of standard emoji built once from the emoji-mart dataset. ~1.8k
+// short entries; scanning it per keystroke (behind the composer's debounce) is
+// sub-millisecond. Module-level so the cost is paid once, not per hook mount.
+let standardIndex: StandardEmoji[] | null = null;
+
+function getStandardIndex(): StandardEmoji[] {
+  if (standardIndex) return standardIndex;
+  const emojis = (data as EmojiMartData).emojis ?? {};
+  const out: StandardEmoji[] = [];
+  for (const [id, emoji] of Object.entries(emojis)) {
+    const native = emoji.skins?.[0]?.native ?? "";
+    if (!native) continue;
+    out.push({ id: emoji.id ?? id, name: emoji.name ?? id, native });
+  }
+  standardIndex = out;
+  return out;
+}
+
+/**
+ * Fuzzy shortcode matches over the standard emoji set, used to top up the
+ * autocomplete when emoji-mart's own search misses (it can't cross `_`). Skips
+ * ids already shown (`excludeIds`) so it only adds, never duplicates.
+ */
+export function fuzzyStandardEmoji(
+  query: string,
+  limit: number,
+  excludeIds: ReadonlySet<string>,
+): StandardEmoji[] {
+  if (limit <= 0) return [];
+  const candidates = getStandardIndex().filter((e) => !excludeIds.has(e.id));
+  return rankByShortcode(query, candidates, (e) => e.id, limit);
+}