diff --git a/mobile/ios/NativeControlTests/Fixtures/gallery-video.mp4 b/mobile/ios/NativeControlTests/Fixtures/gallery-video.mp4 new file mode 100644 index 0000000000..537a209080 Binary files /dev/null and b/mobile/ios/NativeControlTests/Fixtures/gallery-video.mp4 differ diff --git a/mobile/ios/NativeControlTests/NativePhotoZoomTests.swift b/mobile/ios/NativeControlTests/NativePhotoZoomTests.swift index 34b98c71f5..f1d78ecaa7 100644 --- a/mobile/ios/NativeControlTests/NativePhotoZoomTests.swift +++ b/mobile/ios/NativeControlTests/NativePhotoZoomTests.swift @@ -63,8 +63,54 @@ struct NativePhotoZoomTests { viewer.updateFrame(data, at: 0) precondition(closedPhoto.image === closedImage, "Closed viewers must ignore continuing frames") print("PASS: native photo window, stale responses, eviction and teardown") + videoJourney(data: data) + } } + @available(iOS 18.0, *) + @MainActor static func videoJourney(data: Data) { + let path = CommandLine.arguments[1] + let viewer = NativePhotoZoomViewController(image: UIImage(data: data)!, + labels: ["Clip", "Photo", "Second clip"], initialIndex: 0, + canReply: true, canMore: true, canDelete: false, videoIndices: [0, 2]) + var requested: [Int] = [] + var lateVideo: FlutterResult? + viewer.request = { method, index, _, completion in + if method == "video" { + requested.append(index) + if index == 0 { completion(path) } else { lateVideo = completion } + } else { completion(method == "select" ? nil : FlutterStandardTypedData(bytes: data)) } + } + viewer.loadViewIfNeeded() + viewer.view.frame = CGRect(x: 0, y: 0, width: 390, height: 844) + viewer.view.layoutIfNeeded() + viewer.viewDidAppear(false) + let pager = viewer.children.compactMap { $0 as? UIPageViewController }.first! + let indicator = descendants(viewer.view).compactMap { $0 as? UIPageControl }.first! + let first = pager.viewControllers!.first! + first.loadViewIfNeeded() + let playerController = first.children.compactMap { $0 as? AVPlayerViewController }.first! + let player = playerController.player! + let item = player.currentItem! + let deadline = Date().addingTimeInterval(5) + while item.status == .unknown && Date() < deadline { + RunLoop.current.run(until: Date().addingTimeInterval(0.01)) + } + precondition(item.status == .readyToPlay, "Real MP4 fixture must become playable") + precondition(requested == [0], "Only the selected video may download") + indicator.currentPage = 2; sendValueChanged(indicator) + precondition(player.currentItem == nil && playerController.player == nil, "Paging releases the old player before Dart deletes its file") + precondition(requested == [0, 2], "Paging requests the selected video") + indicator.currentPage = 0; sendValueChanged(indicator) + lateVideo?(path) + let current = pager.viewControllers!.first! + current.loadViewIfNeeded() + let active = current.children.compactMap { $0 as? AVPlayerViewController }.first! + precondition(active.player != nil, "Returning to a video restores its player") + viewer.viewDidDisappear(false) + precondition(active.player == nil, "Closing releases the player before returning to Flutter") + print("PASS: native video decoding, selected-only loading, paging and teardown") + } @MainActor static func sendValueChanged(_ control: UIControl) { // A command-line UIKit fixture has no UIApplication action dispatcher. for target in control.allTargets { diff --git a/mobile/ios/Runner/NativePhotoZoomCoordinator.swift b/mobile/ios/Runner/NativePhotoZoomCoordinator.swift index e1b25ae039..3a01176542 100644 --- a/mobile/ios/Runner/NativePhotoZoomCoordinator.swift +++ b/mobile/ios/Runner/NativePhotoZoomCoordinator.swift @@ -1,5 +1,6 @@ import Flutter import UIKit +import AVKit /// Preview-only presentation. Flutter owns authenticated image loading. final class NativePhotoZoomCoordinator { @@ -64,7 +65,8 @@ final class NativePhotoZoomCoordinator { image: image, labels: labels, initialIndex: initial, canReply: args["canReply"] as? Bool ?? false, canMore: args["canMore"] as? Bool ?? false, - canDelete: args["canDelete"] as? Bool ?? false + canDelete: args["canDelete"] as? Bool ?? false, + videoIndices: Set(args["videoIndices"] as? [Int] ?? []) ) controller.request = { [weak self] method, index, action, completion in var payload: [String: Any] = ["session": session, "index": index] @@ -151,6 +153,9 @@ private final class NativePhotoZoomViewController: UIViewController, UIPageViewC private let reply = UIButton(type: .system) private let more = UIButton(type: .system) private let labels: [String] + private let videoIndices: Set + private var videoPath: (Int, String)? + private var appeared = false private var previews: [Int: UIImage] = [:] private var images: [Int: UIImage] = [:] private var pending: [String: UUID] = [:] @@ -165,9 +170,10 @@ private final class NativePhotoZoomViewController: UIViewController, UIPageViewC var request: ((String, Int, String?, @escaping FlutterResult) -> Void)? var onSelection: ((UIImage?, [String: Any]?) -> Void)? - init(image: UIImage, labels: [String], initialIndex: Int, canReply: Bool, canMore: Bool, canDelete: Bool) { + init(image: UIImage, labels: [String], initialIndex: Int, canReply: Bool, canMore: Bool, canDelete: Bool, videoIndices: Set = []) { self.previews[initialIndex] = image self.labels = labels; index = initialIndex + self.videoIndices = videoIndices self.canReply = canReply; self.canMore = canMore; self.canDelete = canDelete super.init(nibName: nil, bundle: nil) } @@ -185,10 +191,10 @@ private final class NativePhotoZoomViewController: UIViewController, UIPageViewC addChild(pager); view.addSubview(pager.view); pager.didMove(toParent: self) pager.setViewControllers([page(index)], direction: .forward, animated: false) loadWindow() - configure(close, symbol: "xmark", label: "Close photo", selector: #selector(closePhoto)) + configure(close, symbol: "xmark", label: videoIndices.isEmpty ? "Close photo" : "Close media", selector: #selector(closePhoto)) configure(reply, symbol: "arrowshape.turn.up.left", label: "Reply in thread", selector: #selector(replyToMessage)) configure(more, symbol: "ellipsis", label: "More image actions", selector: nil) - reply.isEnabled = canReply; more.isEnabled = canMore + reply.isEnabled = canReply; updateMore() more.showsMenuAsPrimaryAction = true var menu = [UIAction(title: "Save image", image: UIImage(systemName: "square.and.arrow.down")) { [weak self] _ in self?.perform("save") }, UIAction(title: "Share image", image: UIImage(systemName: "square.and.arrow.up")) { [weak self] _ in self?.perform("share") }, @@ -202,7 +208,7 @@ private final class NativePhotoZoomViewController: UIViewController, UIPageViewC indicator.hidesForSinglePage = true indicator.currentPageIndicatorTintColor = .white indicator.pageIndicatorTintColor = .gray - indicator.accessibilityLabel = "Photo" + indicator.accessibilityLabel = videoIndices.isEmpty ? "Photo" : "Attachment" indicator.addTarget(self, action: #selector(selectPage), for: .valueChanged) let pageContainer = UIView() let pageSurface: UIVisualEffectView @@ -265,7 +271,7 @@ private final class NativePhotoZoomViewController: UIViewController, UIPageViewC closeCenterConstraint?.constant = navigationSizingBar.sizeThatFits(view.bounds.size).height / 2 } func shouldBeginDismissal(location: CGPoint, velocity: CGVector) -> Bool { - !paging && !finishing && pager.view.frame.contains(location) + !paging && !finishing && !videoIndices.contains(index) && pager.view.frame.contains(location) && velocity.dy > abs(velocity.dx) * 1.5 && ((pager.viewControllers?.first as? NativePhotoPage)?.isAtRestingZoom ?? false) } @@ -322,8 +328,9 @@ private final class NativePhotoZoomViewController: UIViewController, UIPageViewC load("preview", value) } for page in pager.children.compactMap({ $0 as? NativePhotoPage }) where isNearby(page.index) { - load("image", page.index) + if !page.isVideo { load("image", page.index) } } + loadVideo() } private func load(_ method: String, _ value: Int) { guard !finishing, isNearby(value), @@ -352,30 +359,75 @@ private final class NativePhotoZoomViewController: UIViewController, UIPageViewC // Flutter decodes authenticated originals and forwards continuing frames. // Ignore old pages and closed presentations before allocating a UIImage. func updateFrame(_ data: Data, at value: Int) { - guard !finishing, value == index, let image = UIImage(data: data) else { return } + guard !finishing, !videoIndices.contains(value), value == index, let image = UIImage(data: data) else { return } images[value] = image for page in pager.children.compactMap({ $0 as? NativePhotoPage }) where page.index == value { page.setImage(image) } } private func page(_ value: Int) -> NativePhotoPage { - let page = NativePhotoPage(index: value, label: labels[value], image: images[value] ?? previews[value]) - load("image", value) + let page = NativePhotoPage(index: value, label: labels[value], image: images[value] ?? previews[value], isVideo: videoIndices.contains(value)) + page.onRetry = { [weak self] in self?.videoPath = nil; self?.loadVideo() } + if page.isVideo { + if let (loaded, path) = videoPath, loaded == value { page.setVideo(path, autoplay: appeared) } + } else { load("image", value) } return page } + private func loadVideo() { + let value = index + guard !finishing, videoIndices.contains(value), videoPath?.0 != value else { return } + let key = "video:\(value)" + guard pending[key] == nil else { return } + let token = UUID(); pending[key] = token + for page in pager.children.compactMap({ $0 as? NativePhotoPage }) where page.index == value { page.showVideoLoading() } + request?("video", value, nil) { [weak self] result in + guard let self, !self.finishing, self.index == value, self.pending[key] == token else { return } + self.pending.removeValue(forKey: key) + let path = result as? String + if let path { self.videoPath = (value, path) } + for page in self.pager.children.compactMap({ $0 as? NativePhotoPage }) where page.index == value { + page.setVideo(path, autoplay: self.appeared) + } + } + } + private func stopVideos() { + for page in pager.children.compactMap({ $0 as? NativePhotoPage }) { page.stopVideo() } + videoPath = nil + pending = pending.filter { !$0.key.hasPrefix("video:") } + } + override func viewDidAppear(_ animated: Bool) { + super.viewDidAppear(animated); appeared = true + (pager.viewControllers?.first as? NativePhotoPage)?.playVideo() + } + override func viewWillDisappear(_ animated: Bool) { + super.viewWillDisappear(animated); appeared = false + for page in pager.children.compactMap({ $0 as? NativePhotoPage }) { page.pauseVideo() } + } func pageViewController(_ pageViewController: UIPageViewController, viewControllerBefore viewController: UIViewController) -> UIViewController? { guard let page = viewController as? NativePhotoPage, page.index > 0 else { return nil }; return self.page(page.index - 1) } func pageViewController(_ pageViewController: UIPageViewController, viewControllerAfter viewController: UIViewController) -> UIViewController? { guard let page = viewController as? NativePhotoPage, page.index + 1 < labels.count else { return nil }; return self.page(page.index + 1) } - func pageViewController(_ pageViewController: UIPageViewController, willTransitionTo pendingViewControllers: [UIViewController]) { paging = true; indicator.isEnabled = false } + func pageViewController(_ pageViewController: UIPageViewController, willTransitionTo pendingViewControllers: [UIViewController]) { + paging = true; indicator.isEnabled = false + for page in pager.children.compactMap({ $0 as? NativePhotoPage }) { page.pauseVideo() } + } func pageViewController(_ pageViewController: UIPageViewController, didFinishAnimating finished: Bool, previousViewControllers: [UIViewController], transitionCompleted completed: Bool) { paging = false; indicator.isEnabled = true if completed, let page = pager.viewControllers?.first as? NativePhotoPage { selected(page.index) } + else if appeared { (pager.viewControllers?.first as? NativePhotoPage)?.playVideo() } + } + private func updateMore() { + let video = videoIndices.contains(index) + more.isEnabled = canMore && !video + more.alpha = video ? 0 : 1 + more.accessibilityElementsHidden = video } private func selected(_ value: Int) { + stopVideos() index = value; indicator.currentPage = value + updateMore() trimWindow() // Move Dart's request window before asking for newly adjacent images. syncSelection { [weak self] in self?.loadWindow() } @@ -403,6 +455,7 @@ private final class NativePhotoZoomViewController: UIViewController, UIPageViewC @objc private func replyToMessage() { if canReply { finish("reply") } } private func finish(_ result: String) { guard !finishing else { return }; finishing = true; action = result + stopVideos() syncSelection { [weak self] in self?.dismiss(animated: !UIAccessibility.isReduceMotionEnabled) } } private func confirmDelete() { @@ -415,7 +468,7 @@ private final class NativePhotoZoomViewController: UIViewController, UIPageViewC present(alert, animated: true) } private func perform(_ action: String) { - guard canMore, !acting, !finishing else { return }; acting = true; more.isEnabled = false + guard canMore, !videoIndices.contains(index), !acting, !finishing else { return }; acting = true; more.isEnabled = false request?("action", index, action) { [weak self] result in guard let self else { return }; self.acting = false; self.more.isEnabled = self.canMore if let error = result as? FlutterError { self.showStatus(error.message ?? "Could not complete action"); return } @@ -432,6 +485,7 @@ private final class NativePhotoZoomViewController: UIViewController, UIPageViewC super.viewDidDisappear(animated) if isBeingDismissed || presentingViewController == nil { finishing = true + stopVideos() pending.removeAll(); previews.removeAll(); images.removeAll() request = nil onClose?(action); onClose = nil @@ -443,23 +497,29 @@ private final class NativePhotoZoomViewController: UIViewController, UIPageViewC @available(iOS 18.0, *) private final class NativePhotoPage: UIViewController, UIScrollViewDelegate { let index: Int + let isVideo: Bool + var onRetry: (() -> Void)? + private lazy var video = AVPlayerViewController() + private let retry = UIButton(type: .system) + private var observation: NSKeyValueObservation? private let imageView = UIImageView() private let scroll = UIScrollView() private let status = UILabel() private var previousSize = CGSize.zero var image: UIImage? { imageView.image } - var isAtRestingZoom: Bool { scroll.zoomScale <= scroll.minimumZoomScale + 0.01 } + var isAtRestingZoom: Bool { !isVideo && scroll.zoomScale <= scroll.minimumZoomScale + 0.01 } func photoFrame(in target: UIView) -> CGRect? { + if isVideo, video.player != nil, !video.videoBounds.isEmpty { return video.view.convert(video.videoBounds, to: target) } guard let image, image.size.width > 0, image.size.height > 0 else { return nil } let bounds = imageView.bounds let scale = min(bounds.width / image.size.width, bounds.height / image.size.height) let size = CGSize(width: image.size.width * scale, height: image.size.height * scale) return imageView.convert(CGRect(x: bounds.midX - size.width / 2, y: bounds.midY - size.height / 2, width: size.width, height: size.height), to: target) } - init(index: Int, label: String, image: UIImage?) { - self.index = index; super.init(nibName: nil, bundle: nil) - imageView.image = image; imageView.accessibilityLabel = label; imageView.isAccessibilityElement = true - status.text = image == nil ? "Loading photo…" : nil + init(index: Int, label: String, image: UIImage?, isVideo: Bool = false) { + self.index = index; self.isVideo = isVideo; super.init(nibName: nil, bundle: nil) + imageView.image = image; imageView.accessibilityLabel = label; imageView.isAccessibilityElement = !isVideo + status.text = isVideo ? "Loading video…" : image == nil ? "Loading photo…" : nil } required init?(coder: NSCoder) { nil } override func viewDidLoad() { @@ -472,11 +532,60 @@ private final class NativePhotoPage: UIViewController, UIScrollViewDelegate { status.textColor = .white; status.textAlignment = .center; status.numberOfLines = 0 status.font = .preferredFont(forTextStyle: .body); status.adjustsFontForContentSizeCategory = true view.addSubview(status) - let tap = UITapGestureRecognizer(target: self, action: #selector(toggleZoom)); tap.numberOfTapsRequired = 2; scroll.addGestureRecognizer(tap) + if isVideo { + video.allowsPictureInPicturePlayback = false + video.view.backgroundColor = .clear + video.view.isHidden = true + addChild(video); view.insertSubview(video.view, belowSubview: status); video.didMove(toParent: self) + retry.setTitle("Retry video", for: .normal) + retry.titleLabel?.font = .preferredFont(forTextStyle: .body) + retry.titleLabel?.adjustsFontForContentSizeCategory = true + retry.addTarget(self, action: #selector(retryVideo), for: .touchUpInside) + retry.isHidden = true; view.addSubview(retry) + } + let tap = UITapGestureRecognizer(target: self, action: #selector(toggleZoom)); tap.numberOfTapsRequired = 2; if !isVideo { scroll.addGestureRecognizer(tap) } } - func setImage(_ image: UIImage?) { imageView.image = image; status.text = image == nil ? "Image unavailable" : nil } + func setImage(_ image: UIImage?) { + imageView.image = image + if !isVideo { status.text = image == nil ? "Image unavailable" : nil } + } + func showVideoLoading() { guard isVideo else { return }; status.text = "Loading video…"; retry.isHidden = true } + func setVideo(_ path: String?, autoplay: Bool) { + guard isVideo else { return } + loadViewIfNeeded() + stopVideo() + guard let path, FileManager.default.fileExists(atPath: path) else { videoFailed(); return } + let item = AVPlayerItem(url: URL(fileURLWithPath: path)) + let player = AVPlayer(playerItem: item) + video.player = player + observation = item.observe(\.status, options: [.initial, .new]) { [weak self, weak item] _, _ in + DispatchQueue.main.async { + guard let self, let item, self.video.player?.currentItem === item else { return } + if item.status == .failed { self.videoFailed() } + else if item.status == .readyToPlay { self.status.text = nil; self.video.view.isHidden = false; self.scroll.isHidden = true } + } + } + if autoplay { player.play() } + } + func playVideo() { if isVideo { video.player?.play() } } + func pauseVideo() { if isVideo { video.player?.pause() } } + func stopVideo() { + guard isVideo else { return } + observation = nil + video.player?.pause(); video.player?.replaceCurrentItem(with: nil); video.player = nil + if isViewLoaded { video.view.isHidden = true; scroll.isHidden = false } + } + private func videoFailed() { + stopVideo(); status.text = "Video unavailable"; retry.isHidden = false + UIAccessibility.post(notification: .announcement, argument: status.text) + } + @objc private func retryVideo() { showVideoLoading(); onRetry?() } override func viewDidLayoutSubviews() { super.viewDidLayoutSubviews(); scroll.frame = view.bounds; status.frame = view.bounds.insetBy(dx: 24, dy: 100) + if isVideo { + video.view.frame = view.bounds + retry.frame = CGRect(x: 24, y: view.bounds.midY + 32, width: max(0, view.bounds.width - 48), height: 48) + } guard previousSize != scroll.bounds.size else { return }; previousSize = scroll.bounds.size scroll.zoomScale = 1; imageView.frame = scroll.bounds; scroll.contentSize = scroll.bounds.size } diff --git a/mobile/lib/features/channels/message_content.dart b/mobile/lib/features/channels/message_content.dart index 7b232f68e0..b86a201ac7 100644 --- a/mobile/lib/features/channels/message_content.dart +++ b/mobile/lib/features/channels/message_content.dart @@ -41,6 +41,7 @@ import 'channel_ids_by_name_provider.dart'; import 'local_send_bubble_surface.dart'; import 'media_viewer_page.dart'; import 'native_photo_zoom_preview.dart'; +import 'native_video_files.dart'; import 'message_content/link_normalizer.dart'; import 'message_content/markdown_escapes.dart'; import 'message_content/preview_text.dart'; @@ -58,6 +59,7 @@ import 'message_media_geometry.dart'; import 'voice_note_attachment.dart'; part 'message_content/media_carousel.dart'; +part 'message_content/mention_markdown.dart'; part 'message_content/media_stack.dart'; part 'message_content/media_stack_tuning.dart'; part 'message_content/message_preview.dart'; @@ -314,7 +316,11 @@ class MessageContent extends HookConsumerWidget { content, ]); final trailingGallery = maxLines == null - ? extractTrailingImageGallery(normalizedContent, imetaByUrl) + ? extractTrailingImageGallery( + normalizedContent, + imetaByUrl, + includeVideos: bubble, + ) : null; final galleryText = trailingGallery?.content ?? normalizedContent; // gpt_markdown has no backslash escapes, so hold escaped characters as @@ -578,7 +584,7 @@ class MessageContent extends HookConsumerWidget { Padding( padding: const EdgeInsets.symmetric(horizontal: Grid.xs), child: Text( - '${trailingGallery.items.length} photos', + '${trailingGallery.items.length} ${trailingGallery.items.any((item) => item.isVideo) ? 'attachments' : 'photos'}', key: const ValueKey('message-media-stack-count'), style: context.textTheme.labelSmall?.copyWith( color: context.colors.onSurfaceVariant, @@ -1118,82 +1124,3 @@ class _MessageCodeBlock extends HookWidget { ); } } - -class _MentionMd extends InlineMd { - final Map> bindings; - final Map displayLabels; - final Map mentionNames; - final Map mentionLabels; - final Set agentMentionPubkeys; - final void Function(String pubkey)? onMentionTap; - late final RegExp _exp = _buildPrefixPattern( - prefix: '@', - knownNames: bindings.keys.map(_markdownMentionName), - genericTokenPattern: r'[A-Za-z0-9_][A-Za-z0-9_\u00A0-]*', - ); - - _MentionMd({ - required this.bindings, - required this.displayLabels, - required this.mentionNames, - required this.mentionLabels, - required this.agentMentionPubkeys, - this.onMentionTap, - }); - - /// Excluded from link labels: this component renders a [WidgetSpan], and a - /// placeholder nested inside the link's own placeholder does not paint on - /// iOS — an authored `[@mention](url)` renders as nothing. Link resolution - /// wins over token detection inside a label. - @override - Set get scopes => MarkdownComponent.allScopesExceptLinkLabel; - - @override - RegExp get exp => _exp; - - @override - InlineSpan span( - BuildContext context, - String text, - final GptMarkdownConfig config, - ) { - final raw = exp.firstMatch(text.trim())?.group(0); - if (raw == null) { - return TextSpan(text: text, style: config.style); - } - - final name = raw.substring(1).replaceAll('\u00A0', ' ').toLowerCase(); - final matches = bindings[name] ?? const {}; - final pubkey = matches.length == 1 ? matches.single : null; - if (bindings.containsKey(name) && matches.length != 1) { - return TextSpan(text: text, style: config.style); - } - final displayName = name.contains(RegExp(r'\([0-9a-f]{64}\)')) - ? displayLabels[name] - : mentionLabels[pubkey] ?? mentionNames[pubkey]; - - final isAgent = - pubkey != null && agentMentionPubkeys.contains(pubkey.toLowerCase()); - final fullLabel = displayName ?? raw.substring(1); - final visibleLabel = fullLabel.replaceAllMapped( - RegExp(r'\(([0-9a-f]{64})\)'), - (m) => '(${m[1]!.substring(0, 8)}…${m[1]!.substring(60)})', - ); - final pill = MessageMentionPill( - label: visibleLabel, - semanticsLabel: fullLabel, - isAgent: isAgent, - textStyle: config.style, - ); - - return WidgetSpan( - alignment: PlaceholderAlignment.baseline, - baseline: TextBaseline.alphabetic, - child: pubkey != null && onMentionTap != null - ? GestureDetector(onTap: () => onMentionTap!(pubkey), child: pill) - : pill, - ); - } -} - -String _markdownMentionName(String name) => name.replaceAll(' ', '\u00A0'); diff --git a/mobile/lib/features/channels/message_content/media_carousel.dart b/mobile/lib/features/channels/message_content/media_carousel.dart index 55b8e20f82..036a5a3a07 100644 --- a/mobile/lib/features/channels/message_content/media_carousel.dart +++ b/mobile/lib/features/channels/message_content/media_carousel.dart @@ -1,7 +1,7 @@ part of '../message_content.dart'; class _MessageGalleryPrecache extends HookWidget { - final List> providers; + final List?> providers; final int focusedIndex; const _MessageGalleryPrecache({ @@ -27,7 +27,8 @@ class _MessageGalleryPrecache extends HookWidget { ]; prefetch.update(context, [ for (final i in indices) - if (i >= 0 && i < providers.length) providers[i], + if (i >= 0 && i < providers.length && providers[i] != null) + providers[i]!, ]); }); return () => cancelled = true; diff --git a/mobile/lib/features/channels/message_content/media_stack.dart b/mobile/lib/features/channels/message_content/media_stack.dart index 643232821c..17a5807ac3 100644 --- a/mobile/lib/features/channels/message_content/media_stack.dart +++ b/mobile/lib/features/channels/message_content/media_stack.dart @@ -42,6 +42,8 @@ class _MessageImageStack extends HookConsumerWidget { final reducedMotion = MediaQuery.disableAnimationsOf(context); final auth = ref.watch(mediaGetAuthServiceProvider); final index = value.round() % items.length; + final hasVideos = items.any((item) => item.isVideo); + final itemName = hasVideos ? 'Attachment' : 'Photo'; void settle(double target, {bool instant = false}) { if (reducedMotion || instant) { @@ -75,18 +77,25 @@ class _MessageImageStack extends HookConsumerWidget { .ceilToDouble(); final providers = [ for (final item in items) - ResizeImage.resizeIfNeeded( - physicalDecodeWidth.toInt(), - null, - MediaImageProvider( - url: item.url, - auth: auth, - client: ref.watch(mediaHttpClientProvider), + if (item.isVideo && item.posterUrl == null) + null + else + ResizeImage.resizeIfNeeded( + physicalDecodeWidth.toInt(), + null, + MediaImageProvider( + url: item.isVideo ? item.posterUrl! : item.url, + auth: auth, + client: ref.watch(mediaHttpClientProvider), + ), ), - ), + ]; + final imageIndices = [ + for (var i = 0; i < items.length; i++) + if (!items[i].isVideo) i, ]; final viewerItems = [ - for (var i = 0; i < items.length; i++) + for (final i in imageIndices) MediaViewerImage( url: items[i].url, heroTag: tags[i], @@ -100,15 +109,27 @@ class _MessageImageStack extends HookConsumerWidget { if (opening.value) return; opening.value = true; dragging.value = false; + NativeVideoFiles? videoFiles; try { final selected = position.value.round() % items.length; settle(position.value.roundToDouble(), instant: true); if (NativePhotoZoomPreview.enabled) { + videoFiles = NativeVideoFiles( + auth: auth, + client: ref.read(mediaHttpClientProvider), + index: selected, + ); await WidgetsBinding.instance.endOfFrame; if (!context.mounted) return; final gallery = await NativePhotoZoomPreview.open( context: context, sourceKeys: sourceKeys, + videoIndices: { + for (var i = 0; i < items.length; i++) + if (items[i].isVideo) i, + }, + loadVideo: (index) => + videoFiles!.load(index, items[index].url), previewProvider: providers[selected], previewProviders: providers, providers: [ @@ -128,6 +149,7 @@ class _MessageImageStack extends HookConsumerWidget { labels: [for (final item in items) item.semanticLabel], initialIndex: selected, onSelect: (index) async { + await videoFiles!.select(index); if (!context.mounted) return; dragging.value = false; start.value = index.toDouble(); @@ -140,6 +162,15 @@ class _MessageImageStack extends HookConsumerWidget { ); if (!gallery || !context.mounted) return; } + if (items[selected].isVideo) { + openVideoViewer( + context, + videoUrl: items[selected].url, + posterUrl: items[selected].posterUrl, + onReply: onReply, + ); + return; + } openImageViewer( context, imageUrl: items[selected].url, @@ -148,18 +179,20 @@ class _MessageImageStack extends HookConsumerWidget { previewDecodeWidth: decodeWidth, aspectRatio: items[selected].aspectRatio, galleryItems: viewerItems, - initialIndex: selected, + initialIndex: imageIndices.indexOf(selected), onSelectionChanged: (index) { if (!context.mounted) return; dragging.value = false; - start.value = index.toDouble(); + final originalIndex = imageIndices[index]; + start.value = originalIndex.toDouble(); distance.value = 0; - settle(index.toDouble(), instant: true); + settle(originalIndex.toDouble(), instant: true); }, onReply: onReply, onMore: onMore, ); } finally { + await videoFiles?.dispose(); opening.value = false; } } @@ -216,20 +249,48 @@ class _MessageImageStack extends HookConsumerWidget { ); } Widget card(int i) { - final photo = MediaImage( - url: items[i].url, - decodeWidth: decodeWidth, - fit: BoxFit.cover, - semanticLabel: items[i].semanticLabel, - frameBuilder: (context, child, frame, synchronous) => - frame != null || synchronous - ? child - : const MediaLoadingPlaceholder(label: 'Loading image'), - errorBuilder: (_, _, _) => const _MediaPreviewFallback( - icon: BuzzIcons.imageOff, - label: 'Image unavailable', - ), - ); + final item = items[i]; + Widget photo = item.isVideo && item.posterUrl == null + ? ColoredBox(color: context.colors.surfaceContainerHigh) + : MediaImage( + url: item.isVideo ? item.posterUrl! : item.url, + decodeWidth: decodeWidth, + fit: BoxFit.cover, + semanticLabel: items[i].semanticLabel, + frameBuilder: (context, child, frame, synchronous) => + frame != null || synchronous + ? child + : const MediaLoadingPlaceholder(label: 'Loading image'), + errorBuilder: (_, _, _) => const _MediaPreviewFallback( + icon: BuzzIcons.imageOff, + label: 'Image unavailable', + ), + ); + if (item.isVideo) { + photo = Stack( + fit: StackFit.expand, + children: [ + photo, + const ColoredBox(color: Color.fromRGBO(0, 0, 0, 0.28)), + Center( + child: Container( + width: 52, + height: 52, + decoration: const BoxDecoration( + color: Color.fromRGBO(0, 0, 0, 0.6), + shape: BoxShape.circle, + ), + child: Icon( + Icons.play_arrow, + key: ValueKey('message-media-stack-video:${item.url}'), + color: Colors.white, + size: 24, + ), + ), + ), + ], + ); + } return _StackPhotoCard( key: ValueKey('stack-card:$i'), pose: poses[i]!, @@ -242,9 +303,11 @@ class _MessageImageStack extends HookConsumerWidget { : offsets[i]! < 0 ? (1 + offsets[i]!).clamp(0.0, 1.0) : (3 - offsets[i]!).clamp(0.0, 1.0), - child: SizedBox( + child: RepaintBoundary( key: sourceKeys[i], - child: MediaViewerHero(tag: tags[i], child: photo), + child: item.isVideo + ? photo + : MediaViewerHero(tag: tags[i], child: photo), ), ); } @@ -255,18 +318,18 @@ class _MessageImageStack extends HookConsumerWidget { key: const ValueKey('message-media-stack-semantics'), button: true, label: items[index].semanticLabel, - hint: 'Open photo gallery', + hint: hasVideos ? 'Open media gallery' : 'Open photo gallery', customSemanticsActions: _stackTuningEnabled && showTuningControl ? { CustomSemanticsAction(label: 'Tune stack'): () => _openStackTuning(context, items, mirrored), } : null, - value: 'Photo ${index + 1} of ${items.length}', + value: '$itemName ${index + 1} of ${items.length}', increasedValue: - 'Photo ${(index + 1) % items.length + 1} of ${items.length}', + '$itemName ${(index + 1) % items.length + 1} of ${items.length}', decreasedValue: - 'Photo ${(index - 1) % items.length + 1} of ${items.length}', + '$itemName ${(index - 1) % items.length + 1} of ${items.length}', onTap: open, onIncrease: () => settle(position.value.roundToDouble() + 1), onDecrease: () => settle(position.value.roundToDouble() - 1), diff --git a/mobile/lib/features/channels/message_content/mention_markdown.dart b/mobile/lib/features/channels/message_content/mention_markdown.dart new file mode 100644 index 0000000000..f8999e756b --- /dev/null +++ b/mobile/lib/features/channels/message_content/mention_markdown.dart @@ -0,0 +1,80 @@ +part of '../message_content.dart'; + +class _MentionMd extends InlineMd { + final Map> bindings; + final Map displayLabels; + final Map mentionNames; + final Map mentionLabels; + final Set agentMentionPubkeys; + final void Function(String pubkey)? onMentionTap; + late final RegExp _exp = _buildPrefixPattern( + prefix: '@', + knownNames: bindings.keys.map(_markdownMentionName), + genericTokenPattern: r'[A-Za-z0-9_][A-Za-z0-9_\u00A0-]*', + ); + + _MentionMd({ + required this.bindings, + required this.displayLabels, + required this.mentionNames, + required this.mentionLabels, + required this.agentMentionPubkeys, + this.onMentionTap, + }); + + /// Excluded from link labels: this component renders a [WidgetSpan], and a + /// placeholder nested inside the link's own placeholder does not paint on + /// iOS — an authored `[@mention](url)` renders as nothing. Link resolution + /// wins over token detection inside a label. + @override + Set get scopes => MarkdownComponent.allScopesExceptLinkLabel; + + @override + RegExp get exp => _exp; + + @override + InlineSpan span( + BuildContext context, + String text, + final GptMarkdownConfig config, + ) { + final raw = exp.firstMatch(text.trim())?.group(0); + if (raw == null) { + return TextSpan(text: text, style: config.style); + } + + final name = raw.substring(1).replaceAll('\u00A0', ' ').toLowerCase(); + final matches = bindings[name] ?? const {}; + final pubkey = matches.length == 1 ? matches.single : null; + if (bindings.containsKey(name) && matches.length != 1) { + return TextSpan(text: text, style: config.style); + } + final displayName = name.contains(RegExp(r'\([0-9a-f]{64}\)')) + ? displayLabels[name] + : mentionLabels[pubkey] ?? mentionNames[pubkey]; + + final isAgent = + pubkey != null && agentMentionPubkeys.contains(pubkey.toLowerCase()); + final fullLabel = displayName ?? raw.substring(1); + final visibleLabel = fullLabel.replaceAllMapped( + RegExp(r'\(([0-9a-f]{64})\)'), + (m) => '(${m[1]!.substring(0, 8)}…${m[1]!.substring(60)})', + ); + final pill = MessageMentionPill( + label: visibleLabel, + semanticsLabel: fullLabel, + isAgent: isAgent, + textStyle: config.style, + ); + + return WidgetSpan( + alignment: PlaceholderAlignment.baseline, + baseline: TextBaseline.alphabetic, + child: pubkey != null && onMentionTap != null + ? GestureDetector(onTap: () => onMentionTap!(pubkey), child: pill) + : pill, + ); + } +} + +String _markdownMentionName(String name) => name.replaceAll(' ', '\u00A0'); diff --git a/mobile/lib/features/channels/message_content/video_preview.dart b/mobile/lib/features/channels/message_content/video_preview.dart index 6699064381..7a0727d725 100644 --- a/mobile/lib/features/channels/message_content/video_preview.dart +++ b/mobile/lib/features/channels/message_content/video_preview.dart @@ -2,7 +2,7 @@ part of '../message_content.dart'; /// A video bubble. It shows the NIP-71 poster when the event has one; /// otherwise a static placeholder, so scrolling never starts a video decoder. -class _MessageVideoPreview extends StatelessWidget { +class _MessageVideoPreview extends HookConsumerWidget { final String url; final ImetaEntry? imeta; final VoidCallback? onReply; @@ -14,74 +14,117 @@ class _MessageVideoPreview extends StatelessWidget { }); @override - Widget build(BuildContext context) { + Widget build(BuildContext context, WidgetRef ref) { + final sourceKey = useMemoized(GlobalKey.new); + final opening = useRef(false); final aspectRatio = messageVideoAspectRatio(imeta?.aspectRatio); final posterUrl = imeta?.posterUrl; return Padding( padding: const EdgeInsets.only(top: Grid.half), child: GestureDetector( - onTap: () => openVideoViewer( - context, - videoUrl: url, - posterUrl: posterUrl, - onReply: onReply, - ), - child: _MessageMediaPreviewFrame( - previewKey: ValueKey('message-media-video-preview:$url'), - backgroundColor: Colors.black, - child: AspectRatio( - aspectRatio: aspectRatio.toDouble(), - child: Stack( - fit: StackFit.expand, - children: [ - if (posterUrl != null) - MediaImage( - url: posterUrl, - fit: BoxFit.cover, - frameBuilder: (context, child, frame, synchronous) => - frame != null || synchronous - ? child - : const MediaLoadingPlaceholder(label: 'Loading video'), - errorBuilder: (_, _, _) => const _MediaPreviewFallback( + onTap: () async { + if (opening.value) return; + opening.value = true; + NativeVideoFiles? files; + try { + if (NativePhotoZoomPreview.enabled) { + await WidgetsBinding.instance.endOfFrame; + if (!context.mounted) return; + final auth = ref.read(mediaGetAuthServiceProvider); + final client = ref.read(mediaHttpClientProvider); + files = NativeVideoFiles(auth: auth, client: client, index: 0); + final fallback = await NativePhotoZoomPreview.open( + context: context, + sourceKeys: [sourceKey], + providers: [ + MediaImageProvider( + url: posterUrl ?? url, + auth: auth, + client: client, + ), + ], + urls: [url], + labels: [imeta?.alt ?? 'Video'], + initialIndex: 0, + videoIndices: {0}, + loadVideo: (_) => files!.load(0, url), + onSelect: (_) async {}, + onReply: onReply, + ); + if (!fallback || !context.mounted) return; + } + openVideoViewer( + context, + videoUrl: url, + posterUrl: posterUrl, + onReply: onReply, + ); + } finally { + await files?.dispose(); + opening.value = false; + } + }, + child: RepaintBoundary( + key: sourceKey, + child: _MessageMediaPreviewFrame( + previewKey: ValueKey('message-media-video-preview:$url'), + backgroundColor: Colors.black, + child: AspectRatio( + aspectRatio: aspectRatio.toDouble(), + child: Stack( + fit: StackFit.expand, + children: [ + if (posterUrl != null) + MediaImage( + url: posterUrl, + fit: BoxFit.cover, + frameBuilder: (context, child, frame, synchronous) => + frame != null || synchronous + ? child + : const MediaLoadingPlaceholder( + label: 'Loading video', + ), + errorBuilder: (_, _, _) => const _MediaPreviewFallback( + icon: BuzzIcons.video, + label: 'Video preview unavailable', + ), + ) + else + const _MediaPreviewFallback( icon: BuzzIcons.video, - label: 'Video preview unavailable', + label: 'Video attachment', ), - ) - else - const _MediaPreviewFallback( - icon: BuzzIcons.video, - label: 'Video attachment', - ), - const ColoredBox(color: Color.fromRGBO(0, 0, 0, 0.28)), - Center( - child: Container( - width: 52, - height: 52, - decoration: const BoxDecoration( - color: Color.fromRGBO(0, 0, 0, 0.6), - shape: BoxShape.circle, - ), - child: const Icon( - BuzzIcons.play, - color: Colors.white, - size: 24, + const ColoredBox(color: Color.fromRGBO(0, 0, 0, 0.28)), + Center( + child: Container( + width: 52, + height: 52, + decoration: const BoxDecoration( + color: Color.fromRGBO(0, 0, 0, 0.6), + shape: BoxShape.circle, + ), + child: const Icon( + BuzzIcons.play, + color: Colors.white, + size: 24, + ), ), ), - ), - Positioned( - left: Grid.xxs, - right: Grid.xxs, - bottom: Grid.xxs, - child: Text( - 'Video', - style: context.textTheme.labelSmall?.copyWith( - color: Colors.white, - fontWeight: FontWeight.w600, + Positioned( + left: Grid.xxs, + right: Grid.xxs, + bottom: Grid.xxs, + child: Text( + 'Video', + style: context.textTheme.labelSmall?.copyWith( + color: Colors.white, + fontWeight: FontWeight.w600, + ), ), ), - ), - ], + ], + ), ), ), ), diff --git a/mobile/lib/features/channels/message_gallery.dart b/mobile/lib/features/channels/message_gallery.dart index 842a478fe3..862278309f 100644 --- a/mobile/lib/features/channels/message_gallery.dart +++ b/mobile/lib/features/channels/message_gallery.dart @@ -4,25 +4,33 @@ import 'message_media.dart'; import 'message_content/link_normalizer.dart'; import 'message_content/markdown_escapes.dart'; -/// Image metadata shared by carousel rendering and loading projection. +/// Attachment metadata shared by carousel and stack rendering. @immutable class MessageGalleryItem { final String url; final String semanticLabel; final double? aspectRatio; - /// Declared media type, used to keep potentially animated images in Flutter. + /// Declared media type, used when selecting the native gallery. final String? mimeType; + /// Whether this item opens video playback rather than an image decoder. + final bool isVideo; + + /// Optional video poster; absent posters use a static video placeholder. + final String? posterUrl; + const MessageGalleryItem({ required this.url, required this.semanticLabel, required this.aspectRatio, this.mimeType, + this.isVideo = false, + this.posterUrl, }); } -/// A trailing run of image embeds and its preceding message text. +/// A trailing run of media embeds and its preceding message text. @immutable class TrailingImageGallery { final String content; @@ -31,11 +39,12 @@ class TrailingImageGallery { const TrailingImageGallery({required this.content, required this.items}); } -/// Groups trailing image lines using the production carousel policy. +/// Groups trailing images, optionally including videos for message stacks. TrailingImageGallery? extractTrailingImageGallery( String content, - Map imetaByUrl, -) { + Map imetaByUrl, { + bool includeVideos = false, +}) { final lines = content.split('\n'); var cursor = lines.length - 1; while (cursor >= 0 && lines[cursor].trim().isEmpty) { @@ -51,7 +60,8 @@ TrailingImageGallery? extractTrailingImageGallery( final url = unshieldMarkdownEscapes(shieldMarkdownEscapes(match.group(2)!)); final imeta = imetaByUrl[url]; final mediaKind = classifyMediaUrl(url, imeta: imeta); - if (mediaKind != MessageMediaKind.image) { + if (mediaKind != MessageMediaKind.image && + !(includeVideos && mediaKind == MessageMediaKind.video)) { break; } final markdownLabel = decodeMarkdownLabelSyntax( @@ -62,9 +72,15 @@ TrailingImageGallery? extractTrailingImageGallery( url: url, semanticLabel: imeta?.alt ?? - (markdownLabel.isNotEmpty ? markdownLabel : 'Message image'), + (markdownLabel.isNotEmpty + ? markdownLabel + : mediaKind == MessageMediaKind.video + ? 'Video' + : 'Message image'), aspectRatio: imeta?.aspectRatio, mimeType: imeta?.mimeType, + isVideo: mediaKind == MessageMediaKind.video, + posterUrl: imeta?.posterUrl, ), ); cursor -= 1; diff --git a/mobile/lib/features/channels/native_photo_zoom_preview.dart b/mobile/lib/features/channels/native_photo_zoom_preview.dart index 0be132c3c7..3f3c286c8b 100644 --- a/mobile/lib/features/channels/native_photo_zoom_preview.dart +++ b/mobile/lib/features/channels/native_photo_zoom_preview.dart @@ -3,6 +3,7 @@ import 'dart:ui' as ui; import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; +import 'package:flutter/rendering.dart'; import 'package:flutter/services.dart'; import '../../shared/relay/media_image.dart'; @@ -14,9 +15,12 @@ class NativePhotoActions { final Future Function(String action, String url) perform; } -/// UIKit gallery presentation, enabled in the photo-stack preview. +/// UIKit gallery presentation with the Flutter viewer as a fallback. class NativePhotoZoomPreview { - static const enabled = bool.fromEnvironment('BUZZ_NATIVE_PHOTO_ZOOM'); + static const enabled = bool.fromEnvironment( + 'BUZZ_NATIVE_PHOTO_ZOOM', + defaultValue: true, + ); static const channel = MethodChannel('buzz/native_photo_zoom'); static bool _presenting = false; static int _session = 0; @@ -26,6 +30,8 @@ class NativePhotoZoomPreview { /// JPEG, PNG and HEIC/HEIF photos use the native gallery. Continuing frames /// are forwarded for animated PNGs; other formats keep the Flutter renderer. /// [mimeTypes], when provided, contains one declared type per [urls] entry. + /// [videoIndices] use rendered source snapshots and [loadVideo] local files; + /// their URLs never enter the image decoder. static Future open({ required BuildContext context, required List sourceKeys, @@ -34,15 +40,18 @@ class NativePhotoZoomPreview { List? mimeTypes, required List labels, ImageProvider? previewProvider, - List>? previewProviders, + List?>? previewProviders, required int initialIndex, required Future Function(int) onSelect, VoidCallback? onReply, NativePhotoActions? actions, + Set videoIndices = const {}, + Future Function(int)? loadVideo, }) async { if (!enabled || defaultTargetPlatform != TargetPlatform.iOS) return true; // Metadata wins over the extension (e.g. a GIF served at a .jpg URL). for (var i = 0; i < urls.length; i++) { + if (videoIndices.contains(i)) continue; final mime = mimeTypes?[i]?.split(';').first.trim().toLowerCase(); final path = Uri.tryParse(urls[i])?.path.toLowerCase() ?? ''; if (mime != null @@ -85,13 +94,15 @@ class NativePhotoZoomPreview { }); }); window = loads; - final bytes = await loads.load( - initialIndex, - previewProvider ?? - previewProviders?[initialIndex] ?? - providers[initialIndex], - preview: true, - ); + final bytes = videoIndices.contains(initialIndex) + ? await _snapshot(sourceKeys[initialIndex]) + : await loads.load( + initialIndex, + previewProvider ?? + previewProviders?[initialIndex] ?? + providers[initialIndex], + preview: true, + ); if (bytes == null) return true; if (!context.mounted) return false; final geometry = _geometry(sourceKeys[initialIndex]); @@ -106,12 +117,21 @@ class NativePhotoZoomPreview { return null; } switch (call.method) { + case 'video': + if (!videoIndices.contains(index) || index != loads.index) { + return null; + } + return loadVideo?.call(index); case 'preview': + if (videoIndices.contains(index)) { + return _snapshot(sourceKeys[index]); + } final provider = previewProviders?[index]; return provider == null ? null : loads.load(index, provider, preview: true); case 'image': + if (videoIndices.contains(index)) return null; return loads.load(index, providers[index]); case 'select': loads.select(index); @@ -120,7 +140,8 @@ class NativePhotoZoomPreview { return _geometry(sourceKeys[index]); case 'action': final action = args['action']; - if (actions == null || + if (videoIndices.contains(index) || + actions == null || !['save', 'share', 'copy', 'delete'].contains(action)) { return null; } @@ -134,6 +155,7 @@ class NativePhotoZoomPreview { 'bytes': bytes, 'session': session, 'labels': labels, + 'videoIndices': videoIndices.toList(), 'initialIndex': initialIndex, 'canReply': onReply != null, 'canMore': actions != null, @@ -152,6 +174,20 @@ class NativePhotoZoomPreview { } } + static Future _snapshot(GlobalKey key) async { + final boundary = key.currentContext?.findRenderObject(); + if (boundary is! RenderRepaintBoundary || boundary.debugNeedsPaint) { + return null; + } + final image = await boundary.toImage(pixelRatio: 1); + try { + final data = await image.toByteData(format: ui.ImageByteFormat.png); + return data?.buffer.asUint8List(); + } finally { + image.dispose(); + } + } + static Map? _geometry(GlobalKey key) { final box = key.currentContext?.findRenderObject(); if (box is! RenderBox || diff --git a/mobile/lib/features/channels/native_video_files.dart b/mobile/lib/features/channels/native_video_files.dart new file mode 100644 index 0000000000..977e9915c5 --- /dev/null +++ b/mobile/lib/features/channels/native_video_files.dart @@ -0,0 +1,175 @@ +import 'dart:async'; +import 'dart:io'; + +import 'package:http/http.dart' as http; +import 'package:path_provider/path_provider.dart'; + +import '../../shared/relay/media_auth.dart'; +import '../../shared/utils/bounded_fetch.dart'; + +/// Owns only the selected native gallery video's authenticated temporary file. +/// UIKit releases its player before selection changes or this session closes. +class NativeVideoFiles { + /// Creates a session with bounded downloads and injectable temporary storage. + NativeVideoFiles({ + required this.auth, + required this.client, + required this.index, + this.temporaryDirectory = getTemporaryDirectory, + this.maxBytes = 100 * 1024 * 1024, + this.stallTimeout = defaultStallTimeout, + this.maxDuration = defaultMaxFetchDuration, + }); + + final MediaGetAuthService auth; + final http.Client client; + final Future Function() temporaryDirectory; + final int maxBytes; + final Duration stallTimeout, maxDuration; + int index; + bool _disposed = false; + _VideoFileDownload? _download; + Future _cleanup = Future.value(); + + /// Cancels the previous selection and deletes its file before the next load. + Future select(int value) async { + if (value == index) return; + index = value; + final previous = _download; + _download = null; + if (previous != null) _cleanup = previous.dispose(); + await _cleanup; + } + + /// Downloads the selected video; null exposes Retry in the native viewer. + Future load(int value, String url) async { + await _cleanup; + if (_disposed || value != index) return null; + final previous = _download; + if (previous != null && previous.finished) { + _download = null; + _cleanup = previous.dispose(); + await _cleanup; + if (_disposed || value != index) return null; + } + final download = _download ??= _VideoFileDownload(this, url); + final path = await download.result; + if (_disposed || value != index || _download != download) return null; + // A visible Retry starts a fresh request. + if (path == null) { + _download = null; + } + return path; + } + + /// Cancels pending IO and deletes the session file after UIKit releases it. + Future dispose() async { + _disposed = true; + final previous = _download; + _download = null; + await previous?.dispose(); + await _cleanup; + } +} + +class _VideoFileDownload { + _VideoFileDownload(this.owner, this.url) { + result = _fetch().whenComplete(() => finished = true); + } + final NativeVideoFiles owner; + final String url; + final stop = Completer(); + late final Future result; + Directory? directory; + bool disposed = false; + bool finished = false; + + Future _fetch() async { + Timer? stall; + void abort() { + if (!stop.isCompleted) stop.complete(); + } + + void restartStall() { + stall?.cancel(); + stall = Timer(owner.stallTimeout, abort); + } + + final deadline = Timer(owner.maxDuration, abort); + StreamIterator>? body; + RandomAccessFile? output; + var success = false; + restartStall(); + try { + final uri = Uri.parse(url); + final request = http.AbortableRequest( + 'GET', + uri, + abortTrigger: stop.future, + )..headers.addAll(owner.auth.headersFor(url)); + final response = await owner.client.send(request); + body = StreamIterator(response.stream); + if (disposed || + stop.isCompleted || + response.statusCode != 200 || + (response.contentLength ?? 0) > owner.maxBytes) { + return null; + } + final root = await owner.temporaryDirectory(); + directory = await root.createTemp('buzz-gallery-video-'); + if (disposed || stop.isCompleted) return null; + final extension = uri.path.toLowerCase().endsWith('.mov') + ? '.mov' + : '.mp4'; + final file = File('${directory!.path}/video$extension'); + output = await file.open(mode: FileMode.write); + var size = 0; + while (await body.moveNext()) { + restartStall(); + if (disposed || stop.isCompleted) return null; + final chunk = body.current; + if (chunk.length > owner.maxBytes - size) return null; + size += chunk.length; + await output.writeFrom( + chunk, + ); // Backpressure: do not buffer the video in RAM. + } + if (disposed || stop.isCompleted || size == 0) return null; + await output.close(); + output = null; + success = true; + return file.path; + } catch (_) { + return null; // Native page exposes an accessible Retry action. + } finally { + stall?.cancel(); + deadline.cancel(); + abort(); + await body?.cancel(); + try { + await output?.close(); + } on FileSystemException { + /* Clean partial file below. */ + } + if (!success || disposed) await _delete(); + } + } + + Future _delete() async { + final value = directory; + directory = null; + if (value == null) return; + try { + await value.delete(recursive: true); + } on FileSystemException { + // OS temporary storage cleanup must not prevent closing the gallery. + } + } + + Future dispose() async { + disposed = true; + if (!stop.isCompleted) stop.complete(); + await result; + await _delete(); + } +} diff --git a/mobile/test/features/channels/message_stack_cases.dart b/mobile/test/features/channels/message_stack_cases.dart index a4592a778a..102fd04da1 100644 --- a/mobile/test/features/channels/message_stack_cases.dart +++ b/mobile/test/features/channels/message_stack_cases.dart @@ -1,6 +1,90 @@ part of 'message_content_test.dart'; void stackJourneyTests() { + testWidgets('mixed stacks keep video order and return from the fallback player', ( + tester, + ) async { + const clip = 'https://example.com/clip.mp4'; + const photo = 'https://example.com/photo.png'; + const poster = 'https://example.com/second-poster.png'; + final requested = []; + final client = MockClient((request) async { + requested.add(request.url.toString()); + return http.Response.bytes( + base64Decode( + 'iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAYAAABytg0kAAAAEElEQVR4nGNgaPj/H4xhDABS0gn5PEa22gAAAABJRU5ErkJggg==', + ), + 200, + ); + }); + await tester.pumpWidget( + _testable( + const MessageContent( + bubble: true, + content: + '![Clip]($clip)\n![Photo]($photo)\n![Second](https://example.com/second.mov)', + tags: [ + [ + 'imeta', + 'url https://example.com/second.mov', + 'm video/quicktime', + 'image $poster', + ], + ], + ), + disableAnimations: true, + overrides: [mediaHttpClientProvider.overrideWithValue(client)], + ), + ); + await tester.pumpAndSettle(); + expect(requested, contains(photo)); + expect(requested, contains(poster)); + expect(requested, isNot(contains(clip))); + expect(requested, isNot(contains('https://example.com/second.mov'))); + final stack = find.byKey(const ValueKey('message-media-stack')); + final spoken = find.byKey(const ValueKey('message-media-stack-semantics')); + expect(find.text('3 attachments'), findsOneWidget); + expect( + find.byKey(const ValueKey('message-media-stack-video:$clip')), + findsOneWidget, + ); + expect(tester.widget(spoken).properties.label, 'Clip'); + expect( + tester.widget(spoken).properties.value, + 'Attachment 1 of 3', + ); + await tester.tap(stack); + await tester.pumpAndSettle(); + expect( + find.byKey(const ValueKey('message-media-video-viewer')), + findsOneWidget, + ); + await tester.tap( + find.byKey(const ValueKey('message-media-video-viewer-close')), + ); + await tester.pumpAndSettle(); + expect(tester.widget(spoken).properties.label, 'Clip'); + await tester.drag(stack, const Offset(-160, 0)); + await tester.pumpAndSettle(); + expect(tester.widget(spoken).properties.label, 'Photo'); + await tester.tap(stack); + await tester.pumpAndSettle(); + expect(find.byType(MediaImageViewerPage), findsOneWidget); + expect( + tester + .widget(find.byType(MediaImageViewerPage)) + .galleryItems! + .map((item) => item.url), + [photo], + ); + Navigator.of(tester.element(find.byType(MediaImageViewerPage))).pop(); + await tester.pumpAndSettle(); + expect( + tester.widget(spoken).properties.value, + 'Attachment 2 of 3', + ); + }); + testWidgets( 'sent and received stacks browse whole cards and open the selected gallery photo', (tester) async { diff --git a/mobile/test/features/channels/native_photo_zoom_preview_test.dart b/mobile/test/features/channels/native_photo_zoom_preview_test.dart index b036f286da..74830373c1 100644 --- a/mobile/test/features/channels/native_photo_zoom_preview_test.dart +++ b/mobile/test/features/channels/native_photo_zoom_preview_test.dart @@ -1,5 +1,6 @@ import 'dart:convert'; import 'dart:async'; +import 'dart:io'; import 'dart:ui' as ui; import 'package:flutter/services.dart'; @@ -18,11 +19,15 @@ void main() { ), ); // Locally generated 2x2 HEIC, encoded with macOS sips. Exercise the real codec. - final heicProvider = MemoryImage( - base64Decode( - 'AAAAJGZ0eXBoZWljAAAAAG1pZjFNaVBybWlhZk1pSEJoZWljAAABh21ldGEAAAAAAAAAIWhkbHIAAAAAAAAAAHBpY3QAAAAAAAAAAAAAAAAAAAAAJGRpbmYAAAAcZHJlZgAAAAAAAAABAAAADHVybCAAAAABAAAADnBpdG0AAAAAAAEAAAAjaWluZgAAAAAAAQAAABVpbmZlAgAAAAABAABodmMxAAAAAOdpcHJwAAAAxmlwY28AAAATY29scm5jbHgAAgACAAaAAAAADGNsbGkAywBAAAAAFGlzcGUAAAAAAAAAAgAAAAIAAAAJaXJvdAAAAAAQcGl4aQAAAAADCAgIAAAAcmh2Y0MBA3AAAACwAAAAAAAe8AD8/fj4AAALA6AAAQAXQAEMAf//A3AAAAMAsAAAAwAAAwAecCShAAEAJEIBAQNwAAADALAAAAMAAAMAHqAUIEHAoQQYh7kWVTcCAgYAgKIAAQAJRAHAYXLIRFNkAAAAGWlwbWEAAAAAAAAAAQABBoECAwWGhAAAAB5pbG9jAAAAAEQAAAEAAQAAAAEAAAG7AAAAPwAAAAFtZGF0AAAAAAAAAE8AAAA7KAGvovpGgXz/+yvev+23//+zsP1X/1Uf//NsCf9xTPdMsmf6IPnCObL7f8z6O/GSQi9hX4K3CEmyK4A=', - ), - ); + // Linux has no platform HEIC decoder; keep format routing covered there with + // PNG pixels, while macOS exercises Apple's decoder with the real fixture. + final heicProvider = Platform.isMacOS + ? MemoryImage( + base64Decode( + 'AAAAJGZ0eXBoZWljAAAAAG1pZjFNaVBybWlhZk1pSEJoZWljAAABh21ldGEAAAAAAAAAIWhkbHIAAAAAAAAAAHBpY3QAAAAAAAAAAAAAAAAAAAAAJGRpbmYAAAAcZHJlZgAAAAAAAAABAAAADHVybCAAAAABAAAADnBpdG0AAAAAAAEAAAAjaWluZgAAAAAAAQAAABVpbmZlAgAAAAABAABodmMxAAAAAOdpcHJwAAAAxmlwY28AAAATY29scm5jbHgAAgACAAaAAAAADGNsbGkAywBAAAAAFGlzcGUAAAAAAAAAAgAAAAIAAAAJaXJvdAAAAAAQcGl4aQAAAAADCAgIAAAAcmh2Y0MBA3AAAACwAAAAAAAe8AD8/fj4AAALA6AAAQAXQAEMAf//A3AAAAMAsAAAAwAAAwAecCShAAEAJEIBAQNwAAADALAAAAMAAAMAHqAUIEHAoQQYh7kWVTcCAgYAgKIAAQAJRAHAYXLIRFNkAAAAGWlwbWEAAAAAAAAAAQABBoECAwWGhAAAAB5pbG9jAAAAAEQAAAEAAQAAAAEAAAG7AAAAPwAAAAFtZGF0AAAAAAAAAE8AAAA7KAGvovpGgXz/+yvev+23//+zsP1X/1Uf//NsCf9xTPdMsmf6IPnCObL7f8z6O/GSQi9hX4K3CEmyK4A=', + ), + ) + : provider; tearDown(() { debugDefaultTargetPlatformOverride = null; messenger.setMockMethodCallHandler(NativePhotoZoomPreview.channel, null); @@ -37,7 +42,13 @@ void main() { ('photo', 'image/heic'), ('photo', 'image/heif; charset=binary'), ]) { - for (final outcome in ['closed', 'reply', 'unavailable']) { + for (final outcome in [ + 'closed', + 'reply', + 'unavailable', + 'bridge-error', + null, + ]) { testWidgets( 'native photo preview handles ${source.$1} ${source.$2} $outcome and selected source bounds', (tester) async { @@ -62,6 +73,9 @@ void main() { ) async { if (call.method == 'isSupported') return true; presented = call.arguments as Map; + if (outcome == 'bridge-error') { + throw PlatformException(code: 'presentation_failed'); + } return outcome; }); final result = await tester.runAsync( @@ -88,7 +102,7 @@ void main() { ), ); debugDefaultTargetPlatformOverride = null; - expect(result, outcome == 'unavailable'); + expect(result, !['closed', 'reply'].contains(outcome)); expect(presented?['labels'], ['Selected photo']); expect(presented?.containsKey('previews'), isFalse); expect(presented?['width'], 200); @@ -469,6 +483,98 @@ void main() { debugDefaultTargetPlatformOverride = null; }, skip: !NativePhotoZoomPreview.enabled); + testWidgets( + 'video opens from its rendered source and only loads the selected video', + (tester) async { + debugDefaultTargetPlatformOverride = TargetPlatform.iOS; + final key = GlobalKey(); + late BuildContext context; + await tester.pumpWidget( + MaterialApp( + home: Builder( + builder: (value) { + context = value; + return Center( + child: RepaintBoundary( + key: key, + child: const SizedBox( + width: 100, + height: 100, + child: ColoredBox(color: Colors.black), + ), + ), + ); + }, + ), + ), + ); + final image = _PendingPreview(); + final videos = []; + const codec = StandardMethodCodec(); + Future nativeCall(String method, int session, int index) async { + final reply = Completer(); + await messenger.handlePlatformMessage( + NativePhotoZoomPreview.channel.name, + codec.encodeMethodCall( + MethodCall(method, {'session': session, 'index': index}), + ), + (data) => reply.complete(codec.decodeEnvelope(data!)), + ); + return reply.future; + } + + messenger.setMockMethodCallHandler(NativePhotoZoomPreview.channel, ( + call, + ) async { + if (call.method == 'isSupported') return true; + final args = call.arguments as Map; + expect(args['videoIndices'], [0, 2]); + expect(args['bytes'], isA()); + final session = args['session'] as int; + expect(await nativeCall('image', session, 0), isNull); + expect(await nativeCall('video', session, 1), isNull); + expect(await nativeCall('video', session, 2), isNull); + expect(await nativeCall('video', session - 1, 0), isNull); + expect(await nativeCall('video', session, 0), '/temporary/clip.mp4'); + await nativeCall('select', session, 2); + expect(await nativeCall('video', session, 0), isNull); + expect(await nativeCall('video', session, 2), '/temporary/clip.mp4'); + return 'closed'; + }); + await tester.runAsync(() async { + expect( + await NativePhotoZoomPreview.open( + context: context, + sourceKeys: [key, key, key], + providers: [image, provider, image], + urls: [ + 'https://example.org/clip.mp4', + 'https://example.org/photo.jpg', + 'https://example.org/second.mov', + ], + labels: ['Clip', 'Photo', 'Second'], + initialIndex: 0, + videoIndices: {0, 2}, + loadVideo: (index) async { + videos.add(index); + return '/temporary/clip.mp4'; + }, + onSelect: (_) async {}, + ), + isFalse, + ); + }); + expect(videos, [0, 2]); + expect( + image.started, + isFalse, + reason: 'A video URL must never enter the image decoder', + ); + debugDefaultTargetPlatformOverride = null; + }, + skip: !NativePhotoZoomPreview.enabled, + ); + testWidgets( 'unsupported platform preserves existing gallery without decoding', (tester) async { diff --git a/mobile/test/features/channels/native_video_files_test.dart b/mobile/test/features/channels/native_video_files_test.dart new file mode 100644 index 0000000000..bcd2e9baae --- /dev/null +++ b/mobile/test/features/channels/native_video_files_test.dart @@ -0,0 +1,78 @@ +import 'dart:async'; +import 'dart:io'; + +import 'package:buzz/features/channels/native_video_files.dart'; +import 'package:buzz/shared/relay/media_auth.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:http/http.dart' as http; +import 'package:nostr/nostr.dart' as nostr; + +void main() { + test( + 'video downloads authenticate, retry, cancel and release selected files', + () async { + final root = await Directory.systemTemp.createTemp('gallery-video-test-'); + final server = await HttpServer.bind(InternetAddress.loopbackIPv4, 0); + final client = http.Client(); + final base = 'http://127.0.0.1:${server.port}'; + final auth = MediaGetAuthService( + baseUrl: base, + nsec: nostr.Keys.generate().nsec, + ); + final files = NativeVideoFiles( + auth: auth, + client: client, + index: 0, + temporaryDirectory: () async => root, + maxBytes: 8, + ); + addTearDown(() async { + await files.dispose(); + client.close(); + await server.close(force: true); + await root.delete(recursive: true); + }); + final received = []; + var fail = true; + final stalled = Completer(); + server.listen((request) async { + received.add(request.uri.path); + expect(request.headers.value('authorization'), startsWith('Nostr ')); + if (request.uri.path.endsWith('/stall.mp4')) { + request.response.add([1]); + await request.response.flush(); + stalled.complete(); + return; + } + if (request.uri.path.endsWith('/retry.mp4') && fail) { + request.response.statusCode = 503; + } else if (request.uri.path.endsWith('/large.mp4')) { + request.response.add(List.filled(9, 1)); + } else { + request.response.add([1, 2, 3, 4]); + } + await request.response.close(); + }); + expect(await files.load(1, '$base/media/hidden.mp4'), isNull); + expect(received, isEmpty); + expect(await files.load(0, '$base/media/retry.mp4'), isNull); + fail = false; + final first = await files.load(0, '$base/media/retry.mp4'); + expect(await File(first!).readAsBytes(), [1, 2, 3, 4]); + await files.select(1); + expect(await File(first).exists(), isFalse); + expect(await files.load(1, '$base/media/large.mp4'), isNull); + expect(await root.list().toList(), isEmpty); + final pending = files.load(1, '$base/media/stall.mp4'); + await stalled.future.timeout(const Duration(seconds: 5)); + await files.select(2).timeout(const Duration(seconds: 5)); + expect(await pending, isNull); + final last = await files.load(2, '$base/media/last.mov'); + expect(last, endsWith('.mov')); + await files.dispose(); + expect(await File(last!).exists(), isFalse); + expect(await root.list().toList(), isEmpty); + expect(await files.load(2, '$base/media/after.mp4'), isNull); + }, + ); +} diff --git a/scripts/test-mobile-photo-zoom.sh b/scripts/test-mobile-photo-zoom.sh index 36a910cfcb..f5d39a618c 100644 --- a/scripts/test-mobile-photo-zoom.sh +++ b/scripts/test-mobile-photo-zoom.sh @@ -16,4 +16,4 @@ xcrun --sdk iphonesimulator swiftc -parse-as-library \ -F "$frameworks" -framework Flutter \ -Xlinker -rpath -Xlinker "$frameworks" \ "$fixture/test.swift" -o "$fixture/photo-zoom-tests" -xcrun simctl spawn "$device" "$fixture/photo-zoom-tests" +xcrun simctl spawn "$device" "$fixture/photo-zoom-tests" "$repo_root/mobile/ios/NativeControlTests/Fixtures/gallery-video.mp4"