diff --git a/mobile/lib/shared/push/push_bootstrap.dart b/mobile/lib/shared/push/push_bootstrap.dart index 8cb405ce62..a461a34b56 100644 --- a/mobile/lib/shared/push/push_bootstrap.dart +++ b/mobile/lib/shared/push/push_bootstrap.dart @@ -1,4 +1,5 @@ import 'dart:async'; +import 'dart:math'; import 'package:flutter/material.dart'; import 'package:flutter_hooks/flutter_hooks.dart'; @@ -18,14 +19,27 @@ import 'push_subscription_provider.dart'; import 'push_setup_recovery.dart'; const _pushBootstrapRetryDelay = Duration(seconds: 5); +const _pushBootstrapMaxRetryDelay = Duration(minutes: 15); @visibleForTesting class BuzzPushAttemptGate { - BuzzPushAttemptGate({this.retryDelay = _pushBootstrapRetryDelay}); + BuzzPushAttemptGate({ + this.retryDelay = _pushBootstrapRetryDelay, + this.maxRetryDelay = _pushBootstrapMaxRetryDelay, + double Function()? jitter, + }) : _jitter = jitter ?? Random().nextDouble; + /// Delay before the first retry of a failing attempt. Each further + /// consecutive failure of the same attempt doubles it, up to + /// [maxRetryDelay], so a persistent refusal (for example a gateway that + /// blocks this device) is not retried every few seconds forever. final Duration retryDelay; + final Duration maxRetryDelay; + final double Function() _jitter; String? _attempt; Timer? _retryTimer; + String? _failingAttempt; + int _consecutiveFailures = 0; bool tryBegin(String attempt) { if (_attempt == attempt) return false; @@ -37,21 +51,27 @@ class BuzzPushAttemptGate { void failed(String attempt, {required VoidCallback retry}) { if (_attempt != attempt) return; - _retryTimer?.cancel(); - _retryTimer = Timer(retryDelay, () { - _retryTimer = null; - if (_attempt != attempt) return; - _attempt = null; - retry(); - }); + if (_failingAttempt != attempt) { + _failingAttempt = attempt; + _consecutiveFailures = 0; + } + _schedule(attempt, _backoff(_consecutiveFailures++), retry); } - void retryAfter( - String attempt, { - required Duration delay, - required VoidCallback retry, - }) { - if (_attempt != attempt) return; + /// Between half and all of the capped exponential delay, so devices that + /// failed together do not retry together. + Duration _backoff(int priorFailures) { + final ceiling = min( + maxRetryDelay.inMilliseconds, + retryDelay.inMilliseconds * (1 << min(priorFailures, 20)), + ); + final half = ceiling ~/ 2; + return Duration( + milliseconds: half + (_jitter() * (ceiling - half)).floor(), + ); + } + + void _schedule(String attempt, Duration delay, VoidCallback retry) { _retryTimer?.cancel(); _retryTimer = Timer(delay, () { _retryTimer = null; @@ -61,11 +81,35 @@ class BuzzPushAttemptGate { }); } + /// Schedules a retry after a success (for example a renewal) or after a + /// failure that picks its own delay. Ends any failure streak. + void retryAfter( + String attempt, { + required Duration delay, + required VoidCallback retry, + }) { + if (_attempt != attempt) return; + _endFailureStreak(); + _schedule(attempt, delay, retry); + } + void complete(String attempt) { if (_attempt != attempt) return; _retryTimer?.cancel(); _retryTimer = null; _attempt = null; + _endFailureStreak(); + } + + /// Records that [attempt] succeeded without making it runnable again, so a + /// later failure starts from the first retry delay. + void succeeded(String attempt) { + if (_attempt == attempt) _endFailureStreak(); + } + + void _endFailureStreak() { + _failingAttempt = null; + _consecutiveFailures = 0; } void dispose() => _retryTimer?.cancel(); @@ -88,6 +132,14 @@ String buzzPushPublicationAttemptKey({ buzzPushSubscriptionsFingerprint(subscriptions), ].join('|'); +/// Identifies a pass over pending opt-out tombstones by the communities it +/// covers, so a community whose tombstone appears while another is backing +/// off runs now instead of waiting out that delay. Not by generation: each +/// pass advances it, and that must not restart the backoff. +@visibleForTesting +String buzzPushTombstoneAttemptKey(Iterable pending) => + [for (final community in pending) community.id].join('|'); + @visibleForTesting bool buzzPushLifecycleEnabled({ required Community? community, @@ -123,6 +175,7 @@ class BuzzPushRegistrationBootstrap extends HookWidget { unawaited(() async { try { await startRegistration(); + attemptGate.succeeded(attemptKey); } catch (error, stack) { attemptGate.failed( attemptKey, @@ -331,7 +384,7 @@ class _BuzzPushLifecycle extends HookConsumerWidget { pendingCommunities.isEmpty) { return null; } - const attempt = 'pending-tombstones'; + final attempt = buzzPushTombstoneAttemptKey(pendingCommunities); if (!tombstoneAttempt.tryBegin(attempt)) return null; unawaited(() async { try { @@ -429,17 +482,21 @@ class _BuzzPushLifecycle extends HookConsumerWidget { }, ); } catch (error, stack) { - publicationAttempt.retryAfter( - attempt, + void retry() { + if (context.mounted) publicationRetry.value += 1; + } + + if (error is BuzzPushChannelGrammarException) { // Re-fetch capabilities after a relay upgrade without repeatedly // enrolling or publishing a lease the old relay cannot accept. - delay: error is BuzzPushChannelGrammarException - ? const Duration(minutes: 5) - : _pushBootstrapRetryDelay, - retry: () { - if (context.mounted) publicationRetry.value += 1; - }, - ); + publicationAttempt.retryAfter( + attempt, + delay: const Duration(minutes: 5), + retry: retry, + ); + } else { + publicationAttempt.failed(attempt, retry: retry); + } debugPrint('Push lease bootstrap failed: $error'); debugPrintStack(stackTrace: stack); } diff --git a/mobile/test/shared/push/push_bootstrap_test.dart b/mobile/test/shared/push/push_bootstrap_test.dart index ee3b880a72..d2b19cff11 100644 --- a/mobile/test/shared/push/push_bootstrap_test.dart +++ b/mobile/test/shared/push/push_bootstrap_test.dart @@ -1,3 +1,5 @@ +import 'dart:math'; + import 'package:buzz/shared/push/dev_push_lease.dart'; import 'package:buzz/shared/community/community.dart'; import 'package:buzz/shared/push/push_bootstrap.dart'; @@ -83,7 +85,7 @@ void main() { testWidgets('a failed attempt stays blocked until the retry delay', ( tester, ) async { - final gate = BuzzPushAttemptGate(); + final gate = BuzzPushAttemptGate(jitter: () => 1); addTearDown(gate.dispose); var retries = 0; expect(gate.tryBegin('attempt'), isTrue); @@ -97,6 +99,146 @@ void main() { expect(gate.tryBegin('attempt'), isTrue); }); + /// Fails [attempt] once more and returns how long its retry took to fire. + Future failAndTimeRetry( + WidgetTester tester, + BuzzPushAttemptGate gate, + String attempt, + ) async { + expect(gate.tryBegin(attempt), isTrue); + var fired = false; + gate.failed(attempt, retry: () => fired = true); + var waited = Duration.zero; + while (!fired) { + await tester.pump(const Duration(milliseconds: 500)); + waited += const Duration(milliseconds: 500); + } + return waited; + } + + testWidgets('repeated failures back off exponentially up to a cap', ( + tester, + ) async { + for (final jitter in [0.0, 1.0]) { + final gate = BuzzPushAttemptGate(jitter: () => jitter); + final delays = [ + for (var i = 0; i < 12; i++) + await failAndTimeRetry(tester, gate, 'attempt'), + ]; + gate.dispose(); + + // Each delay lies between half and all of 5 s doubled per prior + // failure, capped at 15 minutes. + for (final (index, delay) in delays.indexed) { + final ceiling = min(5000 * (1 << index), 15 * 60 * 1000); + expect( + delay.inMilliseconds, + inInclusiveRange(ceiling ~/ 2, ceiling), + reason: 'failure $index with jitter $jitter', + ); + } + expect(delays.last, lessThanOrEqualTo(const Duration(minutes: 15))); + expect(delays.last, greaterThan(const Duration(minutes: 7))); + } + }); + + testWidgets('a success ends the failure streak', (tester) async { + final gate = BuzzPushAttemptGate(jitter: () => 1); + addTearDown(gate.dispose); + for (var i = 0; i < 4; i++) { + await failAndTimeRetry(tester, gate, 'attempt'); + } + + expect(gate.tryBegin('attempt'), isTrue); + gate.retryAfter('attempt', delay: Duration.zero, retry: () {}); + await tester.pump(const Duration(milliseconds: 1)); + + expect( + await failAndTimeRetry(tester, gate, 'attempt'), + const Duration(seconds: 5), + ); + }); + + testWidgets('a registration success ends its failure streak', (tester) async { + var failing = true; + var registrations = 0; + Widget bootstrap(String key) => BuzzPushRegistrationBootstrap( + shouldRegister: true, + attemptKey: key, + startRegistration: () async { + registrations += 1; + if (failing) throw StateError('refused'); + }, + child: const SizedBox(), + ); + + // Community A fails long enough to back off past a minute. + await tester.pumpWidget(bootstrap('a')); + for (var i = 0; i < 5; i++) { + await tester.pump(const Duration(minutes: 3)); + } + expect(registrations, 6); + failing = false; + await tester.pump(const Duration(minutes: 15)); + expect(registrations, 7); + + // Switch to B and back. A's next failure retries within the first delay. + await tester.pumpWidget(bootstrap('b')); + expect(registrations, 8); + failing = true; + await tester.pumpWidget(bootstrap('a')); + expect(registrations, 9); + await tester.pump(const Duration(seconds: 5)); + await tester.pump(); + expect(registrations, 10); + + await tester.pumpWidget(const SizedBox()); + }); + + test('a new pending tombstone changes the tombstone attempt', () { + Community pending(String name, int generation) => + Community.create(name: name, relayUrl: 'wss://$name.example').copyWith( + pushNotificationsEnabled: false, + pushSubscriptionState: const BuzzPushLeaseSubscriptionState.desired( + desired: [], + ).withPendingTombstone(generation), + ); + final a = pending('a', 4); + final b = pending('b', 2); + + // A failing pass keeps its key, and so its backoff, until the work + // changes; new work gets a new key and runs at once. + expect(buzzPushTombstoneAttemptKey([a]), buzzPushTombstoneAttemptKey([a])); + expect( + buzzPushTombstoneAttemptKey([a, b]), + isNot(buzzPushTombstoneAttemptKey([a])), + ); + // Each pass advances the generation; that keeps the backoff. + expect( + buzzPushTombstoneAttemptKey([ + a.copyWith( + pushSubscriptionState: a.pushSubscriptionState.withPendingTombstone( + 5, + ), + ), + ]), + buzzPushTombstoneAttemptKey([a]), + ); + }); + + testWidgets('a different attempt starts its own backoff', (tester) async { + final gate = BuzzPushAttemptGate(jitter: () => 1); + addTearDown(gate.dispose); + for (var i = 0; i < 4; i++) { + await failAndTimeRetry(tester, gate, 'old'); + } + + expect( + await failAndTimeRetry(tester, gate, 'new'), + const Duration(seconds: 5), + ); + }); + test('a new attempt cancels an obsolete scheduled retry', () async { final gate = BuzzPushAttemptGate(retryDelay: Duration.zero); addTearDown(gate.dispose);