jerry/buzz
1
0
Fork 0
mirror of https://github.com/block/buzz.git synced 2026-11-06 09:21:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(mobile): back off push bootstrap retries (#9092)

Browse Source 
When push setup fails, the app retried every 5 seconds forever. On an
iPhone 12 mini whose push gateway enrollment is refused (a CDN 403 on
the installation challenge), each retry refetched the relay's NIP-11
document over a new TLS connection and logged the whole HTML refusal.
That is about 720 attempts and roughly 14 MB of traffic an hour for as
long as the app stays open, with no way to succeed.

Push registration, lease publication and tombstone retries now back off:
each consecutive failure of the same attempt doubles the delay from 5
seconds up to 15 minutes, with jitter between half and all of it so
devices that failed together don't retry together. A success (including
a renewal), a different attempt, or a channel-grammar retry (which keeps
its 5-minute delay) ends the streak. The first retry still comes within
5 seconds, so a brief blip recovers as quickly as before.

### Validation

- Added tests: `repeated failures back off exponentially up to a cap`
checks every delay's bounds at both jitter extremes over 12 failures, `a
success ends the failure streak`, and `a different attempt starts its
own backoff`. The first fails when the gate goes back to a fixed delay.
- Existing push suite passes.
- On the 12 mini before this change: `Push lease bootstrap failed` with
the 403 page and a fresh NIP-11 fetch every 5 seconds for the whole
session.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Signed-off-by: Tom Brow <tomb@squareup.com>
Co-authored-by: Tom Brow <tomb@squareup.com>
Co-authored-by: studio <dc2fc481343672df31a689840c09299972e704dac592e0da6f18486646a7ee3e@buzz.block.builderlab.xyz>
This commit is contained in:
Tom Brow
2026-11-05 11:15:22 -08:00
committed by GitHub
parent e34acb0ac5
commit 3ff4154ca6
2 changed files with 224 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

105
mobile/lib/shared/push/push_bootstrap.dart
View File

@@ -1,4 +1,5 @@
import 'dart:async';
import 'dart:math';
import 'package:flutter/material.dart';
import 'package:flutter_hooks/flutter_hooks.dart';
@@ -18,14 +19,27 @@ import 'push_subscription_provider.dart';
import 'push_setup_recovery.dart';
const _pushBootstrapRetryDelay = Duration(seconds: 5);
const _pushBootstrapMaxRetryDelay = Duration(minutes: 15);
@visibleForTesting
class BuzzPushAttemptGate {
BuzzPushAttemptGate({this.retryDelay = _pushBootstrapRetryDelay});
BuzzPushAttemptGate({
this.retryDelay = _pushBootstrapRetryDelay,
this.maxRetryDelay = _pushBootstrapMaxRetryDelay,
double Function()? jitter,
}) : _jitter = jitter ?? Random().nextDouble;
/// Delay before the first retry of a failing attempt. Each further
/// consecutive failure of the same attempt doubles it, up to
/// [maxRetryDelay], so a persistent refusal (for example a gateway that
/// blocks this device) is not retried every few seconds forever.
final Duration retryDelay;
final Duration maxRetryDelay;
final double Function() _jitter;
String? _attempt;
Timer? _retryTimer;
String? _failingAttempt;
int _consecutiveFailures = 0;
bool tryBegin(String attempt) {
if (_attempt == attempt) return false;
@@ -37,21 +51,27 @@ class BuzzPushAttemptGate {
void failed(String attempt, {required VoidCallback retry}) {
if (_attempt != attempt) return;
_retryTimer?.cancel();
_retryTimer = Timer(retryDelay, () {
_retryTimer = null;
if (_attempt != attempt) return;
_attempt = null;
retry();
});
if (_failingAttempt != attempt) {
_failingAttempt = attempt;
_consecutiveFailures = 0;
}
_schedule(attempt, _backoff(_consecutiveFailures++), retry);
}
void retryAfter(
String attempt, {
required Duration delay,
required VoidCallback retry,
}) {
if (_attempt != attempt) return;
/// Between half and all of the capped exponential delay, so devices that
/// failed together do not retry together.
Duration _backoff(int priorFailures) {
final ceiling = min(
maxRetryDelay.inMilliseconds,
retryDelay.inMilliseconds * (1 << min(priorFailures, 20)),
);
final half = ceiling ~/ 2;
return Duration(
milliseconds: half + (_jitter() * (ceiling - half)).floor(),
);
}
void _schedule(String attempt, Duration delay, VoidCallback retry) {
_retryTimer?.cancel();
_retryTimer = Timer(delay, () {
_retryTimer = null;
@@ -61,11 +81,35 @@ class BuzzPushAttemptGate {
});
}
/// Schedules a retry after a success (for example a renewal) or after a
/// failure that picks its own delay. Ends any failure streak.
void retryAfter(
String attempt, {
required Duration delay,
required VoidCallback retry,
}) {
if (_attempt != attempt) return;
_endFailureStreak();
_schedule(attempt, delay, retry);
}
void complete(String attempt) {
if (_attempt != attempt) return;
_retryTimer?.cancel();
_retryTimer = null;
_attempt = null;
_endFailureStreak();
}
/// Records that [attempt] succeeded without making it runnable again, so a
/// later failure starts from the first retry delay.
void succeeded(String attempt) {
if (_attempt == attempt) _endFailureStreak();
}
void _endFailureStreak() {
_failingAttempt = null;
_consecutiveFailures = 0;
}
void dispose() => _retryTimer?.cancel();
@@ -88,6 +132,14 @@ String buzzPushPublicationAttemptKey({
buzzPushSubscriptionsFingerprint(subscriptions),
].join('|');
/// Identifies a pass over pending opt-out tombstones by the communities it
/// covers, so a community whose tombstone appears while another is backing
/// off runs now instead of waiting out that delay. Not by generation: each
/// pass advances it, and that must not restart the backoff.
@visibleForTesting
String buzzPushTombstoneAttemptKey(Iterable<Community> pending) =>
[for (final community in pending) community.id].join('|');
@visibleForTesting
bool buzzPushLifecycleEnabled({
required Community? community,
@@ -123,6 +175,7 @@ class BuzzPushRegistrationBootstrap extends HookWidget {
unawaited(() async {
try {
await startRegistration();
attemptGate.succeeded(attemptKey);
} catch (error, stack) {
attemptGate.failed(
attemptKey,
@@ -331,7 +384,7 @@ class _BuzzPushLifecycle extends HookConsumerWidget {
pendingCommunities.isEmpty) {
return null;
}
const attempt = 'pending-tombstones';
final attempt = buzzPushTombstoneAttemptKey(pendingCommunities);
if (!tombstoneAttempt.tryBegin(attempt)) return null;
unawaited(() async {
try {
@@ -429,17 +482,21 @@ class _BuzzPushLifecycle extends HookConsumerWidget {
},
);
} catch (error, stack) {
publicationAttempt.retryAfter(
attempt,
void retry() {
if (context.mounted) publicationRetry.value += 1;
}
if (error is BuzzPushChannelGrammarException) {
// Re-fetch capabilities after a relay upgrade without repeatedly
// enrolling or publishing a lease the old relay cannot accept.
delay: error is BuzzPushChannelGrammarException
? const Duration(minutes: 5)
: _pushBootstrapRetryDelay,
retry: () {
if (context.mounted) publicationRetry.value += 1;
},
);
publicationAttempt.retryAfter(
attempt,
delay: const Duration(minutes: 5),
retry: retry,
);
} else {
publicationAttempt.failed(attempt, retry: retry);
}
debugPrint('Push lease bootstrap failed: $error');
debugPrintStack(stackTrace: stack);
}

144
mobile/test/shared/push/push_bootstrap_test.dart
View File

@@ -1,3 +1,5 @@
import 'dart:math';
import 'package:buzz/shared/push/dev_push_lease.dart';
import 'package:buzz/shared/community/community.dart';
import 'package:buzz/shared/push/push_bootstrap.dart';
@@ -83,7 +85,7 @@ void main() {
testWidgets('a failed attempt stays blocked until the retry delay', (
tester,
) async {
final gate = BuzzPushAttemptGate();
final gate = BuzzPushAttemptGate(jitter: () => 1);
addTearDown(gate.dispose);
var retries = 0;
expect(gate.tryBegin('attempt'), isTrue);
@@ -97,6 +99,146 @@ void main() {
expect(gate.tryBegin('attempt'), isTrue);
});
/// Fails [attempt] once more and returns how long its retry took to fire.
Future<Duration> failAndTimeRetry(
WidgetTester tester,
BuzzPushAttemptGate gate,
String attempt,
) async {
expect(gate.tryBegin(attempt), isTrue);
var fired = false;
gate.failed(attempt, retry: () => fired = true);
var waited = Duration.zero;
while (!fired) {
await tester.pump(const Duration(milliseconds: 500));
waited += const Duration(milliseconds: 500);
}
return waited;
}
testWidgets('repeated failures back off exponentially up to a cap', (
tester,
) async {
for (final jitter in [0.0, 1.0]) {
final gate = BuzzPushAttemptGate(jitter: () => jitter);
final delays = [
for (var i = 0; i < 12; i++)
await failAndTimeRetry(tester, gate, 'attempt'),
];
gate.dispose();
// Each delay lies between half and all of 5 s doubled per prior
// failure, capped at 15 minutes.
for (final (index, delay) in delays.indexed) {
final ceiling = min(5000 * (1 << index), 15 * 60 * 1000);
expect(
delay.inMilliseconds,
inInclusiveRange(ceiling ~/ 2, ceiling),
reason: 'failure $index with jitter $jitter',
);
}
expect(delays.last, lessThanOrEqualTo(const Duration(minutes: 15)));
expect(delays.last, greaterThan(const Duration(minutes: 7)));
}
});
testWidgets('a success ends the failure streak', (tester) async {
final gate = BuzzPushAttemptGate(jitter: () => 1);
addTearDown(gate.dispose);
for (var i = 0; i < 4; i++) {
await failAndTimeRetry(tester, gate, 'attempt');
}
expect(gate.tryBegin('attempt'), isTrue);
gate.retryAfter('attempt', delay: Duration.zero, retry: () {});
await tester.pump(const Duration(milliseconds: 1));
expect(
await failAndTimeRetry(tester, gate, 'attempt'),
const Duration(seconds: 5),
);
});
testWidgets('a registration success ends its failure streak', (tester) async {
var failing = true;
var registrations = 0;
Widget bootstrap(String key) => BuzzPushRegistrationBootstrap(
shouldRegister: true,
attemptKey: key,
startRegistration: () async {
registrations += 1;
if (failing) throw StateError('refused');
},
child: const SizedBox(),
);
// Community A fails long enough to back off past a minute.
await tester.pumpWidget(bootstrap('a'));
for (var i = 0; i < 5; i++) {
await tester.pump(const Duration(minutes: 3));
}
expect(registrations, 6);
failing = false;
await tester.pump(const Duration(minutes: 15));
expect(registrations, 7);
// Switch to B and back. A's next failure retries within the first delay.
await tester.pumpWidget(bootstrap('b'));
expect(registrations, 8);
failing = true;
await tester.pumpWidget(bootstrap('a'));
expect(registrations, 9);
await tester.pump(const Duration(seconds: 5));
await tester.pump();
expect(registrations, 10);
await tester.pumpWidget(const SizedBox());
});
test('a new pending tombstone changes the tombstone attempt', () {
Community pending(String name, int generation) =>
Community.create(name: name, relayUrl: 'wss://$name.example').copyWith(
pushNotificationsEnabled: false,
pushSubscriptionState: const BuzzPushLeaseSubscriptionState.desired(
desired: [],
).withPendingTombstone(generation),
);
final a = pending('a', 4);
final b = pending('b', 2);
// A failing pass keeps its key, and so its backoff, until the work
// changes; new work gets a new key and runs at once.
expect(buzzPushTombstoneAttemptKey([a]), buzzPushTombstoneAttemptKey([a]));
expect(
buzzPushTombstoneAttemptKey([a, b]),
isNot(buzzPushTombstoneAttemptKey([a])),
);
// Each pass advances the generation; that keeps the backoff.
expect(
buzzPushTombstoneAttemptKey([
a.copyWith(
pushSubscriptionState: a.pushSubscriptionState.withPendingTombstone(
5,
),
),
]),
buzzPushTombstoneAttemptKey([a]),
);
});
testWidgets('a different attempt starts its own backoff', (tester) async {
final gate = BuzzPushAttemptGate(jitter: () => 1);
addTearDown(gate.dispose);
for (var i = 0; i < 4; i++) {
await failAndTimeRetry(tester, gate, 'old');
}
expect(
await failAndTimeRetry(tester, gate, 'new'),
const Duration(seconds: 5),
);
});
test('a new attempt cancels an obsolete scheduled retry', () async {
final gate = BuzzPushAttemptGate(retryDelay: Duration.zero);
addTearDown(gate.dispose);