1use std::{
16 num::NonZeroUsize,
17 sync::{
18 Arc, OnceLock,
19 atomic::{AtomicBool, AtomicUsize, Ordering},
20 },
21 time::Duration,
22};
23
24use bitwarden_ipc::{
25 Endpoint, IpcClient, IpcClientExt, IpcClientTypedSubscription, PayloadTypeName, RequestError,
26 RpcHandler, RpcRequest, RpcRequestInfo, Source, SubscribeError, TypedReceiveError,
27 discover::wait_for_peer,
28};
29use bitwarden_threading::{
30 cancellation_token::CancellationToken,
31 spawn,
32 time::{sleep, timeout},
33};
34use serde::{Deserialize, Serialize};
35
36use crate::{
37 CircularBuffer, FlightRecorderEvent, FlightRecorderSink, get_flight_recorder_buffer,
38 read_flight_recorder, set_flight_recorder_sink, without_flight_recorder,
39};
40
41const UNFORWARDED_TARGETS: &[&str] = &["bitwarden_ipc", "bitwarden_threading"];
44
45const FLUSH_INTERVAL: Duration = Duration::from_millis(250);
47
48const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(5);
50
51const COLLECTOR_WAIT: Duration = Duration::from_secs(30);
53
54const DISCOVER_RETRY_DELAY: Duration = Duration::from_millis(250);
56
57#[derive(Debug, Clone, Serialize, Deserialize)]
59pub struct FlightRecorderBatch {
60 pub events: Vec<FlightRecorderEvent>,
62}
63
64impl PayloadTypeName for FlightRecorderBatch {
65 const PAYLOAD_TYPE_NAME: &'static str = "platform.flight-recorder.batch";
66}
67
68#[derive(Debug, Clone, Serialize, Deserialize)]
70pub struct ReadFlightRecorderRequest;
71
72#[derive(Debug, Clone, Serialize, Deserialize)]
74pub struct ReadFlightRecorderResponse {
75 pub events: Vec<FlightRecorderEvent>,
77}
78
79#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
81pub enum ReadFlightRecorderError {
82 SourceNotAccepted,
84}
85
86impl RpcRequest for ReadFlightRecorderRequest {
87 type Response = Result<ReadFlightRecorderResponse, ReadFlightRecorderError>;
88
89 const NAME: &str = "ReadFlightRecorder";
90}
91
92#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
94#[bitwarden_ffi::wasm_record]
95pub enum FlightRecorderCollectorRole {
96 BrowserBackground,
98 DesktopMain,
100}
101
102impl FlightRecorderCollectorRole {
103 fn label(self) -> &'static str {
105 match self {
106 Self::BrowserBackground => "browser-background",
107 Self::DesktopMain => "desktop-main",
108 }
109 }
110
111 fn accepts(self, source: &Source) -> bool {
113 match self {
114 Self::BrowserBackground => matches!(source, Source::BrowserForeground { .. }),
115 Self::DesktopMain => matches!(source, Source::DesktopRenderer),
116 }
117 }
118}
119
120fn process_label(source: &Source) -> &'static str {
122 match source {
123 Source::Web { .. } => "web",
124 Source::BrowserForeground { .. } => "browser-popup",
125 Source::BrowserBackground { .. } => "browser-background",
126 Source::Cli { .. } => "cli",
127 Source::DesktopRenderer => "desktop-renderer",
128 Source::DesktopMain => "desktop-main",
129 }
130}
131
132struct ForwarderState {
133 client: Arc<dyn IpcClient>,
134 destination: Endpoint,
135 local: Arc<CircularBuffer<FlightRecorderEvent>>,
136 pending: CircularBuffer<FlightRecorderEvent>,
137 flushing: AtomicBool,
138 reading: AtomicUsize,
140}
141
142#[derive(Clone)]
151pub struct IpcForwardSink(Arc<ForwarderState>);
152
153impl IpcForwardSink {
154 pub fn new(
157 client: Arc<dyn IpcClient>,
158 destination: Endpoint,
159 capacity: NonZeroUsize,
160 local: Arc<CircularBuffer<FlightRecorderEvent>>,
161 ) -> Self {
162 Self(Arc::new(ForwarderState {
163 client,
164 destination,
165 local,
166 pending: CircularBuffer::new(capacity),
167 flushing: AtomicBool::new(false),
168 reading: AtomicUsize::new(0),
169 }))
170 }
171
172 fn enqueue(&self, event: FlightRecorderEvent) {
173 self.0.pending.push(event);
174 }
175
176 fn schedule_flush(&self) {
177 if !self.0.flushing.swap(true, Ordering::AcqRel) {
178 spawn(without_flight_recorder(flush(Arc::clone(&self.0))));
179 }
180 }
181
182 async fn read_aggregated(
185 &self,
186 cancellation_token: Option<CancellationToken>,
187 ) -> Vec<FlightRecorderEvent> {
188 let state = &self.0;
189 let _reading = ReadingGuard::new(&state.reading);
190 let request = without_flight_recorder(state.client.request(
192 ReadFlightRecorderRequest,
193 state.destination.clone(),
194 cancellation_token.clone(),
195 ));
196 let response = match cancellation_token {
197 Some(_) => request.await.ok(),
198 None => timeout(DEFAULT_READ_TIMEOUT, request)
199 .await
200 .ok()
201 .and_then(Result::ok),
202 };
203
204 let mut events = response
205 .and_then(Result::ok)
206 .map(|response| response.events)
207 .unwrap_or_default();
208 events.extend(state.local.read());
209 events.extend(state.pending.read());
210 events
211 }
212}
213
214impl FlightRecorderSink for IpcForwardSink {
215 fn push(&self, event: FlightRecorderEvent) {
216 if UNFORWARDED_TARGETS
217 .iter()
218 .any(|prefix| event.target.starts_with(prefix))
219 {
220 self.0.local.push(event);
221 return;
222 }
223 self.enqueue(event);
224 self.schedule_flush();
225 }
226}
227
228async fn flush(state: Arc<ForwarderState>) {
229 loop {
230 sleep(FLUSH_INTERVAL).await;
231 if state.reading.load(Ordering::Acquire) > 0 {
232 continue;
233 }
234 let events = state.pending.take_all();
235 if events.is_empty() {
236 if release_flush(&state) {
237 return;
238 }
239 continue;
240 }
241
242 let _ = state
244 .client
245 .send_typed(FlightRecorderBatch { events }, state.destination.clone())
246 .await;
247 }
248}
249
250struct ReadingGuard<'a>(&'a AtomicUsize);
252
253impl<'a> ReadingGuard<'a> {
254 fn new(reading: &'a AtomicUsize) -> Self {
255 reading.fetch_add(1, Ordering::AcqRel);
256 Self(reading)
257 }
258}
259
260impl Drop for ReadingGuard<'_> {
261 fn drop(&mut self) {
262 self.0.fetch_sub(1, Ordering::AcqRel);
263 }
264}
265
266fn release_flush(state: &ForwarderState) -> bool {
268 state.flushing.store(false, Ordering::Release);
269 state.pending.is_empty() || state.flushing.swap(true, Ordering::AcqRel)
272}
273
274static FORWARDER: OnceLock<IpcForwardSink> = OnceLock::new();
275
276pub fn enable_flight_recorder_forwarding(client: Arc<dyn IpcClient>, destination: Endpoint) {
284 let Some(buffer) = get_flight_recorder_buffer() else {
285 return;
286 };
287 let capacity = NonZeroUsize::new(buffer.capacity()).expect("buffer capacity is non-zero");
288 let sink = IpcForwardSink::new(client, destination, capacity, Arc::clone(&buffer));
289 if FORWARDER.set(sink.clone()).is_err() {
290 return;
291 }
292
293 spawn(async move {
294 let state = &sink.0;
295 let ready = without_flight_recorder(timeout(
296 COLLECTOR_WAIT,
297 wait_for_collector(state.client.as_ref(), state.destination.clone()),
298 ))
299 .await;
300 if !matches!(ready, Ok(true)) {
301 return;
302 }
303
304 set_flight_recorder_sink(Arc::new(sink.clone()));
306 for event in buffer.take_all() {
307 sink.enqueue(event);
308 }
309 sink.schedule_flush();
310 });
311}
312
313async fn wait_for_collector(client: &dyn IpcClient, destination: Endpoint) -> bool {
319 loop {
320 match wait_for_peer(client, destination.clone(), None).await {
321 Ok(_) => return true,
322 Err(RequestError::Rpc(_)) => sleep(DISCOVER_RETRY_DELAY).await,
323 Err(_) => return false,
324 }
325 }
326}
327
328static COLLECTING: AtomicBool = AtomicBool::new(false);
329
330pub async fn enable_flight_recorder_collection(
334 client: Arc<dyn IpcClient>,
335 role: FlightRecorderCollectorRole,
336) -> Result<(), SubscribeError> {
337 let Some(buffer) = get_flight_recorder_buffer() else {
338 return Ok(());
339 };
340 if COLLECTING.swap(true, Ordering::AcqRel) {
342 return Ok(());
343 }
344 let result = start_collector(client, buffer, role).await;
345 if result.is_err() {
346 COLLECTING.store(false, Ordering::Release);
347 }
348 result
349}
350
351async fn start_collector(
352 client: Arc<dyn IpcClient>,
353 buffer: Arc<CircularBuffer<FlightRecorderEvent>>,
354 role: FlightRecorderCollectorRole,
355) -> Result<(), SubscribeError> {
356 let batches = client.subscribe_typed::<FlightRecorderBatch>().await?;
357 client
358 .register_rpc_handler(ReadFlightRecorderHandler {
359 buffer: Arc::clone(&buffer),
360 role,
361 })
362 .await;
363 spawn(collect_batches(batches, buffer, role));
364 Ok(())
365}
366
367fn is_closed(error: &TypedReceiveError) -> bool {
368 matches!(
369 error,
370 TypedReceiveError::Channel(tokio::sync::broadcast::error::RecvError::Closed)
371 )
372}
373
374async fn collect_batches(
375 mut batches: IpcClientTypedSubscription<FlightRecorderBatch>,
376 buffer: Arc<CircularBuffer<FlightRecorderEvent>>,
377 role: FlightRecorderCollectorRole,
378) {
379 loop {
380 match batches.receive(None).await {
381 Ok(message) => {
382 if !role.accepts(&message.source) {
383 continue;
384 }
385 let process = process_label(&message.source);
387 for mut event in message.payload.events {
388 event.process = Some(process.to_owned());
389 buffer.push(event);
390 }
391 }
392 Err(error) if is_closed(&error) => return,
393 Err(_) => continue,
394 }
395 }
396}
397
398struct ReadFlightRecorderHandler {
400 buffer: Arc<CircularBuffer<FlightRecorderEvent>>,
401 role: FlightRecorderCollectorRole,
402}
403
404impl RpcHandler for ReadFlightRecorderHandler {
405 type Request = ReadFlightRecorderRequest;
406
407 async fn handle(
408 &self,
409 _request: Self::Request,
410 info: RpcRequestInfo,
411 ) -> Result<ReadFlightRecorderResponse, ReadFlightRecorderError> {
412 if !self.role.accepts(&info.source) {
413 return Err(ReadFlightRecorderError::SourceNotAccepted);
414 }
415 Ok(ReadFlightRecorderResponse {
416 events: attribute(self.buffer.read(), self.role.label()),
417 })
418 }
419}
420
421fn attribute(mut events: Vec<FlightRecorderEvent>, process: &str) -> Vec<FlightRecorderEvent> {
422 for event in &mut events {
423 if event.process.is_none() {
424 event.process = Some(process.to_owned());
425 }
426 }
427 events
428}
429
430pub async fn read_flight_recorder_aggregated(
437 cancellation_token: Option<CancellationToken>,
438) -> Vec<FlightRecorderEvent> {
439 match FORWARDER.get() {
440 Some(forwarder) => forwarder.read_aggregated(cancellation_token).await,
441 None => read_flight_recorder(),
442 }
443}
444
445#[cfg(test)]
446mod tests {
447 use std::{collections::HashMap, time::Duration};
448
449 use bitwarden_ipc::{
450 InMemorySessionRepository, IncomingMessage, IpcClientImpl, NoEncryptionCryptoProvider,
451 OutgoingMessage, TestCommunicationBackend, TypedIncomingMessage, TypedOutgoingMessage,
452 discover::{DiscoverHandler, DiscoverResponse},
453 };
454
455 use super::*;
456
457 fn event(target: &str, message: &str) -> FlightRecorderEvent {
458 FlightRecorderEvent {
459 timestamp: 0,
460 level: "INFO".to_string(),
461 target: target.to_string(),
462 message: message.to_string(),
463 fields: HashMap::new(),
464 process: None,
465 }
466 }
467
468 async fn start_client(backend: &TestCommunicationBackend) -> Arc<dyn IpcClient> {
469 let client: Arc<dyn IpcClient> = Arc::new(IpcClientImpl::new(
470 NoEncryptionCryptoProvider,
471 backend.clone(),
472 InMemorySessionRepository::<()>::default(),
473 ));
474 client.start(None).await.expect("client starts");
475 client
476 }
477
478 fn incoming<P: Serialize + PayloadTypeName>(payload: P, source: Source) -> IncomingMessage {
479 TypedIncomingMessage {
480 payload,
481 destination: Endpoint::BrowserBackground {
482 id: bitwarden_ipc::HostId::Own,
483 },
484 source,
485 }
486 .try_into()
487 .expect("serializes")
488 }
489
490 fn decode<P: serde::de::DeserializeOwned + PayloadTypeName>(
491 messages: &[OutgoingMessage],
492 ) -> Vec<P> {
493 messages
494 .iter()
495 .filter(|m| m.topic.as_deref() == Some(P::PAYLOAD_TYPE_NAME))
496 .map(|m| {
497 TypedOutgoingMessage::<P>::try_from(m.clone())
498 .expect("deserializes")
499 .payload
500 })
501 .collect()
502 }
503
504 async fn settle() {
505 tokio::time::sleep(Duration::from_millis(20)).await;
506 }
507
508 async fn wait_for_flush() {
509 tokio::time::sleep(FLUSH_INTERVAL + Duration::from_millis(50)).await;
510 }
511
512 fn popup() -> Source {
513 Source::BrowserForeground { id: 1 }
514 }
515
516 fn web_page() -> Source {
517 Source::Web {
518 tab_id: 1,
519 document_id: "doc".to_string(),
520 origin: "https://example.com".to_string(),
521 }
522 }
523
524 fn buffer() -> Arc<CircularBuffer<FlightRecorderEvent>> {
525 Arc::new(CircularBuffer::new(
526 NonZeroUsize::new(100).expect("non-zero"),
527 ))
528 }
529
530 #[tokio::test(start_paused = true)]
531 async fn test_forward_sink_batches_events_pushed_within_the_interval() {
532 let backend = TestCommunicationBackend::new();
533 let client = start_client(&backend).await;
534 let sink = IpcForwardSink::new(
535 client,
536 Endpoint::DesktopMain,
537 NonZeroUsize::new(10).expect("non-zero"),
538 buffer(),
539 );
540
541 sink.push(event("app", "one"));
542 tokio::time::sleep(FLUSH_INTERVAL / 2).await;
543 assert!(decode::<FlightRecorderBatch>(&backend.outgoing().await).is_empty());
544
545 sink.push(event("app", "two"));
546 wait_for_flush().await;
547
548 let batches = decode::<FlightRecorderBatch>(&backend.outgoing().await);
549 assert_eq!(batches.len(), 1);
550 assert_eq!(messages(&batches[0].events), ["one", "two"]);
551
552 sink.push(event("app", "three"));
554 wait_for_flush().await;
555 assert_eq!(
556 decode::<FlightRecorderBatch>(&backend.outgoing().await).len(),
557 2
558 );
559 }
560
561 #[tokio::test(start_paused = true)]
562 async fn test_forward_sink_keeps_ipc_targets_locally() {
563 let backend = TestCommunicationBackend::new();
564 let client = start_client(&backend).await;
565 let local = buffer();
566 let sink = IpcForwardSink::new(
567 client,
568 Endpoint::DesktopMain,
569 NonZeroUsize::new(10).expect("non-zero"),
570 Arc::clone(&local),
571 );
572
573 sink.push(event("bitwarden_ipc::ipc_client", "ipc"));
574 sink.push(event("bitwarden_threading", "threading"));
575 wait_for_flush().await;
576
577 assert!(decode::<FlightRecorderBatch>(&backend.outgoing().await).is_empty());
578 let kept: Vec<_> = local.read().into_iter().map(|e| e.message).collect();
579 assert_eq!(kept, ["ipc", "threading"]);
580 }
581
582 #[tokio::test(start_paused = true)]
583 async fn test_collector_stores_batches_from_its_forwarder_only() {
584 let backend = TestCommunicationBackend::new();
585 let client = start_client(&backend).await;
586 let buffer = buffer();
587 start_collector(
588 client,
589 Arc::clone(&buffer),
590 FlightRecorderCollectorRole::BrowserBackground,
591 )
592 .await
593 .expect("collector starts");
594
595 let batch = |message: &str| FlightRecorderBatch {
596 events: vec![FlightRecorderEvent {
597 process: Some("spoofed".to_string()),
598 ..event("typescript", message)
599 }],
600 };
601 backend.push_incoming(incoming(batch("from popup"), popup()));
602 backend.push_incoming(incoming(batch("from web"), web_page()));
603 settle().await;
604
605 let events = buffer.read();
606 assert_eq!(events.len(), 1);
607 assert_eq!(events[0].message, "from popup");
608 assert_eq!(events[0].process.as_deref(), Some("browser-popup"));
609 }
610
611 fn background() -> Source {
612 Source::BrowserBackground {
613 id: bitwarden_ipc::HostId::Own,
614 }
615 }
616
617 async fn relay(from: &TestCommunicationBackend, to: &TestCommunicationBackend, source: Source) {
619 for message in from.drain_outgoing().await {
620 to.push_incoming(IncomingMessage {
621 payload: message.payload,
622 destination: message.destination,
623 source: source.clone(),
624 topic: message.topic,
625 });
626 }
627 }
628
629 async fn start_test_collector(events: &[&str]) -> TestCommunicationBackend {
630 let backend = TestCommunicationBackend::new();
631 let buffer = buffer();
632 for message in events {
633 buffer.push(event("app", message));
634 }
635 start_collector(
636 start_client(&backend).await,
637 buffer,
638 FlightRecorderCollectorRole::BrowserBackground,
639 )
640 .await
641 .expect("collector starts");
642 backend
643 }
644
645 fn messages(events: &[FlightRecorderEvent]) -> Vec<&str> {
646 events.iter().map(|e| e.message.as_str()).collect()
647 }
648
649 #[tokio::test(start_paused = true)]
650 async fn test_collector_answers_reads_from_its_forwarder_only() {
651 let collector = start_test_collector(&["own event"]).await;
652 let backend = TestCommunicationBackend::new();
653 let client = start_client(&backend).await;
654
655 let read_as = async |source: Source| {
656 let request = tokio::spawn({
657 let client = Arc::clone(&client);
658 async move {
659 client
660 .request(ReadFlightRecorderRequest, background().to_endpoint(), None)
661 .await
662 }
663 });
664 settle().await;
665 relay(&backend, &collector, source).await;
666 settle().await;
667 relay(&collector, &backend, background()).await;
668 request
669 .await
670 .expect("task joins")
671 .expect("request succeeds")
672 };
673
674 let from_web = read_as(web_page()).await;
675 assert!(matches!(
676 from_web,
677 Err(ReadFlightRecorderError::SourceNotAccepted)
678 ));
679
680 let events = read_as(popup()).await.expect("popup is accepted").events;
681 assert_eq!(messages(&events), ["own event"]);
682 assert_eq!(events[0].process.as_deref(), Some("browser-background"));
683 }
684
685 #[tokio::test(start_paused = true)]
686 async fn test_forwarder_read_returns_collector_events_then_local_and_pending() {
687 let collector = start_test_collector(&["collected"]).await;
688 let backend = TestCommunicationBackend::new();
689 let local = buffer();
690 local.push(event("bitwarden_ipc", "ipc"));
691 let sink = IpcForwardSink::new(
692 start_client(&backend).await,
693 background().to_endpoint(),
694 NonZeroUsize::new(10).expect("non-zero"),
695 local,
696 );
697 sink.enqueue(event("app", "unsent"));
698
699 let read = tokio::spawn({
700 let sink = sink.clone();
701 async move { sink.read_aggregated(None).await }
702 });
703 settle().await;
704 relay(&backend, &collector, popup()).await;
705 settle().await;
706 relay(&collector, &backend, background()).await;
707
708 let events = read.await.expect("task joins");
709 assert_eq!(messages(&events), ["collected", "ipc", "unsent"]);
710 }
711
712 #[tokio::test(start_paused = true)]
713 async fn test_forwarder_read_falls_back_to_local_and_pending_without_an_answer() {
714 let backend = TestCommunicationBackend::new();
715 let local = buffer();
716 local.push(event("bitwarden_ipc", "ipc"));
717 let sink = IpcForwardSink::new(
718 start_client(&backend).await,
719 background().to_endpoint(),
720 NonZeroUsize::new(10).expect("non-zero"),
721 local,
722 );
723 sink.enqueue(event("app", "unsent"));
724
725 let cancellation_token = CancellationToken::new();
726 let read = tokio::spawn({
727 let sink = sink.clone();
728 let cancellation_token = cancellation_token.clone();
729 async move { sink.read_aggregated(Some(cancellation_token)).await }
730 });
731 settle().await;
732 cancellation_token.cancel();
733
734 let events = read.await.expect("task joins");
735 assert_eq!(messages(&events), ["ipc", "unsent"]);
736 }
737
738 #[tokio::test(start_paused = true)]
739 async fn test_forwarder_read_keeps_pending_events_a_flush_would_send_mid_read() {
740 let collector = start_test_collector(&["collected"]).await;
741 let backend = TestCommunicationBackend::new();
742 let sink = IpcForwardSink::new(
743 start_client(&backend).await,
744 background().to_endpoint(),
745 NonZeroUsize::new(10).expect("non-zero"),
746 buffer(),
747 );
748 sink.push(event("app", "just before read"));
749
750 let read = tokio::spawn({
751 let sink = sink.clone();
752 async move { sink.read_aggregated(None).await }
753 });
754 settle().await;
755 relay(&backend, &collector, popup()).await;
756 wait_for_flush().await;
758 relay(&collector, &backend, background()).await;
759
760 let events = read.await.expect("task joins");
761 assert_eq!(messages(&events), ["collected", "just before read"]);
762
763 wait_for_flush().await;
765 let batches = decode::<FlightRecorderBatch>(&backend.outgoing().await);
766 assert_eq!(batches.len(), 1);
767 assert_eq!(messages(&batches[0].events), ["just before read"]);
768 }
769
770 #[tokio::test(start_paused = true)]
771 async fn test_flush_keeps_running_for_an_event_pushed_after_its_drain() {
772 let backend = TestCommunicationBackend::new();
773 let sink = IpcForwardSink::new(
774 start_client(&backend).await,
775 Endpoint::DesktopMain,
776 NonZeroUsize::new(10).expect("non-zero"),
777 buffer(),
778 );
779 let state = &sink.0;
780
781 state.flushing.store(true, Ordering::Release);
783 sink.push(event("app", "late"));
784
785 assert!(!release_flush(state));
786 assert!(state.flushing.load(Ordering::Acquire));
787
788 flush(Arc::clone(state)).await;
790 let batches = decode::<FlightRecorderBatch>(&backend.outgoing().await);
791 assert_eq!(batches.len(), 1);
792 assert_eq!(messages(&batches[0].events), ["late"]);
793 assert!(!state.flushing.load(Ordering::Acquire));
794 }
795
796 #[tokio::test(start_paused = true)]
797 async fn test_wait_for_collector_retries_until_the_discover_handler_is_registered() {
798 let collector_backend = TestCommunicationBackend::new();
799 let collector = start_client(&collector_backend).await;
800 let backend = TestCommunicationBackend::new();
801 let client = start_client(&backend).await;
802
803 let wait = tokio::spawn({
804 let client = Arc::clone(&client);
805 async move { wait_for_collector(client.as_ref(), background().to_endpoint()).await }
806 });
807
808 settle().await;
810 relay(&backend, &collector_backend, popup()).await;
811 settle().await;
812 relay(&collector_backend, &backend, background()).await;
813 settle().await;
814 assert!(!wait.is_finished());
815
816 collector
817 .register_rpc_handler(DiscoverHandler::new(DiscoverResponse {
818 version: "test".to_string(),
819 }))
820 .await;
821 tokio::time::sleep(DISCOVER_RETRY_DELAY).await;
822 settle().await;
823 relay(&backend, &collector_backend, popup()).await;
824 settle().await;
825 relay(&collector_backend, &backend, background()).await;
826
827 assert!(wait.await.expect("task joins"));
828 }
829}