Skip to main content

bitwarden_logging/
ipc.rs

1//! Aggregation of Flight Recorder events from several contexts of one client over
2//! [`bitwarden_ipc`].
3//!
4//! Clients with more than one context (browser popup + background, desktop renderer + main) run
5//! one context as the collector and the others as forwarders. Once forwarding is enabled, a
6//! forwarder sends captured events to the collector and keeps only events from the IPC stack
7//! itself locally, since sending those would loop. The collector stores
8//! received events in its own buffer, attributed to the sending process, and answers read
9//! requests from its forwarders.
10//!
11//! Collectors only accept messages from their own client's forwarder endpoint. The browser
12//! background also receives IPC traffic from web pages, which must not be able to read or inject
13//! events.
14
15use std::{
16    num::NonZeroUsize,
17    sync::{
18        Arc, OnceLock,
19        atomic::{AtomicBool, AtomicUsize, Ordering},
20    },
21    time::Duration,
22};
23
24use bitwarden_ipc::{
25    Endpoint, IpcClient, IpcClientExt, IpcClientTypedSubscription, PayloadTypeName, RequestError,
26    RpcHandler, RpcRequest, RpcRequestInfo, Source, SubscribeError, TypedReceiveError,
27    discover::wait_for_peer,
28};
29use bitwarden_threading::{
30    cancellation_token::CancellationToken,
31    spawn,
32    time::{sleep, timeout},
33};
34use serde::{Deserialize, Serialize};
35
36use crate::{
37    CircularBuffer, FlightRecorderEvent, FlightRecorderSink, get_flight_recorder_buffer,
38    read_flight_recorder, set_flight_recorder_sink, without_flight_recorder,
39};
40
41/// Target prefixes kept in the forwarder's local buffer instead of forwarded, as a backstop for
42/// events the IPC stack emits outside [`without_flight_recorder`] (e.g. from tasks it spawns).
43const UNFORWARDED_TARGETS: &[&str] = &["bitwarden_ipc", "bitwarden_threading"];
44
45/// How long a flush waits before draining, so events logged close together share one IPC message.
46const FLUSH_INTERVAL: Duration = Duration::from_millis(250);
47
48/// How long an aggregated read waits for the collector when the caller gives no cancellation token.
49const DEFAULT_READ_TIMEOUT: Duration = Duration::from_secs(5);
50
51/// How long a forwarder waits for its collector to answer discover before it keeps events local.
52const COLLECTOR_WAIT: Duration = Duration::from_secs(30);
53
54/// How long a forwarder waits before asking again when the collector's discover answer is an error.
55const DISCOVER_RETRY_DELAY: Duration = Duration::from_millis(250);
56
57/// Events sent from a forwarder to its collector.
58#[derive(Debug, Clone, Serialize, Deserialize)]
59pub struct FlightRecorderBatch {
60    /// Events in capture order.
61    pub events: Vec<FlightRecorderEvent>,
62}
63
64impl PayloadTypeName for FlightRecorderBatch {
65    const PAYLOAD_TYPE_NAME: &'static str = "platform.flight-recorder.batch";
66}
67
68/// RPC request from a forwarder for the collector's events.
69#[derive(Debug, Clone, Serialize, Deserialize)]
70pub struct ReadFlightRecorderRequest;
71
72/// RPC response for [`ReadFlightRecorderRequest`].
73#[derive(Debug, Clone, Serialize, Deserialize)]
74pub struct ReadFlightRecorderResponse {
75    /// Events in the collector's buffer, each attributed to a process.
76    pub events: Vec<FlightRecorderEvent>,
77}
78
79/// Why the collector refused a [`ReadFlightRecorderRequest`].
80#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
81pub enum ReadFlightRecorderError {
82    /// The request did not come from the collector's forwarder.
83    SourceNotAccepted,
84}
85
86impl RpcRequest for ReadFlightRecorderRequest {
87    type Response = Result<ReadFlightRecorderResponse, ReadFlightRecorderError>;
88
89    const NAME: &str = "ReadFlightRecorder";
90}
91
92/// The context that collects events for its client.
93#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
94#[bitwarden_ffi::wasm_record]
95pub enum FlightRecorderCollectorRole {
96    /// Browser extension background, collecting from extension pages (popup, popout, sidebar).
97    BrowserBackground,
98    /// Desktop main process, collecting from the renderer.
99    DesktopMain,
100}
101
102impl FlightRecorderCollectorRole {
103    /// Process label for events produced by the collector itself.
104    fn label(self) -> &'static str {
105        match self {
106            Self::BrowserBackground => "browser-background",
107            Self::DesktopMain => "desktop-main",
108        }
109    }
110
111    /// Whether messages from `source` come from this client's forwarder.
112    fn accepts(self, source: &Source) -> bool {
113        match self {
114            Self::BrowserBackground => matches!(source, Source::BrowserForeground { .. }),
115            Self::DesktopMain => matches!(source, Source::DesktopRenderer),
116        }
117    }
118}
119
120/// Process label for events received from `source`.
121fn process_label(source: &Source) -> &'static str {
122    match source {
123        Source::Web { .. } => "web",
124        Source::BrowserForeground { .. } => "browser-popup",
125        Source::BrowserBackground { .. } => "browser-background",
126        Source::Cli { .. } => "cli",
127        Source::DesktopRenderer => "desktop-renderer",
128        Source::DesktopMain => "desktop-main",
129    }
130}
131
132struct ForwarderState {
133    client: Arc<dyn IpcClient>,
134    destination: Endpoint,
135    local: Arc<CircularBuffer<FlightRecorderEvent>>,
136    pending: CircularBuffer<FlightRecorderEvent>,
137    flushing: AtomicBool,
138    /// Number of aggregated reads in flight. Flushes wait while it is non-zero.
139    reading: AtomicUsize,
140}
141
142/// Sink that forwards events to a collector over IPC.
143///
144/// Events wait in a bounded pending buffer. One flush task at a time waits 250 ms, drains the
145/// buffer, and sends the drained events as a single [`FlightRecorderBatch`], so at most one batch
146/// goes out per interval. Flushes do not drain while an aggregated read is in flight, so the read
147/// still finds in `pending` every event the collector's answer lacks. Send failures drop the batch.
148/// When the buffer is full, the oldest events are dropped. Events from the IPC stack itself go to
149/// `local` and are never forwarded.
150#[derive(Clone)]
151pub struct IpcForwardSink(Arc<ForwarderState>);
152
153impl IpcForwardSink {
154    /// Create a sink forwarding to `destination`, holding at most `capacity` unsent events.
155    /// Events from the IPC stack are stored in `local` instead.
156    pub fn new(
157        client: Arc<dyn IpcClient>,
158        destination: Endpoint,
159        capacity: NonZeroUsize,
160        local: Arc<CircularBuffer<FlightRecorderEvent>>,
161    ) -> Self {
162        Self(Arc::new(ForwarderState {
163            client,
164            destination,
165            local,
166            pending: CircularBuffer::new(capacity),
167            flushing: AtomicBool::new(false),
168            reading: AtomicUsize::new(0),
169        }))
170    }
171
172    fn enqueue(&self, event: FlightRecorderEvent) {
173        self.0.pending.push(event);
174    }
175
176    fn schedule_flush(&self) {
177        if !self.0.flushing.swap(true, Ordering::AcqRel) {
178            spawn(without_flight_recorder(flush(Arc::clone(&self.0))));
179        }
180    }
181
182    /// Read the collector's events followed by this context's IPC events and unsent events. The
183    /// collector's events are omitted if it refuses, fails, or does not answer in time.
184    async fn read_aggregated(
185        &self,
186        cancellation_token: Option<CancellationToken>,
187    ) -> Vec<FlightRecorderEvent> {
188        let state = &self.0;
189        let _reading = ReadingGuard::new(&state.reading);
190        // `request` accepts the answer only from `destination`.
191        let request = without_flight_recorder(state.client.request(
192            ReadFlightRecorderRequest,
193            state.destination.clone(),
194            cancellation_token.clone(),
195        ));
196        let response = match cancellation_token {
197            Some(_) => request.await.ok(),
198            None => timeout(DEFAULT_READ_TIMEOUT, request)
199                .await
200                .ok()
201                .and_then(Result::ok),
202        };
203
204        let mut events = response
205            .and_then(Result::ok)
206            .map(|response| response.events)
207            .unwrap_or_default();
208        events.extend(state.local.read());
209        events.extend(state.pending.read());
210        events
211    }
212}
213
214impl FlightRecorderSink for IpcForwardSink {
215    fn push(&self, event: FlightRecorderEvent) {
216        if UNFORWARDED_TARGETS
217            .iter()
218            .any(|prefix| event.target.starts_with(prefix))
219        {
220            self.0.local.push(event);
221            return;
222        }
223        self.enqueue(event);
224        self.schedule_flush();
225    }
226}
227
228async fn flush(state: Arc<ForwarderState>) {
229    loop {
230        sleep(FLUSH_INTERVAL).await;
231        if state.reading.load(Ordering::Acquire) > 0 {
232            continue;
233        }
234        let events = state.pending.take_all();
235        if events.is_empty() {
236            if release_flush(&state) {
237                return;
238            }
239            continue;
240        }
241
242        // Best-effort: a failed send drops the batch.
243        let _ = state
244            .client
245            .send_typed(FlightRecorderBatch { events }, state.destination.clone())
246            .await;
247    }
248}
249
250/// Marks an aggregated read in flight until dropped, including when the read is cancelled.
251struct ReadingGuard<'a>(&'a AtomicUsize);
252
253impl<'a> ReadingGuard<'a> {
254    fn new(reading: &'a AtomicUsize) -> Self {
255        reading.fetch_add(1, Ordering::AcqRel);
256        Self(reading)
257    }
258}
259
260impl Drop for ReadingGuard<'_> {
261    fn drop(&mut self) {
262        self.0.fetch_sub(1, Ordering::AcqRel);
263    }
264}
265
266/// End a flush that drained an empty buffer. Returns false if the flush must keep running.
267fn release_flush(state: &ForwarderState) -> bool {
268    state.flushing.store(false, Ordering::Release);
269    // An event pushed after the drain but before the store above found a flush in progress and
270    // did not schedule one. Take over unless another flush already has.
271    state.pending.is_empty() || state.flushing.swap(true, Ordering::AcqRel)
272}
273
274static FORWARDER: OnceLock<IpcForwardSink> = OnceLock::new();
275
276/// Make this context a forwarder: send captured events to the collector at `destination`, except
277/// events from the IPC stack itself, which stay in the local buffer.
278///
279/// Forwarding starts once the collector answers a discover request, so events are not lost to a
280/// collector that has not started yet. Until then, and if it does not answer within 30 seconds,
281/// events stay in the local buffer. Events captured before forwarding starts
282/// are sent first. Only the first call has effect. No-op if the Flight Recorder is disabled.
283pub fn enable_flight_recorder_forwarding(client: Arc<dyn IpcClient>, destination: Endpoint) {
284    let Some(buffer) = get_flight_recorder_buffer() else {
285        return;
286    };
287    let capacity = NonZeroUsize::new(buffer.capacity()).expect("buffer capacity is non-zero");
288    let sink = IpcForwardSink::new(client, destination, capacity, Arc::clone(&buffer));
289    if FORWARDER.set(sink.clone()).is_err() {
290        return;
291    }
292
293    spawn(async move {
294        let state = &sink.0;
295        let ready = without_flight_recorder(timeout(
296            COLLECTOR_WAIT,
297            wait_for_collector(state.client.as_ref(), state.destination.clone()),
298        ))
299        .await;
300        if !matches!(ready, Ok(true)) {
301            return;
302        }
303
304        // Swap before draining so no event lands in the local buffer after the drain.
305        set_flight_recorder_sink(Arc::new(sink.clone()));
306        for event in buffer.take_all() {
307            sink.enqueue(event);
308        }
309        sink.schedule_flush();
310    });
311}
312
313/// Wait until the collector at `destination` answers discover. Returns false if the local client
314/// cannot subscribe.
315///
316/// An error answer is retried, since a collector answers `NoHandlerFound` between starting its IPC
317/// client and registering its discover handler.
318async fn wait_for_collector(client: &dyn IpcClient, destination: Endpoint) -> bool {
319    loop {
320        match wait_for_peer(client, destination.clone(), None).await {
321            Ok(_) => return true,
322            Err(RequestError::Rpc(_)) => sleep(DISCOVER_RETRY_DELAY).await,
323            Err(_) => return false,
324        }
325    }
326}
327
328static COLLECTING: AtomicBool = AtomicBool::new(false);
329
330/// Make this context the collector for its client, storing events from `role`'s forwarders and
331/// answering their read requests. Only the first successful call has effect. No-op if the Flight
332/// Recorder is disabled.
333pub async fn enable_flight_recorder_collection(
334    client: Arc<dyn IpcClient>,
335    role: FlightRecorderCollectorRole,
336) -> Result<(), SubscribeError> {
337    let Some(buffer) = get_flight_recorder_buffer() else {
338        return Ok(());
339    };
340    // A second collector would store every received event twice.
341    if COLLECTING.swap(true, Ordering::AcqRel) {
342        return Ok(());
343    }
344    let result = start_collector(client, buffer, role).await;
345    if result.is_err() {
346        COLLECTING.store(false, Ordering::Release);
347    }
348    result
349}
350
351async fn start_collector(
352    client: Arc<dyn IpcClient>,
353    buffer: Arc<CircularBuffer<FlightRecorderEvent>>,
354    role: FlightRecorderCollectorRole,
355) -> Result<(), SubscribeError> {
356    let batches = client.subscribe_typed::<FlightRecorderBatch>().await?;
357    client
358        .register_rpc_handler(ReadFlightRecorderHandler {
359            buffer: Arc::clone(&buffer),
360            role,
361        })
362        .await;
363    spawn(collect_batches(batches, buffer, role));
364    Ok(())
365}
366
367fn is_closed(error: &TypedReceiveError) -> bool {
368    matches!(
369        error,
370        TypedReceiveError::Channel(tokio::sync::broadcast::error::RecvError::Closed)
371    )
372}
373
374async fn collect_batches(
375    mut batches: IpcClientTypedSubscription<FlightRecorderBatch>,
376    buffer: Arc<CircularBuffer<FlightRecorderEvent>>,
377    role: FlightRecorderCollectorRole,
378) {
379    loop {
380        match batches.receive(None).await {
381            Ok(message) => {
382                if !role.accepts(&message.source) {
383                    continue;
384                }
385                // Attribute from the transport-provided source; forwarders do not self-declare.
386                let process = process_label(&message.source);
387                for mut event in message.payload.events {
388                    event.process = Some(process.to_owned());
389                    buffer.push(event);
390                }
391            }
392            Err(error) if is_closed(&error) => return,
393            Err(_) => continue,
394        }
395    }
396}
397
398/// RPC handler for [`ReadFlightRecorderRequest`], answering only `role`'s forwarder.
399struct ReadFlightRecorderHandler {
400    buffer: Arc<CircularBuffer<FlightRecorderEvent>>,
401    role: FlightRecorderCollectorRole,
402}
403
404impl RpcHandler for ReadFlightRecorderHandler {
405    type Request = ReadFlightRecorderRequest;
406
407    async fn handle(
408        &self,
409        _request: Self::Request,
410        info: RpcRequestInfo,
411    ) -> Result<ReadFlightRecorderResponse, ReadFlightRecorderError> {
412        if !self.role.accepts(&info.source) {
413            return Err(ReadFlightRecorderError::SourceNotAccepted);
414        }
415        Ok(ReadFlightRecorderResponse {
416            events: attribute(self.buffer.read(), self.role.label()),
417        })
418    }
419}
420
421fn attribute(mut events: Vec<FlightRecorderEvent>, process: &str) -> Vec<FlightRecorderEvent> {
422    for event in &mut events {
423        if event.process.is_none() {
424            event.process = Some(process.to_owned());
425        }
426    }
427    events
428}
429
430/// Read the Flight Recorder events for this client.
431///
432/// In a forwarder, requests the collector's events and appends this context's IPC events and
433/// events not yet sent. If the collector cannot be reached or does not answer before
434/// `cancellation_token` is cancelled, only this context's events are returned. Without a token,
435/// the wait is bounded at 5 seconds. Elsewhere, reads the local buffer.
436pub async fn read_flight_recorder_aggregated(
437    cancellation_token: Option<CancellationToken>,
438) -> Vec<FlightRecorderEvent> {
439    match FORWARDER.get() {
440        Some(forwarder) => forwarder.read_aggregated(cancellation_token).await,
441        None => read_flight_recorder(),
442    }
443}
444
445#[cfg(test)]
446mod tests {
447    use std::{collections::HashMap, time::Duration};
448
449    use bitwarden_ipc::{
450        InMemorySessionRepository, IncomingMessage, IpcClientImpl, NoEncryptionCryptoProvider,
451        OutgoingMessage, TestCommunicationBackend, TypedIncomingMessage, TypedOutgoingMessage,
452        discover::{DiscoverHandler, DiscoverResponse},
453    };
454
455    use super::*;
456
457    fn event(target: &str, message: &str) -> FlightRecorderEvent {
458        FlightRecorderEvent {
459            timestamp: 0,
460            level: "INFO".to_string(),
461            target: target.to_string(),
462            message: message.to_string(),
463            fields: HashMap::new(),
464            process: None,
465        }
466    }
467
468    async fn start_client(backend: &TestCommunicationBackend) -> Arc<dyn IpcClient> {
469        let client: Arc<dyn IpcClient> = Arc::new(IpcClientImpl::new(
470            NoEncryptionCryptoProvider,
471            backend.clone(),
472            InMemorySessionRepository::<()>::default(),
473        ));
474        client.start(None).await.expect("client starts");
475        client
476    }
477
478    fn incoming<P: Serialize + PayloadTypeName>(payload: P, source: Source) -> IncomingMessage {
479        TypedIncomingMessage {
480            payload,
481            destination: Endpoint::BrowserBackground {
482                id: bitwarden_ipc::HostId::Own,
483            },
484            source,
485        }
486        .try_into()
487        .expect("serializes")
488    }
489
490    fn decode<P: serde::de::DeserializeOwned + PayloadTypeName>(
491        messages: &[OutgoingMessage],
492    ) -> Vec<P> {
493        messages
494            .iter()
495            .filter(|m| m.topic.as_deref() == Some(P::PAYLOAD_TYPE_NAME))
496            .map(|m| {
497                TypedOutgoingMessage::<P>::try_from(m.clone())
498                    .expect("deserializes")
499                    .payload
500            })
501            .collect()
502    }
503
504    async fn settle() {
505        tokio::time::sleep(Duration::from_millis(20)).await;
506    }
507
508    async fn wait_for_flush() {
509        tokio::time::sleep(FLUSH_INTERVAL + Duration::from_millis(50)).await;
510    }
511
512    fn popup() -> Source {
513        Source::BrowserForeground { id: 1 }
514    }
515
516    fn web_page() -> Source {
517        Source::Web {
518            tab_id: 1,
519            document_id: "doc".to_string(),
520            origin: "https://example.com".to_string(),
521        }
522    }
523
524    fn buffer() -> Arc<CircularBuffer<FlightRecorderEvent>> {
525        Arc::new(CircularBuffer::new(
526            NonZeroUsize::new(100).expect("non-zero"),
527        ))
528    }
529
530    #[tokio::test(start_paused = true)]
531    async fn test_forward_sink_batches_events_pushed_within_the_interval() {
532        let backend = TestCommunicationBackend::new();
533        let client = start_client(&backend).await;
534        let sink = IpcForwardSink::new(
535            client,
536            Endpoint::DesktopMain,
537            NonZeroUsize::new(10).expect("non-zero"),
538            buffer(),
539        );
540
541        sink.push(event("app", "one"));
542        tokio::time::sleep(FLUSH_INTERVAL / 2).await;
543        assert!(decode::<FlightRecorderBatch>(&backend.outgoing().await).is_empty());
544
545        sink.push(event("app", "two"));
546        wait_for_flush().await;
547
548        let batches = decode::<FlightRecorderBatch>(&backend.outgoing().await);
549        assert_eq!(batches.len(), 1);
550        assert_eq!(messages(&batches[0].events), ["one", "two"]);
551
552        // The flush stopped once its queue was empty, so a later event starts a new batch.
553        sink.push(event("app", "three"));
554        wait_for_flush().await;
555        assert_eq!(
556            decode::<FlightRecorderBatch>(&backend.outgoing().await).len(),
557            2
558        );
559    }
560
561    #[tokio::test(start_paused = true)]
562    async fn test_forward_sink_keeps_ipc_targets_locally() {
563        let backend = TestCommunicationBackend::new();
564        let client = start_client(&backend).await;
565        let local = buffer();
566        let sink = IpcForwardSink::new(
567            client,
568            Endpoint::DesktopMain,
569            NonZeroUsize::new(10).expect("non-zero"),
570            Arc::clone(&local),
571        );
572
573        sink.push(event("bitwarden_ipc::ipc_client", "ipc"));
574        sink.push(event("bitwarden_threading", "threading"));
575        wait_for_flush().await;
576
577        assert!(decode::<FlightRecorderBatch>(&backend.outgoing().await).is_empty());
578        let kept: Vec<_> = local.read().into_iter().map(|e| e.message).collect();
579        assert_eq!(kept, ["ipc", "threading"]);
580    }
581
582    #[tokio::test(start_paused = true)]
583    async fn test_collector_stores_batches_from_its_forwarder_only() {
584        let backend = TestCommunicationBackend::new();
585        let client = start_client(&backend).await;
586        let buffer = buffer();
587        start_collector(
588            client,
589            Arc::clone(&buffer),
590            FlightRecorderCollectorRole::BrowserBackground,
591        )
592        .await
593        .expect("collector starts");
594
595        let batch = |message: &str| FlightRecorderBatch {
596            events: vec![FlightRecorderEvent {
597                process: Some("spoofed".to_string()),
598                ..event("typescript", message)
599            }],
600        };
601        backend.push_incoming(incoming(batch("from popup"), popup()));
602        backend.push_incoming(incoming(batch("from web"), web_page()));
603        settle().await;
604
605        let events = buffer.read();
606        assert_eq!(events.len(), 1);
607        assert_eq!(events[0].message, "from popup");
608        assert_eq!(events[0].process.as_deref(), Some("browser-popup"));
609    }
610
611    fn background() -> Source {
612        Source::BrowserBackground {
613            id: bitwarden_ipc::HostId::Own,
614        }
615    }
616
617    /// Deliver the messages sent through `from` to `to`, as if they came from `source`.
618    async fn relay(from: &TestCommunicationBackend, to: &TestCommunicationBackend, source: Source) {
619        for message in from.drain_outgoing().await {
620            to.push_incoming(IncomingMessage {
621                payload: message.payload,
622                destination: message.destination,
623                source: source.clone(),
624                topic: message.topic,
625            });
626        }
627    }
628
629    async fn start_test_collector(events: &[&str]) -> TestCommunicationBackend {
630        let backend = TestCommunicationBackend::new();
631        let buffer = buffer();
632        for message in events {
633            buffer.push(event("app", message));
634        }
635        start_collector(
636            start_client(&backend).await,
637            buffer,
638            FlightRecorderCollectorRole::BrowserBackground,
639        )
640        .await
641        .expect("collector starts");
642        backend
643    }
644
645    fn messages(events: &[FlightRecorderEvent]) -> Vec<&str> {
646        events.iter().map(|e| e.message.as_str()).collect()
647    }
648
649    #[tokio::test(start_paused = true)]
650    async fn test_collector_answers_reads_from_its_forwarder_only() {
651        let collector = start_test_collector(&["own event"]).await;
652        let backend = TestCommunicationBackend::new();
653        let client = start_client(&backend).await;
654
655        let read_as = async |source: Source| {
656            let request = tokio::spawn({
657                let client = Arc::clone(&client);
658                async move {
659                    client
660                        .request(ReadFlightRecorderRequest, background().to_endpoint(), None)
661                        .await
662                }
663            });
664            settle().await;
665            relay(&backend, &collector, source).await;
666            settle().await;
667            relay(&collector, &backend, background()).await;
668            request
669                .await
670                .expect("task joins")
671                .expect("request succeeds")
672        };
673
674        let from_web = read_as(web_page()).await;
675        assert!(matches!(
676            from_web,
677            Err(ReadFlightRecorderError::SourceNotAccepted)
678        ));
679
680        let events = read_as(popup()).await.expect("popup is accepted").events;
681        assert_eq!(messages(&events), ["own event"]);
682        assert_eq!(events[0].process.as_deref(), Some("browser-background"));
683    }
684
685    #[tokio::test(start_paused = true)]
686    async fn test_forwarder_read_returns_collector_events_then_local_and_pending() {
687        let collector = start_test_collector(&["collected"]).await;
688        let backend = TestCommunicationBackend::new();
689        let local = buffer();
690        local.push(event("bitwarden_ipc", "ipc"));
691        let sink = IpcForwardSink::new(
692            start_client(&backend).await,
693            background().to_endpoint(),
694            NonZeroUsize::new(10).expect("non-zero"),
695            local,
696        );
697        sink.enqueue(event("app", "unsent"));
698
699        let read = tokio::spawn({
700            let sink = sink.clone();
701            async move { sink.read_aggregated(None).await }
702        });
703        settle().await;
704        relay(&backend, &collector, popup()).await;
705        settle().await;
706        relay(&collector, &backend, background()).await;
707
708        let events = read.await.expect("task joins");
709        assert_eq!(messages(&events), ["collected", "ipc", "unsent"]);
710    }
711
712    #[tokio::test(start_paused = true)]
713    async fn test_forwarder_read_falls_back_to_local_and_pending_without_an_answer() {
714        let backend = TestCommunicationBackend::new();
715        let local = buffer();
716        local.push(event("bitwarden_ipc", "ipc"));
717        let sink = IpcForwardSink::new(
718            start_client(&backend).await,
719            background().to_endpoint(),
720            NonZeroUsize::new(10).expect("non-zero"),
721            local,
722        );
723        sink.enqueue(event("app", "unsent"));
724
725        let cancellation_token = CancellationToken::new();
726        let read = tokio::spawn({
727            let sink = sink.clone();
728            let cancellation_token = cancellation_token.clone();
729            async move { sink.read_aggregated(Some(cancellation_token)).await }
730        });
731        settle().await;
732        cancellation_token.cancel();
733
734        let events = read.await.expect("task joins");
735        assert_eq!(messages(&events), ["ipc", "unsent"]);
736    }
737
738    #[tokio::test(start_paused = true)]
739    async fn test_forwarder_read_keeps_pending_events_a_flush_would_send_mid_read() {
740        let collector = start_test_collector(&["collected"]).await;
741        let backend = TestCommunicationBackend::new();
742        let sink = IpcForwardSink::new(
743            start_client(&backend).await,
744            background().to_endpoint(),
745            NonZeroUsize::new(10).expect("non-zero"),
746            buffer(),
747        );
748        sink.push(event("app", "just before read"));
749
750        let read = tokio::spawn({
751            let sink = sink.clone();
752            async move { sink.read_aggregated(None).await }
753        });
754        settle().await;
755        relay(&backend, &collector, popup()).await;
756        // The flush interval elapses while the collector's answer is on its way.
757        wait_for_flush().await;
758        relay(&collector, &backend, background()).await;
759
760        let events = read.await.expect("task joins");
761        assert_eq!(messages(&events), ["collected", "just before read"]);
762
763        // The paused flush sends the event once the read is done.
764        wait_for_flush().await;
765        let batches = decode::<FlightRecorderBatch>(&backend.outgoing().await);
766        assert_eq!(batches.len(), 1);
767        assert_eq!(messages(&batches[0].events), ["just before read"]);
768    }
769
770    #[tokio::test(start_paused = true)]
771    async fn test_flush_keeps_running_for_an_event_pushed_after_its_drain() {
772        let backend = TestCommunicationBackend::new();
773        let sink = IpcForwardSink::new(
774            start_client(&backend).await,
775            Endpoint::DesktopMain,
776            NonZeroUsize::new(10).expect("non-zero"),
777            buffer(),
778        );
779        let state = &sink.0;
780
781        // A flush is running and has just drained an empty buffer, so this push schedules nothing.
782        state.flushing.store(true, Ordering::Release);
783        sink.push(event("app", "late"));
784
785        assert!(!release_flush(state));
786        assert!(state.flushing.load(Ordering::Acquire));
787
788        // The rest of the running flush.
789        flush(Arc::clone(state)).await;
790        let batches = decode::<FlightRecorderBatch>(&backend.outgoing().await);
791        assert_eq!(batches.len(), 1);
792        assert_eq!(messages(&batches[0].events), ["late"]);
793        assert!(!state.flushing.load(Ordering::Acquire));
794    }
795
796    #[tokio::test(start_paused = true)]
797    async fn test_wait_for_collector_retries_until_the_discover_handler_is_registered() {
798        let collector_backend = TestCommunicationBackend::new();
799        let collector = start_client(&collector_backend).await;
800        let backend = TestCommunicationBackend::new();
801        let client = start_client(&backend).await;
802
803        let wait = tokio::spawn({
804            let client = Arc::clone(&client);
805            async move { wait_for_collector(client.as_ref(), background().to_endpoint()).await }
806        });
807
808        // Without a discover handler, the collector answers with an error.
809        settle().await;
810        relay(&backend, &collector_backend, popup()).await;
811        settle().await;
812        relay(&collector_backend, &backend, background()).await;
813        settle().await;
814        assert!(!wait.is_finished());
815
816        collector
817            .register_rpc_handler(DiscoverHandler::new(DiscoverResponse {
818                version: "test".to_string(),
819            }))
820            .await;
821        tokio::time::sleep(DISCOVER_RETRY_DELAY).await;
822        settle().await;
823        relay(&backend, &collector_backend, popup()).await;
824        settle().await;
825        relay(&collector_backend, &backend, background()).await;
826
827        assert!(wait.await.expect("task joins"));
828    }
829}