Skip to main content

confluent_kafka/consumer/
kafka_consumer.rs

1// Copyright 2025 Confluent Inc.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! The public consumer type.
16//!
17//! Translated from `org.apache.kafka.clients.consumer.KafkaConsumer`.
18
19use std::cell::Cell;
20use std::collections::{HashMap, HashSet};
21use std::marker::PhantomData;
22use std::sync::Arc;
23use std::time::Duration;
24
25use crate::common::metrics::MetricsView;
26use crate::common::serialization::{Deserializer, DynDeserializer};
27use crate::common::{Error, PartitionInfo, TopicPartition};
28use crate::consumer::internals::AsyncKafkaConsumer;
29use crate::consumer::{
30    CloseOptions, Consumer, ConsumerConfig, ConsumerGroupMetadata, ConsumerHandle, ConsumerRebalanceListener,
31    ConsumerRecords, GroupProtocol, OffsetAndMetadata, OffsetAndTimestamp, OffsetCommitCallback, SubscriptionPattern,
32};
33
34/// A client that consumes records from a Kafka cluster.
35///
36/// # Translation notes
37///
38/// The concrete type lets callers dispatch statically: calling a [`Consumer`]
39/// method on a `KafkaConsumer`, or through generic code taking
40/// `C: Consumer<K, V>`, involves no boxed future. To hold it next to other
41/// consumer implementations behind one type, box it as
42/// [`DynConsumer`](crate::consumer::DynConsumer).
43///
44/// The consumer is `Send` but not `Sync`: it can move to another task or
45/// thread, but it cannot be shared between them. This is the type-level form
46/// of Java's `KafkaConsumer.acquire()` guard, which throws
47/// `ConcurrentModificationException` when a second thread calls into a
48/// consumer another thread is using: every [`Consumer`] method Java guards
49/// takes `&mut self`, and without `Sync` not even the methods taking `&self`
50/// can be reached from two threads at once. To reach the consumer from
51/// another task or thread — to call `wakeup()` while it waits, or to call back
52/// into it from a rebalance listener — take a
53/// [`ConsumerHandle`](crate::consumer::ConsumerHandle) with
54/// [`Consumer::handle`], which is `Send + Sync`.
55///
56/// The consumer cannot be shared between threads:
57///
58/// ```compile_fail,E0277
59/// use confluent_kafka::consumer::KafkaConsumer;
60///
61/// fn assert_sync<T: Sync>() {}
62/// assert_sync::<KafkaConsumer<String, String>>();
63/// ```
64///
65/// and its getters cannot be called through a shared reference:
66///
67/// ```compile_fail,E0596
68/// use std::collections::HashSet;
69///
70/// use confluent_kafka::common::TopicPartition;
71/// use confluent_kafka::consumer::{Consumer, KafkaConsumer};
72///
73/// fn read_assignment(consumer: &KafkaConsumer<String, String>) -> HashSet<TopicPartition> {
74///     consumer.assignment()
75/// }
76/// ```
77// Java's class is a thin delegating wrapper: its constructor calls
78// `ConsumerDelegateCreator.create(config, keyDeserializer, valueDeserializer)`,
79// which switches on `group.protocol` and returns either an `AsyncKafkaConsumer`
80// or a `ClassicKafkaConsumer`; its other methods forward verbatim to that
81// delegate. The Rust struct has the same shape. `ConsumerDelegate` and
82// `ConsumerDelegateCreator` are out of scope per `consumer-threading.md` §20:
83// only the KIP-848 delegate exists, so it is held as `AsyncKafkaConsumer`
84// directly and `new` is where the collapsed creator lives. When the classic
85// protocol is added, the field becomes a private enum over the two delegates.
86#[doc(alias = "org.apache.kafka.clients.consumer.KafkaConsumer")]
87pub struct KafkaConsumer<K, V>
88where
89    K: Send + Sync + 'static,
90    V: Send + Sync + 'static,
91{
92    delegate: AsyncKafkaConsumer<K, V>,
93    /// Zero-sized: keeps the consumer `Send` and makes it `!Sync`, the
94    /// type-level form of Java's `acquire()` guard (see the type's docs).
95    /// Cross-thread access goes through [`ConsumerHandle`].
96    not_sync: PhantomData<Cell<()>>,
97}
98
99impl<K, V> std::fmt::Debug for KafkaConsumer<K, V>
100where
101    K: Send + Sync + 'static,
102    V: Send + Sync + 'static,
103{
104    // The delegate's client id and group id, never the configuration.
105    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
106        f.debug_struct("KafkaConsumer")
107            .field("delegate", &self.delegate)
108            .finish_non_exhaustive()
109    }
110}
111
112impl<K, V> KafkaConsumer<K, V>
113where
114    K: Send + Sync + 'static,
115    V: Send + Sync + 'static,
116{
117    /// A consumer is instantiated by providing a set of key-value pairs as
118    /// configuration, and a key and a value [`Deserializer`].
119    ///
120    /// Valid configuration strings are documented at [`ConsumerConfig`].
121    ///
122    /// Note: after creating a `KafkaConsumer` you must always
123    /// [`close`](Consumer::close) it to avoid resource leaks.
124    ///
125    /// # Arguments
126    ///
127    /// - `config` - The consumer configs
128    /// - `key_deserializer` - The deserializer for key that implements `Deserializer`. The
129    ///   configure() method won't be called in the consumer when the deserializer is passed in
130    ///   directly.
131    /// - `value_deserializer` - The deserializer for value that implements `Deserializer`. The
132    ///   configure() method won't be called in the consumer when the deserializer is passed in
133    ///   directly.
134    ///
135    /// # Returns
136    ///
137    /// The new consumer.
138    ///
139    /// # Errors
140    ///
141    /// - `Error::is_local_illegal_argument_error` if `group.protocol` is invalid.
142    /// - `Error::is_local_unsupported_operation_error` if `group.protocol` is
143    ///   `classic`.
144    /// - Any error building the consumer from `config`.
145    ///
146    /// # Translation notes
147    ///
148    /// The configuration is a built [`ConsumerConfig`], and the deserializers
149    /// are not configuration keys. Only `group.protocol=consumer` (KIP-848) is
150    /// supported.
151    // `classic` fails with `LocalUnsupportedOperationError`, the class Java's
152    // `ClassicKafkaConsumer` throws for a protocol feature it does not support
153    // (consumer-threading.md §20). The deserializers are boxed once behind the
154    // crate-private `DynDeserializer`, so the consumer's type carries only `K`
155    // and `V`; the `FetchCollector` calls them per record and the consumer
156    // closes them on close. `MockConsumer` has its own constructor.
157    #[doc(alias = "org.apache.kafka.clients.consumer.KafkaConsumer#KafkaConsumer(Map,Deserializer,Deserializer)")]
158    pub fn new(
159        config: ConsumerConfig,
160        key_deserializer: impl Deserializer<K>,
161        value_deserializer: impl Deserializer<V>,
162    ) -> Result<Self, Error> {
163        Self::with_boxed_deserializers(config, Box::new(key_deserializer), Box::new(value_deserializer))
164    }
165
166    /// [`new`](Self::new) over deserializers already boxed as `dyn`.
167    ///
168    /// For the C binding, whose deserializers are trait objects before they
169    /// reach the consumer: boxing them as `dyn` a second time would nest one
170    /// `InlineFuture` in another, which no longer fits its buffer, so every
171    /// fetched record would allocate (`consumer-threading.md` §27).
172    pub(crate) fn with_boxed_deserializers(
173        config: ConsumerConfig,
174        key_deserializer: Box<dyn DynDeserializer<K>>,
175        value_deserializer: Box<dyn DynDeserializer<V>>,
176    ) -> Result<Self, Error> {
177        // The `GroupProtocol::Consumer` arm builds the production delegate with
178        // [`AsyncKafkaConsumer::new`], which translates the Java primary
179        // constructor `AsyncKafkaConsumer#AsyncKafkaConsumer` end-to-end. The ctor
180        // builds the full dependency closure (`SubscriptionState`,
181        // `ConsumerMetadata`, `NetworkClient` + PLAINTEXT `ChannelBuilder`,
182        // every `RequestManager`, `ApplicationEventHandler`,
183        // `ConsumerNetworkThread` bg task) and hands off to
184        // `AsyncKafkaConsumer::with_components` so the Phase-11 test seam is
185        // preserved.
186        //
187        // `GroupProtocol::Classic` remains a `local_unsupported_operation` error
188        // per `consumer-threading.md` §20 (classic protocol deferred to a
189        // later milestone).
190        let protocol = GroupProtocol::of(config.group_protocol())?;
191        match protocol {
192            GroupProtocol::Consumer => Ok(Self {
193                delegate: AsyncKafkaConsumer::<K, V>::new(config, key_deserializer, value_deserializer)?,
194                not_sync: PhantomData,
195            }),
196            GroupProtocol::Classic => Err(Error::local_unsupported_operation(
197                "Classic group protocol is not yet supported in this client; \
198                 set group.protocol=consumer (KIP-848).",
199            )),
200        }
201    }
202}
203
204// Every method forwards to the delegate's own `Consumer` impl, as Java's
205// `KafkaConsumer` forwards to `delegate`. The delegate's futures are returned
206// as they are, so no call is boxed.
207impl<K, V> Consumer<K, V> for KafkaConsumer<K, V>
208where
209    K: Send + Sync + 'static,
210    V: Send + Sync + 'static,
211{
212    fn assignment(&mut self) -> HashSet<TopicPartition> {
213        Consumer::assignment(&mut self.delegate)
214    }
215
216    fn subscription(&mut self) -> HashSet<String> {
217        Consumer::subscription(&mut self.delegate)
218    }
219
220    fn paused(&mut self) -> HashSet<TopicPartition> {
221        Consumer::paused(&mut self.delegate)
222    }
223
224    fn group_metadata(&mut self) -> Arc<dyn ConsumerGroupMetadata> {
225        Consumer::group_metadata(&mut self.delegate)
226    }
227
228    fn current_lag(
229        &mut self,
230        topic_partition: &TopicPartition,
231    ) -> impl Future<Output = Result<Option<i64>, Error>> + Send {
232        Consumer::current_lag(&mut self.delegate, topic_partition)
233    }
234
235    fn metrics(&self) -> MetricsView {
236        Consumer::metrics(&self.delegate)
237    }
238
239    fn subscribe_with_topics(&mut self, topics: Vec<String>) -> impl Future<Output = Result<(), Error>> + Send {
240        Consumer::subscribe_with_topics(&mut self.delegate, topics)
241    }
242
243    fn subscribe_with_topics_listener(
244        &mut self,
245        topics: Vec<String>,
246        listener: Arc<dyn ConsumerRebalanceListener>,
247    ) -> impl Future<Output = Result<(), Error>> + Send {
248        Consumer::subscribe_with_topics_listener(&mut self.delegate, topics, listener)
249    }
250
251    fn subscribe_with_pattern(
252        &mut self,
253        pattern: SubscriptionPattern,
254    ) -> impl Future<Output = Result<(), Error>> + Send {
255        Consumer::subscribe_with_pattern(&mut self.delegate, pattern)
256    }
257
258    fn subscribe_with_pattern_listener(
259        &mut self,
260        pattern: SubscriptionPattern,
261        listener: Arc<dyn ConsumerRebalanceListener>,
262    ) -> impl Future<Output = Result<(), Error>> + Send {
263        Consumer::subscribe_with_pattern_listener(&mut self.delegate, pattern, listener)
264    }
265
266    fn assign(&mut self, partitions: Vec<TopicPartition>) -> impl Future<Output = Result<(), Error>> + Send {
267        Consumer::assign(&mut self.delegate, partitions)
268    }
269
270    fn unsubscribe(&mut self) -> impl Future<Output = Result<(), Error>> + Send {
271        Consumer::unsubscribe(&mut self.delegate)
272    }
273
274    fn poll(&mut self, timeout: Duration) -> impl Future<Output = Result<ConsumerRecords<K, V>, Error>> + Send {
275        Consumer::poll(&mut self.delegate, timeout)
276    }
277
278    fn commit_sync(&mut self) -> impl Future<Output = Result<(), Error>> + Send {
279        Consumer::commit_sync(&mut self.delegate)
280    }
281
282    fn commit_sync_with_timeout(&mut self, timeout: Duration) -> impl Future<Output = Result<(), Error>> + Send {
283        Consumer::commit_sync_with_timeout(&mut self.delegate, timeout)
284    }
285
286    fn commit_sync_with_offsets(
287        &mut self,
288        offsets: HashMap<TopicPartition, OffsetAndMetadata>,
289    ) -> impl Future<Output = Result<(), Error>> + Send {
290        Consumer::commit_sync_with_offsets(&mut self.delegate, offsets)
291    }
292
293    fn commit_sync_with_offsets_timeout(
294        &mut self,
295        offsets: HashMap<TopicPartition, OffsetAndMetadata>,
296        timeout: Duration,
297    ) -> impl Future<Output = Result<(), Error>> + Send {
298        Consumer::commit_sync_with_offsets_timeout(&mut self.delegate, offsets, timeout)
299    }
300
301    fn commit_async(&mut self) -> impl Future<Output = Result<(), Error>> + Send {
302        Consumer::commit_async(&mut self.delegate)
303    }
304
305    fn commit_async_with_callback(
306        &mut self,
307        callback: Arc<dyn OffsetCommitCallback>,
308    ) -> impl Future<Output = Result<(), Error>> + Send {
309        Consumer::commit_async_with_callback(&mut self.delegate, callback)
310    }
311
312    fn commit_async_with_offsets_callback(
313        &mut self,
314        offsets: HashMap<TopicPartition, OffsetAndMetadata>,
315        callback: Arc<dyn OffsetCommitCallback>,
316    ) -> impl Future<Output = Result<(), Error>> + Send {
317        Consumer::commit_async_with_offsets_callback(&mut self.delegate, offsets, callback)
318    }
319
320    fn seek_with_offset(
321        &mut self,
322        partition: TopicPartition,
323        offset: i64,
324    ) -> impl Future<Output = Result<(), Error>> + Send {
325        Consumer::seek_with_offset(&mut self.delegate, partition, offset)
326    }
327
328    fn seek_with_offset_and_metadata(
329        &mut self,
330        partition: TopicPartition,
331        offset_and_metadata: OffsetAndMetadata,
332    ) -> impl Future<Output = Result<(), Error>> + Send {
333        Consumer::seek_with_offset_and_metadata(&mut self.delegate, partition, offset_and_metadata)
334    }
335
336    fn seek_to_beginning(&mut self, partitions: &[TopicPartition]) -> impl Future<Output = Result<(), Error>> + Send {
337        Consumer::seek_to_beginning(&mut self.delegate, partitions)
338    }
339
340    fn seek_to_end(&mut self, partitions: &[TopicPartition]) -> impl Future<Output = Result<(), Error>> + Send {
341        Consumer::seek_to_end(&mut self.delegate, partitions)
342    }
343
344    fn position(&mut self, partition: &TopicPartition) -> impl Future<Output = Result<i64, Error>> + Send {
345        Consumer::position(&mut self.delegate, partition)
346    }
347
348    fn position_with_timeout(
349        &mut self,
350        partition: &TopicPartition,
351        timeout: Duration,
352    ) -> impl Future<Output = Result<i64, Error>> + Send {
353        Consumer::position_with_timeout(&mut self.delegate, partition, timeout)
354    }
355
356    fn committed(
357        &mut self,
358        partitions: &[TopicPartition],
359    ) -> impl Future<Output = Result<HashMap<TopicPartition, Option<OffsetAndMetadata>>, Error>> + Send {
360        Consumer::committed(&mut self.delegate, partitions)
361    }
362
363    fn committed_with_timeout(
364        &mut self,
365        partitions: &[TopicPartition],
366        timeout: Duration,
367    ) -> impl Future<Output = Result<HashMap<TopicPartition, Option<OffsetAndMetadata>>, Error>> + Send {
368        Consumer::committed_with_timeout(&mut self.delegate, partitions, timeout)
369    }
370
371    fn partitions_for(&mut self, topic: &str) -> impl Future<Output = Result<Vec<PartitionInfo>, Error>> + Send {
372        Consumer::partitions_for(&mut self.delegate, topic)
373    }
374
375    fn partitions_for_with_timeout(
376        &mut self,
377        topic: &str,
378        timeout: Duration,
379    ) -> impl Future<Output = Result<Vec<PartitionInfo>, Error>> + Send {
380        Consumer::partitions_for_with_timeout(&mut self.delegate, topic, timeout)
381    }
382
383    fn list_topics(&mut self) -> impl Future<Output = Result<HashMap<String, Vec<PartitionInfo>>, Error>> + Send {
384        Consumer::list_topics(&mut self.delegate)
385    }
386
387    fn list_topics_with_timeout(
388        &mut self,
389        timeout: Duration,
390    ) -> impl Future<Output = Result<HashMap<String, Vec<PartitionInfo>>, Error>> + Send {
391        Consumer::list_topics_with_timeout(&mut self.delegate, timeout)
392    }
393
394    fn offsets_for_times(
395        &mut self,
396        timestamps_to_search: HashMap<TopicPartition, i64>,
397    ) -> impl Future<Output = Result<HashMap<TopicPartition, Option<OffsetAndTimestamp>>, Error>> + Send {
398        Consumer::offsets_for_times(&mut self.delegate, timestamps_to_search)
399    }
400
401    fn offsets_for_times_with_timeout(
402        &mut self,
403        timestamps_to_search: HashMap<TopicPartition, i64>,
404        timeout: Duration,
405    ) -> impl Future<Output = Result<HashMap<TopicPartition, Option<OffsetAndTimestamp>>, Error>> + Send {
406        Consumer::offsets_for_times_with_timeout(&mut self.delegate, timestamps_to_search, timeout)
407    }
408
409    fn beginning_offsets(
410        &mut self,
411        partitions: &[TopicPartition],
412    ) -> impl Future<Output = Result<HashMap<TopicPartition, i64>, Error>> + Send {
413        Consumer::beginning_offsets(&mut self.delegate, partitions)
414    }
415
416    fn beginning_offsets_with_timeout(
417        &mut self,
418        partitions: &[TopicPartition],
419        timeout: Duration,
420    ) -> impl Future<Output = Result<HashMap<TopicPartition, i64>, Error>> + Send {
421        Consumer::beginning_offsets_with_timeout(&mut self.delegate, partitions, timeout)
422    }
423
424    fn end_offsets(
425        &mut self,
426        partitions: &[TopicPartition],
427    ) -> impl Future<Output = Result<HashMap<TopicPartition, i64>, Error>> + Send {
428        Consumer::end_offsets(&mut self.delegate, partitions)
429    }
430
431    fn end_offsets_with_timeout(
432        &mut self,
433        partitions: &[TopicPartition],
434        timeout: Duration,
435    ) -> impl Future<Output = Result<HashMap<TopicPartition, i64>, Error>> + Send {
436        Consumer::end_offsets_with_timeout(&mut self.delegate, partitions, timeout)
437    }
438
439    fn pause(&mut self, partitions: &[TopicPartition]) -> impl Future<Output = Result<(), Error>> + Send {
440        Consumer::pause(&mut self.delegate, partitions)
441    }
442
443    fn resume(&mut self, partitions: &[TopicPartition]) -> impl Future<Output = Result<(), Error>> + Send {
444        Consumer::resume(&mut self.delegate, partitions)
445    }
446
447    fn enforce_rebalance(&mut self) -> impl Future<Output = Result<(), Error>> + Send {
448        Consumer::enforce_rebalance(&mut self.delegate)
449    }
450
451    fn enforce_rebalance_with_reason(&mut self, reason: &str) -> impl Future<Output = Result<(), Error>> + Send {
452        Consumer::enforce_rebalance_with_reason(&mut self.delegate, reason)
453    }
454
455    fn close(&mut self) -> impl Future<Output = Result<(), Error>> + Send {
456        Consumer::close(&mut self.delegate)
457    }
458
459    fn close_with_options(&mut self, options: CloseOptions) -> impl Future<Output = Result<(), Error>> + Send {
460        Consumer::close_with_options(&mut self.delegate, options)
461    }
462
463    fn wakeup(&self) {
464        Consumer::wakeup(&self.delegate)
465    }
466
467    fn handle(&self) -> ConsumerHandle {
468        Consumer::handle(&self.delegate)
469    }
470}
471
472#[cfg(test)]
473mod tests {
474    use super::*;
475    use crate::common::protocol::Errors;
476    use crate::common::serialization::Deserializer;
477    use crate::consumer::CloseOptions;
478    use crate::test_fake_broker::{FakeBroker, FetchAnswer, ListOffsetsAnswer};
479    use crate::test_thread_safety::{is_send, is_sync};
480
481    const TOPIC: &str = "current-lag-topic";
482
483    /// `ListOffsetsRequest.LATEST_TIMESTAMP`: the request `currentLag` sends
484    /// for the log-end offset.
485    const LATEST_TIMESTAMP: i64 = -1;
486
487    /// How long a test waits for the background task, as Java's
488    /// `TestUtils.waitForCondition` default.
489    const WAIT: Duration = Duration::from_secs(15);
490
491    struct StringDeserializer;
492
493    impl Deserializer<String> for StringDeserializer {
494        async fn deserialize(&self, _topic: &str, data: &[u8]) -> Result<String, Error> {
495            String::from_utf8(data.to_vec()).map_err(|e| Error::serialization(format!("invalid utf-8: {e}")))
496        }
497    }
498
499    fn tp0() -> TopicPartition {
500        TopicPartition::new(TOPIC, 0)
501    }
502
503    fn latest_list_offsets(broker: &FakeBroker) -> usize {
504        broker
505            .list_offsets_timestamps()
506            .into_iter()
507            .filter(|timestamp| *timestamp == LATEST_TIMESTAMP)
508            .count()
509    }
510
511    /// A real `KafkaConsumer` against the fake broker. `earliest` makes a
512    /// position reset ask for `EARLIEST_TIMESTAMP`, so the `LATEST_TIMESTAMP`
513    /// requests counted by the tests are `currentLag`'s only.
514    fn new_consumer(broker: &FakeBroker) -> KafkaConsumer<String, String> {
515        let props = HashMap::from([
516            ("bootstrap.servers".to_string(), broker.bootstrap()),
517            ("group.protocol".to_string(), "consumer".to_string()),
518            ("auto.offset.reset".to_string(), "earliest".to_string()),
519            // Java's `newConsumer` sets it too (`KafkaConsumerTest#newConsumer`).
520            ("isolation.level".to_string(), "read_uncommitted".to_string()),
521        ]);
522        KafkaConsumer::new(
523            ConsumerConfig::new(&props).expect("config validates"),
524            StringDeserializer,
525            StringDeserializer,
526        )
527        .expect("constructs against the fake broker")
528    }
529
530    /// Java's `KafkaConsumerTest.setUpConsumerForCurrentLag`: the lag of an
531    /// unassigned partition is an error, then the partition is assigned and
532    /// the consumer learns its leader.
533    ///
534    /// Java seeds the metadata (`initMetadata`) and polls once. Here the
535    /// background task fetches it from the fake broker: until it knows the
536    /// leader, the one-shot LIST_OFFSETS of `currentLag` completes without
537    /// being sent, so the lag is asked for until the request reaches the
538    /// broker, where it is held. Every Java test then makes its first
539    /// `currentLag` call against that held request. No poll: one before a
540    /// `seek` would reset the position with a LIST_OFFSETS of its own.
541    ///
542    /// Without the group: Java's helper answers the FIND_COORDINATOR its
543    /// `groupId` causes, which plays no part in the lag, and the fake broker
544    /// has no coordinator.
545    async fn set_up_consumer_for_current_lag(broker: &FakeBroker) -> KafkaConsumer<String, String> {
546        let mut consumer = new_consumer(broker);
547        // throws for unassigned partition
548        let error = consumer.current_lag(&tp0()).await.expect_err("unassigned partition");
549        assert!(error.is_local_illegal_state_error(), "{error:?}");
550        assert_eq!(format!("No current assignment for partition {}", tp0()), error.message());
551        consumer.assign(vec![tp0()]).await.expect("assign");
552
553        let deadline = tokio::time::Instant::now() + WAIT;
554        while latest_list_offsets(broker) == 0 {
555            assert!(
556                tokio::time::Instant::now() < deadline,
557                "No LIST_OFFSETS request sent within allotted timeout"
558            );
559            // no error for no current position
560            assert_eq!(None, consumer.current_lag(&tp0()).await.expect("lag"));
561            tokio::time::sleep(Duration::from_millis(10)).await;
562        }
563        consumer
564    }
565
566    /// Waits until the fake broker has received `count` `LATEST_TIMESTAMP`
567    /// requests, polling the consumer in between when `poll` is set.
568    async fn wait_for_latest_list_offsets(
569        consumer: &mut KafkaConsumer<String, String>,
570        broker: &FakeBroker,
571        count: usize,
572        poll: bool,
573    ) {
574        let deadline = tokio::time::Instant::now() + WAIT;
575        while latest_list_offsets(broker) < count {
576            assert!(
577                tokio::time::Instant::now() < deadline,
578                "No LIST_OFFSETS request sent within allotted timeout"
579            );
580            if poll {
581                consumer.poll(Duration::ZERO).await.expect("poll");
582            }
583            tokio::time::sleep(Duration::from_millis(10)).await;
584        }
585    }
586
587    /// Closes the consumer, then checks `currentLag` fails as Java's
588    /// `acquireAndEnsureOpen()` does.
589    async fn close_and_assert_current_lag_fails(mut consumer: KafkaConsumer<String, String>) {
590        consumer
591            .close_with_options(CloseOptions::new_with_timeout(Duration::ZERO))
592            .await
593            .expect("close");
594        let error = consumer.current_lag(&tp0()).await.expect_err("closed consumer");
595        assert!(error.is_local_illegal_state_error(), "{error:?}");
596        assert_eq!("This consumer has already been closed.", error.message());
597    }
598
599    /// Translated from `KafkaConsumerTest.testCurrentLag` (CONSUMER arm),
600    /// through the public API and the background task, against a fake broker
601    /// in place of Java's `MockClient`. The fake answers the FETCHes sent
602    /// before the test's own one empty, with unknown offsets the consumer does
603    /// not apply, so they leave the lag as Java's unanswered FETCH does.
604    #[tokio::test(flavor = "multi_thread")]
605    async fn test_current_lag() {
606        let broker = FakeBroker::start("fake-cluster", TOPIC).await;
607        let mut consumer = set_up_consumer_for_current_lag(&broker).await;
608
609        // no error for no current position
610        assert_eq!(None, consumer.current_lag(&tp0()).await.expect("lag"));
611        // poll once again, which should send the list-offset request
612        consumer.seek_with_offset(tp0(), 50).await.expect("seek");
613        // requests: list-offset, fetch
614        let deadline = tokio::time::Instant::now() + WAIT;
615        while latest_list_offsets(&broker) < 1 || broker.fetch_requests() < 1 {
616            assert!(tokio::time::Instant::now() < deadline, "No list-offset & fetch request sent");
617            consumer.poll(Duration::ZERO).await.expect("poll");
618            tokio::time::sleep(Duration::from_millis(10)).await;
619        }
620
621        // no error for no end offset (so unknown lag)
622        assert_eq!(None, consumer.current_lag(&tp0()).await.expect("lag"));
623
624        // answer the list-offset request, so a next call returns the lag
625        broker.answer_list_offsets(Some(ListOffsetsAnswer { error: Errors::None, offset: 90 }));
626        consumer.poll(Duration::ZERO).await.expect("poll");
627
628        // The subscription state is updated in the background, so the result
629        // will eventually be updated.
630        let deadline = tokio::time::Instant::now() + WAIT;
631        while consumer.current_lag(&tp0()).await.expect("lag") != Some(40) {
632            assert!(tokio::time::Instant::now() < deadline, "Subscription state is not updated");
633            consumer.poll(Duration::ZERO).await.expect("poll");
634            tokio::time::sleep(Duration::from_millis(10)).await;
635        }
636        assert_eq!(1, latest_list_offsets(&broker), "one LIST_OFFSETS for the end offset");
637
638        // one successful fetch should update the log end offset and the position
639        broker.answer_fetch(FetchAnswer { log_first_offset: 1, log_last_offset: 99, offset: 50, count: 5 });
640        // requests: fetch
641        let deadline = tokio::time::Instant::now() + WAIT;
642        let records = loop {
643            assert!(tokio::time::Instant::now() < deadline, "No fetch request sent");
644            let records = consumer.poll(Duration::from_millis(1)).await.expect("poll");
645            if !records.is_empty() {
646                break records;
647            }
648            tokio::time::sleep(Duration::from_millis(10)).await;
649        };
650        assert_eq!(5, records.count());
651        assert_eq!(55, consumer.position(&tp0()).await.expect("position"));
652        assert_eq!(1, records.next_offsets().len());
653        assert_eq!(
654            Some(&OffsetAndMetadata::new(55).expect("valid offset")),
655            records.next_offsets().get(&tp0())
656        );
657
658        // correct lag result
659        assert_eq!(Some(45), consumer.current_lag(&tp0()).await.expect("lag"));
660
661        close_and_assert_current_lag_fails(consumer).await;
662    }
663
664    /// Translated from `KafkaConsumerTest.testCurrentLagPreventsMultipleInFlightRequests`
665    /// (CONSUMER arm, KAFKA-20187).
666    #[tokio::test(flavor = "multi_thread")]
667    async fn test_current_lag_prevents_multiple_in_flight_requests() {
668        let broker = FakeBroker::start("fake-cluster", TOPIC).await;
669        let mut consumer = set_up_consumer_for_current_lag(&broker).await;
670
671        assert_eq!(None, consumer.current_lag(&tp0()).await.expect("lag"));
672        assert_eq!(None, consumer.current_lag(&tp0()).await.expect("lag"));
673        wait_for_latest_list_offsets(&mut consumer, &broker, 1, false).await;
674        // Give the background task time to send a second request, if it did.
675        tokio::time::sleep(Duration::from_millis(200)).await;
676
677        let count = latest_list_offsets(&broker);
678        assert_eq!(
679            1, count,
680            "Expected only one in-flight LIST_OFFSETS request for consumerLag(), but consumer submitted {count} requests"
681        );
682        close_and_assert_current_lag_fails(consumer).await;
683    }
684
685    /// The two `testCurrentLagClearsFlagOn*PartitionError` tests. Java reads
686    /// the 'end offset requested' flag on its spied `SubscriptionState`; here
687    /// it shows through the public API: while it is set, `currentLag` sends no
688    /// new LIST_OFFSETS, and once the error response clears it, the next call
689    /// sends one.
690    async fn assert_current_lag_clears_flag_on_partition_error(error: Errors) {
691        let broker = FakeBroker::start("fake-cluster", TOPIC).await;
692        let mut consumer = set_up_consumer_for_current_lag(&broker).await;
693
694        assert_eq!(None, consumer.current_lag(&tp0()).await.expect("lag"));
695        wait_for_latest_list_offsets(&mut consumer, &broker, 1, false).await;
696
697        // Still set, because the previous LIST_OFFSETS call has not received a
698        // response: no second request.
699        assert_eq!(None, consumer.current_lag(&tp0()).await.expect("lag"));
700        tokio::time::sleep(Duration::from_millis(200)).await;
701        assert_eq!(1, latest_list_offsets(&broker));
702
703        // Now respond to the LIST_OFFSETS request with an error in the partition.
704        broker.answer_list_offsets(Some(ListOffsetsAnswer { error, offset: -1 }));
705
706        // The flag is cleared, so a later call sends a new request.
707        let deadline = tokio::time::Instant::now() + WAIT;
708        while latest_list_offsets(&broker) < 2 {
709            assert!(
710                tokio::time::Instant::now() < deadline,
711                "endOffsetRequested flag was not cleared within allotted timeout"
712            );
713            assert_eq!(None, consumer.current_lag(&tp0()).await.expect("lag"));
714            tokio::time::sleep(Duration::from_millis(10)).await;
715        }
716        close_and_assert_current_lag_fails(consumer).await;
717    }
718
719    /// Translated from `KafkaConsumerTest.testCurrentLagClearsFlagOnFatalPartitionError`
720    /// (CONSUMER arm, KAFKA-20187).
721    #[tokio::test(flavor = "multi_thread")]
722    async fn test_current_lag_clears_flag_on_fatal_partition_error() {
723        assert_current_lag_clears_flag_on_partition_error(Errors::TopicAuthorizationFailed).await;
724    }
725
726    /// Translated from `KafkaConsumerTest.testCurrentLagClearsFlagOnRetriablePartitionError`
727    /// (CONSUMER arm, KAFKA-20187).
728    #[tokio::test(flavor = "multi_thread")]
729    async fn test_current_lag_clears_flag_on_retriable_partition_error() {
730        assert_current_lag_clears_flag_on_partition_error(Errors::OffsetNotAvailable).await;
731    }
732
733    /// Translated from `KafkaConsumerTest.testListOffsetShouldUpdateSubscriptions`
734    /// (CONSUMER arm): `endOffsets` updates the subscription state, so
735    /// `currentLag` returns the lag at once.
736    #[tokio::test(flavor = "multi_thread")]
737    async fn test_list_offset_should_update_subscriptions() {
738        let broker = FakeBroker::start("fake-cluster", TOPIC).await;
739        let mut consumer = new_consumer(&broker);
740        consumer.assign(vec![tp0()]).await.expect("assign");
741        consumer.seek_with_offset(tp0(), 50).await.expect("seek");
742
743        broker.answer_list_offsets(Some(ListOffsetsAnswer { error: Errors::None, offset: 90 }));
744        assert_eq!(
745            HashMap::from([(tp0(), 90)]),
746            consumer.end_offsets(&[tp0()]).await.expect("end offsets")
747        );
748        // correct lag result should be returned as well
749        assert_eq!(Some(40), consumer.current_lag(&tp0()).await.expect("lag"));
750        close_and_assert_current_lag_fails(consumer).await;
751    }
752
753    /// An offsets query completes as soon as its response is read, as Java's
754    /// does inside `client.poll` (`NetworkClientDelegate#poll` →
755    /// `OffsetsRequestManager`'s response handlers). The LIST_OFFSETS forwarder
756    /// must wake the background task; without that wake the completion waits
757    /// for the network poll's own timeout (`MAX_POLL_TIMEOUT_MS`, 5 s) and a
758    /// shorter timeout fails every time.
759    #[tokio::test(flavor = "multi_thread")]
760    async fn test_offsets_query_completes_without_waiting_for_poll_timeout() {
761        let broker = FakeBroker::start("fake-cluster", TOPIC).await;
762        let mut consumer = new_consumer(&broker);
763        consumer.assign(vec![tp0()]).await.expect("assign");
764
765        broker.answer_list_offsets(Some(ListOffsetsAnswer { error: Errors::None, offset: 90 }));
766        assert_eq!(
767            HashMap::from([(tp0(), 90)]),
768            consumer
769                .end_offsets_with_timeout(&[tp0()], Duration::from_secs(1))
770                .await
771                .expect("end offsets")
772        );
773        assert_eq!(
774            HashMap::from([(tp0(), 90)]),
775            consumer
776                .beginning_offsets_with_timeout(&[tp0()], Duration::from_secs(1))
777                .await
778                .expect("beginning offsets")
779        );
780        consumer.close().await.expect("close");
781    }
782
783    /// A manually assigned consumer with a group: the first `poll()` fetches
784    /// the committed offset and then the records.
785    async fn time_to_first_records_after_assign(retry_backoff_ms: Option<&str>) -> Duration {
786        let broker = FakeBroker::start("fake-cluster", TOPIC).await;
787        broker.set_committed_offset(50);
788        broker.answer_fetch(FetchAnswer { log_first_offset: 1, log_last_offset: 99, offset: 50, count: 5 });
789        let mut props = HashMap::from([
790            ("bootstrap.servers".to_string(), broker.bootstrap()),
791            ("group.protocol".to_string(), "consumer".to_string()),
792            ("group.id".to_string(), "first-poll-group".to_string()),
793            ("auto.offset.reset".to_string(), "earliest".to_string()),
794            // The fake does not answer OffsetCommit; keep auto-commit out of
795            // the timing and out of close().
796            ("enable.auto.commit".to_string(), "false".to_string()),
797        ]);
798        if let Some(backoff) = retry_backoff_ms {
799            props.insert("retry.backoff.ms".to_string(), backoff.to_string());
800            props.insert("retry.backoff.max.ms".to_string(), backoff.to_string());
801        }
802        let mut consumer = KafkaConsumer::new(
803            ConsumerConfig::new(&props).expect("config validates"),
804            StringDeserializer,
805            StringDeserializer,
806        )
807        .expect("constructs against the fake broker");
808        consumer.assign(vec![tp0()]).await.expect("assign");
809        // Let the metadata and the coordinator lookup complete, so what is
810        // timed below is the committed-offset fetch and the FETCH.
811        tokio::time::sleep(Duration::from_millis(500)).await;
812
813        let start = std::time::Instant::now();
814        let records = consumer.poll(Duration::from_secs(15)).await.expect("poll");
815        let took = start.elapsed();
816        assert_eq!(1, broker.offset_fetch_requests(), "one OffsetFetch for the committed offset");
817        assert_eq!(
818            5,
819            records.count(),
820            "the first poll() returns the records at the committed offset ({took:?})"
821        );
822        assert_eq!(55, consumer.position(&tp0()).await.expect("position"));
823        consumer
824            .close_with_options(CloseOptions::new_with_timeout(Duration::ZERO))
825            .await
826            .expect("close");
827        took
828    }
829
830    /// After a manual `assign`, the first `poll()` returns records within a few round
831    /// trips, as Java's does: Java's network thread sends the OffsetFetch that
832    /// `updateFetchPositions` queues while processing the `AsyncPollEvent` in
833    /// the same `runOnce`, and the FETCH as soon as the offset is applied
834    /// (`ApplicationEventProcessor.process(AsyncPollEvent)`,
835    /// `OffsetsRequestManager.initWithCommittedOffsetsIfNeeded`,
836    /// `FetchRequestManager.createFetchRequests`). Rust queues the OffsetFetch
837    /// from the spawned AsyncPoll task, so `CommitRequestManager::fetch_offsets`
838    /// must wake the background loop: without that wake the request waited for
839    /// the next unrelated wake, here the application's own wake at the start
840    /// of each `retry.backoff.ms`-bounded wait for the fetch buffer
841    /// (`AsyncKafkaConsumer::poll_for_fetches` → `send_prefetches`), so the
842    /// first `poll()` took one `retry.backoff.ms`.
843    #[tokio::test(flavor = "multi_thread")]
844    async fn first_poll_after_assign_does_not_wait_for_unrelated_wakes() {
845        let with_default = time_to_first_records_after_assign(None).await;
846        let with_slow_backoff = time_to_first_records_after_assign(Some("2000")).await;
847        eprintln!(
848            "time to first records: default retry.backoff.ms {with_default:?}, retry.backoff.ms=2000 {with_slow_backoff:?}"
849        );
850        assert!(
851            with_slow_backoff < Duration::from_secs(1),
852            "with retry.backoff.ms=2000 the first poll() after assign took {with_slow_backoff:?} to return records \
853             (default retry.backoff.ms: {with_default:?}): the OffsetFetch and the FETCH queued by the AsyncPoll \
854             task waited for unrelated wakes of the background loop. Java sends each in the ConsumerNetworkThread \
855             runOnce that queues it, so the time is a few round trips whatever retry.backoff.ms is"
856        );
857    }
858
859    /// A closed and dropped consumer frees its state. The offsets manager's
860    /// cluster listener, boxed into the consumer's `Metadata`, once held the
861    /// manager's shared state, which owns that `Metadata`: an `Arc` cycle that
862    /// kept the metadata, the subscription state and the manager allocated for
863    /// the life of the process. Java builds the same cycle
864    /// (`OffsetsRequestManager#OffsetsRequestManager`) and its garbage collector frees it.
865    #[tokio::test(flavor = "multi_thread")]
866    async fn test_closed_consumer_is_freed() {
867        for query_offsets in [false, true] {
868            let broker = FakeBroker::start("fake-cluster", TOPIC).await;
869            let mut consumer = new_consumer(&broker);
870            let (metadata, subscriptions) = consumer.delegate.metadata_and_subscriptions_for_test();
871            if query_offsets {
872                consumer.assign(vec![tp0()]).await.expect("assign");
873                broker.answer_list_offsets(Some(ListOffsetsAnswer { error: Errors::None, offset: 90 }));
874                consumer.end_offsets(&[tp0()]).await.expect("end offsets");
875            }
876            consumer.close().await.expect("close");
877            drop(consumer);
878
879            // Spawned tasks holding a clone end shortly after the close.
880            let deadline = std::time::Instant::now() + Duration::from_secs(5);
881            while (metadata.strong_count() > 0 || subscriptions.strong_count() > 0)
882                && std::time::Instant::now() < deadline
883            {
884                tokio::time::sleep(Duration::from_millis(10)).await;
885            }
886            assert_eq!(
887                (0, 0),
888                (metadata.strong_count(), subscriptions.strong_count()),
889                "ConsumerMetadata / SubscriptionState still referenced (query_offsets = {query_offsets})"
890            );
891        }
892    }
893
894    /// A closed and dropped group consumer frees its subscription state. The
895    /// rebalance metrics' `assigned-partitions` gauge captures it; before the
896    /// registry's "count" gauge held the registry weakly, the registry kept
897    /// that gauge, and so the state, allocated for the life of the process.
898    /// Java's garbage collector frees the same graph.
899    #[tokio::test(flavor = "multi_thread")]
900    async fn test_closed_group_consumer_is_freed() {
901        let broker = FakeBroker::start("fake-cluster", TOPIC).await;
902        let props = HashMap::from([
903            ("bootstrap.servers".to_string(), broker.bootstrap()),
904            ("group.protocol".to_string(), "consumer".to_string()),
905            ("group.id".to_string(), "freed-group".to_string()),
906        ]);
907        let mut consumer: KafkaConsumer<String, String> = KafkaConsumer::new(
908            ConsumerConfig::new(&props).expect("config validates"),
909            StringDeserializer,
910            StringDeserializer,
911        )
912        .expect("constructs against the fake broker");
913        let (metadata, subscriptions) = consumer.delegate.metadata_and_subscriptions_for_test();
914        consumer
915            .close_with_options(CloseOptions::new_with_timeout(Duration::from_millis(500)))
916            .await
917            .expect("close");
918        drop(consumer);
919
920        // Spawned tasks holding a clone end shortly after the close.
921        let deadline = std::time::Instant::now() + Duration::from_secs(5);
922        while (metadata.strong_count() > 0 || subscriptions.strong_count() > 0) && std::time::Instant::now() < deadline
923        {
924            tokio::time::sleep(Duration::from_millis(10)).await;
925        }
926        assert_eq!(
927            (0, 0),
928            (metadata.strong_count(), subscriptions.strong_count()),
929            "ConsumerMetadata / SubscriptionState still referenced"
930        );
931    }
932
933    /// A listener that captured `consumer.handle()` holds the handle state, and
934    /// through it the `SubscriptionState` that stores the listener. `close()`
935    /// releases the listener once its last callback has run, so a closed and
936    /// dropped consumer frees the listener and the subscription state. A
937    /// consumer dropped without `close()` keeps the cycle.
938    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
939    async fn test_closed_consumer_frees_a_listener_capturing_the_handle() {
940        use std::sync::atomic::{AtomicBool, Ordering};
941
942        use async_trait::async_trait;
943
944        use crate::consumer::{ConsumerHandle, ConsumerRebalanceListener};
945
946        struct Sentinel(Arc<AtomicBool>);
947        impl Drop for Sentinel {
948            fn drop(&mut self) {
949                self.0.store(true, Ordering::SeqCst);
950            }
951        }
952        struct HandleListener {
953            _handle: ConsumerHandle,
954            _sentinel: Sentinel,
955        }
956        #[async_trait]
957        impl ConsumerRebalanceListener for HandleListener {
958            async fn on_partitions_revoked(&self, _partitions: &[TopicPartition]) -> Result<(), Error> {
959                Ok(())
960            }
961            async fn on_partitions_assigned(&self, _partitions: &[TopicPartition]) -> Result<(), Error> {
962                Ok(())
963            }
964        }
965
966        let broker = FakeBroker::start("fake-cluster", TOPIC).await;
967        let props = HashMap::from([
968            ("bootstrap.servers".to_string(), broker.bootstrap()),
969            ("group.protocol".to_string(), "consumer".to_string()),
970            ("group.id".to_string(), "listener-group".to_string()),
971        ]);
972        let mut consumer: KafkaConsumer<String, String> = KafkaConsumer::new(
973            ConsumerConfig::new(&props).expect("config validates"),
974            StringDeserializer,
975            StringDeserializer,
976        )
977        .expect("constructs against the fake broker");
978        let dropped = Arc::new(AtomicBool::new(false));
979        let listener = HandleListener { _handle: consumer.handle(), _sentinel: Sentinel(Arc::clone(&dropped)) };
980        consumer
981            .subscribe_with_topics_listener(vec![TOPIC.to_string()], Arc::new(listener))
982            .await
983            .expect("subscribe");
984        let (_, subscriptions) = consumer.delegate.metadata_and_subscriptions_for_test();
985        consumer
986            .close_with_options(CloseOptions::new_with_timeout(Duration::from_millis(500)))
987            .await
988            .expect("close");
989        drop(consumer);
990
991        // Spawned tasks holding a clone end shortly after the close.
992        let deadline = std::time::Instant::now() + Duration::from_secs(5);
993        while (!dropped.load(Ordering::SeqCst) || subscriptions.strong_count() > 0)
994            && std::time::Instant::now() < deadline
995        {
996            tokio::time::sleep(Duration::from_millis(10)).await;
997        }
998        assert!(dropped.load(Ordering::SeqCst), "listener still referenced");
999        assert_eq!(0, subscriptions.strong_count(), "SubscriptionState still referenced");
1000    }
1001
1002    /// A real `KafkaConsumer` against `broker` with the given extra
1003    /// properties, assigned to `tp0` at offset 0.
1004    async fn new_assigned_consumer(broker: &FakeBroker, extra: &[(&str, &str)]) -> KafkaConsumer<String, String> {
1005        let mut props = HashMap::from([
1006            ("bootstrap.servers".to_string(), broker.bootstrap()),
1007            ("group.protocol".to_string(), "consumer".to_string()),
1008        ]);
1009        props.extend(extra.iter().map(|(k, v)| (k.to_string(), v.to_string())));
1010        let mut consumer: KafkaConsumer<String, String> = KafkaConsumer::new(
1011            ConsumerConfig::new(&props).expect("config validates"),
1012            StringDeserializer,
1013            StringDeserializer,
1014        )
1015        .expect("constructs against the fake broker");
1016        consumer.assign(vec![tp0()]).await.expect("assign");
1017        consumer.seek_with_offset(tp0(), 0).await.expect("seek");
1018        consumer
1019    }
1020
1021    /// Polls `consumer` with `poll(100 ms)` for `window`, returning every
1022    /// error `poll` returned.
1023    async fn poll_for(consumer: &mut KafkaConsumer<String, String>, window: Duration) -> Vec<Error> {
1024        let mut errors = Vec::new();
1025        let until = tokio::time::Instant::now() + window;
1026        while tokio::time::Instant::now() < until {
1027            if let Err(e) = consumer.poll(Duration::from_millis(100)).await {
1028                errors.push(e);
1029            }
1030        }
1031        errors
1032    }
1033
1034    /// Kills the leader once a FETCH has reached it, and waits until the
1035    /// consumer has seen the disconnect (no more fetches are being built).
1036    async fn kill_leader_after_first_fetch(consumer: &mut KafkaConsumer<String, String>, broker: &FakeBroker) {
1037        let deadline = tokio::time::Instant::now() + WAIT;
1038        while broker.fetch_requests() < 1 {
1039            assert!(tokio::time::Instant::now() < deadline, "No FETCH request sent");
1040            let _ = consumer.poll(Duration::from_millis(10)).await;
1041        }
1042        broker.kill();
1043        poll_for(consumer, Duration::from_millis(300)).await;
1044    }
1045
1046    /// Once the partition leader has gone away and its node is inside its
1047    /// reconnect backoff, the consumer must not build fetches for it.
1048    ///
1049    /// Java's `AbstractFetch.prepareFetchRequests` skips a node for which
1050    /// `NetworkClientDelegate.isUnavailable(node)` (`connectionFailed(node) &&
1051    /// connectionDelay(node, now) > 0`) holds, so with a 10 s backoff it builds
1052    /// none in the second measured here. The Rust consumer used to wire
1053    /// `is_unavailable = |_| false`: it built a fetch to the dead node on every
1054    /// background-loop iteration (~400 in this second), each failed at once by
1055    /// the delegate's disconnect check, which woke the loop again.
1056    #[tokio::test(flavor = "multi_thread")]
1057    async fn fetch_skips_dead_leader_in_reconnect_backoff() {
1058        let broker = FakeBroker::start("fake-cluster", TOPIC).await;
1059        let mut consumer = new_assigned_consumer(
1060            &broker,
1061            &[("reconnect.backoff.ms", "10000"), ("reconnect.backoff.max.ms", "10000")],
1062        )
1063        .await;
1064        kill_leader_after_first_fetch(&mut consumer, &broker).await;
1065
1066        let before = consumer
1067            .delegate
1068            .fetch_requests_built_for_test(crate::test_fake_broker::FAKE_BROKER_NODE_ID);
1069        poll_for(&mut consumer, Duration::from_millis(1000)).await;
1070        let built = consumer
1071            .delegate
1072            .fetch_requests_built_for_test(crate::test_fake_broker::FAKE_BROKER_NODE_ID)
1073            - before;
1074        consumer
1075            .close_with_options(CloseOptions::new_with_timeout(Duration::ZERO))
1076            .await
1077            .expect("close");
1078        assert!(
1079            built <= 1,
1080            "{built} FETCH requests were built in 1 s for node 0, which is disconnected and inside its 10 s \
1081             reconnect backoff; Java's AbstractFetch.prepareFetchRequests skips it because \
1082             NetworkClientDelegate.isUnavailable(node) = connectionFailed(node) && connectionDelay(node, now) > 0"
1083        );
1084    }
1085
1086    /// The other side of [`fetch_skips_dead_leader_in_reconnect_backoff`]:
1087    /// with a short reconnect backoff the consumer keeps trying the dead
1088    /// leader, about once per backoff (each fetch it builds starts a
1089    /// reconnect, which is refused and starts the next backoff), rather than
1090    /// giving up on the node or building one per loop iteration.
1091    #[tokio::test(flavor = "multi_thread")]
1092    async fn fetch_retries_dead_leader_once_reconnect_backoff_elapses() {
1093        let broker = FakeBroker::start("fake-cluster", TOPIC).await;
1094        let mut consumer =
1095            new_assigned_consumer(&broker, &[("reconnect.backoff.ms", "200"), ("reconnect.backoff.max.ms", "200")])
1096                .await;
1097        kill_leader_after_first_fetch(&mut consumer, &broker).await;
1098
1099        let before = consumer
1100            .delegate
1101            .fetch_requests_built_for_test(crate::test_fake_broker::FAKE_BROKER_NODE_ID);
1102        poll_for(&mut consumer, Duration::from_millis(2000)).await;
1103        let built = consumer
1104            .delegate
1105            .fetch_requests_built_for_test(crate::test_fake_broker::FAKE_BROKER_NODE_ID)
1106            - before;
1107        consumer
1108            .close_with_options(CloseOptions::new_with_timeout(Duration::ZERO))
1109            .await
1110            .expect("close");
1111        assert!(
1112            (2..=40).contains(&built),
1113            "{built} FETCH requests were built in 2 s for node 0 with a 200 ms reconnect backoff; Java builds about \
1114             one per elapsed backoff (isUnavailable(node) is false once connectionDelay(node, now) is 0)"
1115        );
1116    }
1117
1118    /// The connection to the partition leader fails SASL authentication
1119    /// (metadata came from a bootstrap connection that authenticated), so
1120    /// `poll()` must keep reporting the authentication failure.
1121    ///
1122    /// Java's `AbstractFetch.prepareFetchRequests` calls
1123    /// `maybeThrowAuthFailure(node)` for an unavailable node, which rethrows
1124    /// `NetworkClient.authenticationException(node)`; `createFetchRequests`
1125    /// fails, `ApplicationEventProcessor.process(AsyncPollEvent)` completes the
1126    /// poll event with it and `AsyncKafkaConsumer.poll` throws it. Against this
1127    /// fake broker the 4.3.1 jar threw `UnsupportedSaslMechanismException` from
1128    /// 26 of 55 `poll(100 ms)` calls in 3 s. The Rust consumer used to wire
1129    /// `maybe_auth = |_| Ok(())` and reported it once.
1130    #[tokio::test(flavor = "multi_thread")]
1131    async fn poll_reports_cached_auth_failure_of_leader() {
1132        let broker = FakeBroker::start("fake-cluster", TOPIC).await;
1133        broker.accept_sasl_handshakes(1);
1134        let mut consumer = new_assigned_consumer(
1135            &broker,
1136            &[
1137                ("security.protocol", "SASL_PLAINTEXT"),
1138                ("sasl.mechanism", "PLAIN"),
1139                (
1140                    "sasl.jaas.config",
1141                    "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"u\" password=\"p\";",
1142                ),
1143                ("reconnect.backoff.ms", "10000"),
1144                ("reconnect.backoff.max.ms", "10000"),
1145            ],
1146        )
1147        .await;
1148
1149        let errors = poll_for(&mut consumer, Duration::from_millis(3000)).await;
1150        consumer
1151            .close_with_options(CloseOptions::new_with_timeout(Duration::ZERO))
1152            .await
1153            .expect("close");
1154        println!("poll errors: {} (first: {:?})", errors.len(), errors.first());
1155        assert_eq!(
1156            0,
1157            broker.fetch_requests(),
1158            "the leader never authenticated, so no FETCH reached it"
1159        );
1160        assert!(
1161            errors.iter().all(|e| e.is_authentication_error()),
1162            "every poll error is the leader's authentication failure: {errors:?}"
1163        );
1164        assert!(
1165            errors.len() >= 10,
1166            "poll() reported the leader's cached authentication failure {} times in 3 s of poll(100 ms); Java's \
1167             AbstractFetch.prepareFetchRequests rethrows NetworkClient.authenticationException(node) on every \
1168             fetch round, so 4.3.1 threw it from about every other poll (26 of 55)",
1169            errors.len()
1170        );
1171    }
1172
1173    /// `KafkaConsumer` can move to another task but not be shared between
1174    /// tasks, as Java's `KafkaConsumer` is single-threaded and enforces it at
1175    /// run time with `acquire()` (`consumer-threading.md` §2). The compiler
1176    /// enforces it here, through the `!Sync` marker.
1177    #[test]
1178    fn kafka_consumer_is_send_and_not_sync() {
1179        assert!(is_send!(KafkaConsumer<String, String>));
1180        assert!(!is_sync!(KafkaConsumer<String, String>));
1181    }
1182
1183    /// The handle is the one consumer value that may be shared between tasks:
1184    /// it is what a listener or another task uses to call `wakeup()`.
1185    #[test]
1186    fn consumer_handle_is_send_and_sync() {
1187        assert!(is_send!(ConsumerHandle));
1188        assert!(is_sync!(ConsumerHandle));
1189    }
1190}