1use std::cell::Cell;
20use std::collections::{HashMap, HashSet};
21use std::marker::PhantomData;
22use std::sync::Arc;
23use std::time::Duration;
24
25use crate::common::metrics::MetricsView;
26use crate::common::serialization::{Deserializer, DynDeserializer};
27use crate::common::{Error, PartitionInfo, TopicPartition};
28use crate::consumer::internals::AsyncKafkaConsumer;
29use crate::consumer::{
30 CloseOptions, Consumer, ConsumerConfig, ConsumerGroupMetadata, ConsumerHandle, ConsumerRebalanceListener,
31 ConsumerRecords, GroupProtocol, OffsetAndMetadata, OffsetAndTimestamp, OffsetCommitCallback, SubscriptionPattern,
32};
33
34#[doc(alias = "org.apache.kafka.clients.consumer.KafkaConsumer")]
87pub struct KafkaConsumer<K, V>
88where
89 K: Send + Sync + 'static,
90 V: Send + Sync + 'static,
91{
92 delegate: AsyncKafkaConsumer<K, V>,
93 not_sync: PhantomData<Cell<()>>,
97}
98
99impl<K, V> std::fmt::Debug for KafkaConsumer<K, V>
100where
101 K: Send + Sync + 'static,
102 V: Send + Sync + 'static,
103{
104 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
106 f.debug_struct("KafkaConsumer")
107 .field("delegate", &self.delegate)
108 .finish_non_exhaustive()
109 }
110}
111
112impl<K, V> KafkaConsumer<K, V>
113where
114 K: Send + Sync + 'static,
115 V: Send + Sync + 'static,
116{
117 #[doc(alias = "org.apache.kafka.clients.consumer.KafkaConsumer#KafkaConsumer(Map,Deserializer,Deserializer)")]
158 pub fn new(
159 config: ConsumerConfig,
160 key_deserializer: impl Deserializer<K>,
161 value_deserializer: impl Deserializer<V>,
162 ) -> Result<Self, Error> {
163 Self::with_boxed_deserializers(config, Box::new(key_deserializer), Box::new(value_deserializer))
164 }
165
166 pub(crate) fn with_boxed_deserializers(
173 config: ConsumerConfig,
174 key_deserializer: Box<dyn DynDeserializer<K>>,
175 value_deserializer: Box<dyn DynDeserializer<V>>,
176 ) -> Result<Self, Error> {
177 let protocol = GroupProtocol::of(config.group_protocol())?;
191 match protocol {
192 GroupProtocol::Consumer => Ok(Self {
193 delegate: AsyncKafkaConsumer::<K, V>::new(config, key_deserializer, value_deserializer)?,
194 not_sync: PhantomData,
195 }),
196 GroupProtocol::Classic => Err(Error::local_unsupported_operation(
197 "Classic group protocol is not yet supported in this client; \
198 set group.protocol=consumer (KIP-848).",
199 )),
200 }
201 }
202}
203
204impl<K, V> Consumer<K, V> for KafkaConsumer<K, V>
208where
209 K: Send + Sync + 'static,
210 V: Send + Sync + 'static,
211{
212 fn assignment(&mut self) -> HashSet<TopicPartition> {
213 Consumer::assignment(&mut self.delegate)
214 }
215
216 fn subscription(&mut self) -> HashSet<String> {
217 Consumer::subscription(&mut self.delegate)
218 }
219
220 fn paused(&mut self) -> HashSet<TopicPartition> {
221 Consumer::paused(&mut self.delegate)
222 }
223
224 fn group_metadata(&mut self) -> Arc<dyn ConsumerGroupMetadata> {
225 Consumer::group_metadata(&mut self.delegate)
226 }
227
228 fn current_lag(
229 &mut self,
230 topic_partition: &TopicPartition,
231 ) -> impl Future<Output = Result<Option<i64>, Error>> + Send {
232 Consumer::current_lag(&mut self.delegate, topic_partition)
233 }
234
235 fn metrics(&self) -> MetricsView {
236 Consumer::metrics(&self.delegate)
237 }
238
239 fn subscribe_with_topics(&mut self, topics: Vec<String>) -> impl Future<Output = Result<(), Error>> + Send {
240 Consumer::subscribe_with_topics(&mut self.delegate, topics)
241 }
242
243 fn subscribe_with_topics_listener(
244 &mut self,
245 topics: Vec<String>,
246 listener: Arc<dyn ConsumerRebalanceListener>,
247 ) -> impl Future<Output = Result<(), Error>> + Send {
248 Consumer::subscribe_with_topics_listener(&mut self.delegate, topics, listener)
249 }
250
251 fn subscribe_with_pattern(
252 &mut self,
253 pattern: SubscriptionPattern,
254 ) -> impl Future<Output = Result<(), Error>> + Send {
255 Consumer::subscribe_with_pattern(&mut self.delegate, pattern)
256 }
257
258 fn subscribe_with_pattern_listener(
259 &mut self,
260 pattern: SubscriptionPattern,
261 listener: Arc<dyn ConsumerRebalanceListener>,
262 ) -> impl Future<Output = Result<(), Error>> + Send {
263 Consumer::subscribe_with_pattern_listener(&mut self.delegate, pattern, listener)
264 }
265
266 fn assign(&mut self, partitions: Vec<TopicPartition>) -> impl Future<Output = Result<(), Error>> + Send {
267 Consumer::assign(&mut self.delegate, partitions)
268 }
269
270 fn unsubscribe(&mut self) -> impl Future<Output = Result<(), Error>> + Send {
271 Consumer::unsubscribe(&mut self.delegate)
272 }
273
274 fn poll(&mut self, timeout: Duration) -> impl Future<Output = Result<ConsumerRecords<K, V>, Error>> + Send {
275 Consumer::poll(&mut self.delegate, timeout)
276 }
277
278 fn commit_sync(&mut self) -> impl Future<Output = Result<(), Error>> + Send {
279 Consumer::commit_sync(&mut self.delegate)
280 }
281
282 fn commit_sync_with_timeout(&mut self, timeout: Duration) -> impl Future<Output = Result<(), Error>> + Send {
283 Consumer::commit_sync_with_timeout(&mut self.delegate, timeout)
284 }
285
286 fn commit_sync_with_offsets(
287 &mut self,
288 offsets: HashMap<TopicPartition, OffsetAndMetadata>,
289 ) -> impl Future<Output = Result<(), Error>> + Send {
290 Consumer::commit_sync_with_offsets(&mut self.delegate, offsets)
291 }
292
293 fn commit_sync_with_offsets_timeout(
294 &mut self,
295 offsets: HashMap<TopicPartition, OffsetAndMetadata>,
296 timeout: Duration,
297 ) -> impl Future<Output = Result<(), Error>> + Send {
298 Consumer::commit_sync_with_offsets_timeout(&mut self.delegate, offsets, timeout)
299 }
300
301 fn commit_async(&mut self) -> impl Future<Output = Result<(), Error>> + Send {
302 Consumer::commit_async(&mut self.delegate)
303 }
304
305 fn commit_async_with_callback(
306 &mut self,
307 callback: Arc<dyn OffsetCommitCallback>,
308 ) -> impl Future<Output = Result<(), Error>> + Send {
309 Consumer::commit_async_with_callback(&mut self.delegate, callback)
310 }
311
312 fn commit_async_with_offsets_callback(
313 &mut self,
314 offsets: HashMap<TopicPartition, OffsetAndMetadata>,
315 callback: Arc<dyn OffsetCommitCallback>,
316 ) -> impl Future<Output = Result<(), Error>> + Send {
317 Consumer::commit_async_with_offsets_callback(&mut self.delegate, offsets, callback)
318 }
319
320 fn seek_with_offset(
321 &mut self,
322 partition: TopicPartition,
323 offset: i64,
324 ) -> impl Future<Output = Result<(), Error>> + Send {
325 Consumer::seek_with_offset(&mut self.delegate, partition, offset)
326 }
327
328 fn seek_with_offset_and_metadata(
329 &mut self,
330 partition: TopicPartition,
331 offset_and_metadata: OffsetAndMetadata,
332 ) -> impl Future<Output = Result<(), Error>> + Send {
333 Consumer::seek_with_offset_and_metadata(&mut self.delegate, partition, offset_and_metadata)
334 }
335
336 fn seek_to_beginning(&mut self, partitions: &[TopicPartition]) -> impl Future<Output = Result<(), Error>> + Send {
337 Consumer::seek_to_beginning(&mut self.delegate, partitions)
338 }
339
340 fn seek_to_end(&mut self, partitions: &[TopicPartition]) -> impl Future<Output = Result<(), Error>> + Send {
341 Consumer::seek_to_end(&mut self.delegate, partitions)
342 }
343
344 fn position(&mut self, partition: &TopicPartition) -> impl Future<Output = Result<i64, Error>> + Send {
345 Consumer::position(&mut self.delegate, partition)
346 }
347
348 fn position_with_timeout(
349 &mut self,
350 partition: &TopicPartition,
351 timeout: Duration,
352 ) -> impl Future<Output = Result<i64, Error>> + Send {
353 Consumer::position_with_timeout(&mut self.delegate, partition, timeout)
354 }
355
356 fn committed(
357 &mut self,
358 partitions: &[TopicPartition],
359 ) -> impl Future<Output = Result<HashMap<TopicPartition, Option<OffsetAndMetadata>>, Error>> + Send {
360 Consumer::committed(&mut self.delegate, partitions)
361 }
362
363 fn committed_with_timeout(
364 &mut self,
365 partitions: &[TopicPartition],
366 timeout: Duration,
367 ) -> impl Future<Output = Result<HashMap<TopicPartition, Option<OffsetAndMetadata>>, Error>> + Send {
368 Consumer::committed_with_timeout(&mut self.delegate, partitions, timeout)
369 }
370
371 fn partitions_for(&mut self, topic: &str) -> impl Future<Output = Result<Vec<PartitionInfo>, Error>> + Send {
372 Consumer::partitions_for(&mut self.delegate, topic)
373 }
374
375 fn partitions_for_with_timeout(
376 &mut self,
377 topic: &str,
378 timeout: Duration,
379 ) -> impl Future<Output = Result<Vec<PartitionInfo>, Error>> + Send {
380 Consumer::partitions_for_with_timeout(&mut self.delegate, topic, timeout)
381 }
382
383 fn list_topics(&mut self) -> impl Future<Output = Result<HashMap<String, Vec<PartitionInfo>>, Error>> + Send {
384 Consumer::list_topics(&mut self.delegate)
385 }
386
387 fn list_topics_with_timeout(
388 &mut self,
389 timeout: Duration,
390 ) -> impl Future<Output = Result<HashMap<String, Vec<PartitionInfo>>, Error>> + Send {
391 Consumer::list_topics_with_timeout(&mut self.delegate, timeout)
392 }
393
394 fn offsets_for_times(
395 &mut self,
396 timestamps_to_search: HashMap<TopicPartition, i64>,
397 ) -> impl Future<Output = Result<HashMap<TopicPartition, Option<OffsetAndTimestamp>>, Error>> + Send {
398 Consumer::offsets_for_times(&mut self.delegate, timestamps_to_search)
399 }
400
401 fn offsets_for_times_with_timeout(
402 &mut self,
403 timestamps_to_search: HashMap<TopicPartition, i64>,
404 timeout: Duration,
405 ) -> impl Future<Output = Result<HashMap<TopicPartition, Option<OffsetAndTimestamp>>, Error>> + Send {
406 Consumer::offsets_for_times_with_timeout(&mut self.delegate, timestamps_to_search, timeout)
407 }
408
409 fn beginning_offsets(
410 &mut self,
411 partitions: &[TopicPartition],
412 ) -> impl Future<Output = Result<HashMap<TopicPartition, i64>, Error>> + Send {
413 Consumer::beginning_offsets(&mut self.delegate, partitions)
414 }
415
416 fn beginning_offsets_with_timeout(
417 &mut self,
418 partitions: &[TopicPartition],
419 timeout: Duration,
420 ) -> impl Future<Output = Result<HashMap<TopicPartition, i64>, Error>> + Send {
421 Consumer::beginning_offsets_with_timeout(&mut self.delegate, partitions, timeout)
422 }
423
424 fn end_offsets(
425 &mut self,
426 partitions: &[TopicPartition],
427 ) -> impl Future<Output = Result<HashMap<TopicPartition, i64>, Error>> + Send {
428 Consumer::end_offsets(&mut self.delegate, partitions)
429 }
430
431 fn end_offsets_with_timeout(
432 &mut self,
433 partitions: &[TopicPartition],
434 timeout: Duration,
435 ) -> impl Future<Output = Result<HashMap<TopicPartition, i64>, Error>> + Send {
436 Consumer::end_offsets_with_timeout(&mut self.delegate, partitions, timeout)
437 }
438
439 fn pause(&mut self, partitions: &[TopicPartition]) -> impl Future<Output = Result<(), Error>> + Send {
440 Consumer::pause(&mut self.delegate, partitions)
441 }
442
443 fn resume(&mut self, partitions: &[TopicPartition]) -> impl Future<Output = Result<(), Error>> + Send {
444 Consumer::resume(&mut self.delegate, partitions)
445 }
446
447 fn enforce_rebalance(&mut self) -> impl Future<Output = Result<(), Error>> + Send {
448 Consumer::enforce_rebalance(&mut self.delegate)
449 }
450
451 fn enforce_rebalance_with_reason(&mut self, reason: &str) -> impl Future<Output = Result<(), Error>> + Send {
452 Consumer::enforce_rebalance_with_reason(&mut self.delegate, reason)
453 }
454
455 fn close(&mut self) -> impl Future<Output = Result<(), Error>> + Send {
456 Consumer::close(&mut self.delegate)
457 }
458
459 fn close_with_options(&mut self, options: CloseOptions) -> impl Future<Output = Result<(), Error>> + Send {
460 Consumer::close_with_options(&mut self.delegate, options)
461 }
462
463 fn wakeup(&self) {
464 Consumer::wakeup(&self.delegate)
465 }
466
467 fn handle(&self) -> ConsumerHandle {
468 Consumer::handle(&self.delegate)
469 }
470}
471
472#[cfg(test)]
473mod tests {
474 use super::*;
475 use crate::common::protocol::Errors;
476 use crate::common::serialization::Deserializer;
477 use crate::consumer::CloseOptions;
478 use crate::test_fake_broker::{FakeBroker, FetchAnswer, ListOffsetsAnswer};
479 use crate::test_thread_safety::{is_send, is_sync};
480
481 const TOPIC: &str = "current-lag-topic";
482
483 const LATEST_TIMESTAMP: i64 = -1;
486
487 const WAIT: Duration = Duration::from_secs(15);
490
491 struct StringDeserializer;
492
493 impl Deserializer<String> for StringDeserializer {
494 async fn deserialize(&self, _topic: &str, data: &[u8]) -> Result<String, Error> {
495 String::from_utf8(data.to_vec()).map_err(|e| Error::serialization(format!("invalid utf-8: {e}")))
496 }
497 }
498
499 fn tp0() -> TopicPartition {
500 TopicPartition::new(TOPIC, 0)
501 }
502
503 fn latest_list_offsets(broker: &FakeBroker) -> usize {
504 broker
505 .list_offsets_timestamps()
506 .into_iter()
507 .filter(|timestamp| *timestamp == LATEST_TIMESTAMP)
508 .count()
509 }
510
511 fn new_consumer(broker: &FakeBroker) -> KafkaConsumer<String, String> {
515 let props = HashMap::from([
516 ("bootstrap.servers".to_string(), broker.bootstrap()),
517 ("group.protocol".to_string(), "consumer".to_string()),
518 ("auto.offset.reset".to_string(), "earliest".to_string()),
519 ("isolation.level".to_string(), "read_uncommitted".to_string()),
521 ]);
522 KafkaConsumer::new(
523 ConsumerConfig::new(&props).expect("config validates"),
524 StringDeserializer,
525 StringDeserializer,
526 )
527 .expect("constructs against the fake broker")
528 }
529
530 async fn set_up_consumer_for_current_lag(broker: &FakeBroker) -> KafkaConsumer<String, String> {
546 let mut consumer = new_consumer(broker);
547 let error = consumer.current_lag(&tp0()).await.expect_err("unassigned partition");
549 assert!(error.is_local_illegal_state_error(), "{error:?}");
550 assert_eq!(format!("No current assignment for partition {}", tp0()), error.message());
551 consumer.assign(vec![tp0()]).await.expect("assign");
552
553 let deadline = tokio::time::Instant::now() + WAIT;
554 while latest_list_offsets(broker) == 0 {
555 assert!(
556 tokio::time::Instant::now() < deadline,
557 "No LIST_OFFSETS request sent within allotted timeout"
558 );
559 assert_eq!(None, consumer.current_lag(&tp0()).await.expect("lag"));
561 tokio::time::sleep(Duration::from_millis(10)).await;
562 }
563 consumer
564 }
565
566 async fn wait_for_latest_list_offsets(
569 consumer: &mut KafkaConsumer<String, String>,
570 broker: &FakeBroker,
571 count: usize,
572 poll: bool,
573 ) {
574 let deadline = tokio::time::Instant::now() + WAIT;
575 while latest_list_offsets(broker) < count {
576 assert!(
577 tokio::time::Instant::now() < deadline,
578 "No LIST_OFFSETS request sent within allotted timeout"
579 );
580 if poll {
581 consumer.poll(Duration::ZERO).await.expect("poll");
582 }
583 tokio::time::sleep(Duration::from_millis(10)).await;
584 }
585 }
586
587 async fn close_and_assert_current_lag_fails(mut consumer: KafkaConsumer<String, String>) {
590 consumer
591 .close_with_options(CloseOptions::new_with_timeout(Duration::ZERO))
592 .await
593 .expect("close");
594 let error = consumer.current_lag(&tp0()).await.expect_err("closed consumer");
595 assert!(error.is_local_illegal_state_error(), "{error:?}");
596 assert_eq!("This consumer has already been closed.", error.message());
597 }
598
599 #[tokio::test(flavor = "multi_thread")]
605 async fn test_current_lag() {
606 let broker = FakeBroker::start("fake-cluster", TOPIC).await;
607 let mut consumer = set_up_consumer_for_current_lag(&broker).await;
608
609 assert_eq!(None, consumer.current_lag(&tp0()).await.expect("lag"));
611 consumer.seek_with_offset(tp0(), 50).await.expect("seek");
613 let deadline = tokio::time::Instant::now() + WAIT;
615 while latest_list_offsets(&broker) < 1 || broker.fetch_requests() < 1 {
616 assert!(tokio::time::Instant::now() < deadline, "No list-offset & fetch request sent");
617 consumer.poll(Duration::ZERO).await.expect("poll");
618 tokio::time::sleep(Duration::from_millis(10)).await;
619 }
620
621 assert_eq!(None, consumer.current_lag(&tp0()).await.expect("lag"));
623
624 broker.answer_list_offsets(Some(ListOffsetsAnswer { error: Errors::None, offset: 90 }));
626 consumer.poll(Duration::ZERO).await.expect("poll");
627
628 let deadline = tokio::time::Instant::now() + WAIT;
631 while consumer.current_lag(&tp0()).await.expect("lag") != Some(40) {
632 assert!(tokio::time::Instant::now() < deadline, "Subscription state is not updated");
633 consumer.poll(Duration::ZERO).await.expect("poll");
634 tokio::time::sleep(Duration::from_millis(10)).await;
635 }
636 assert_eq!(1, latest_list_offsets(&broker), "one LIST_OFFSETS for the end offset");
637
638 broker.answer_fetch(FetchAnswer { log_first_offset: 1, log_last_offset: 99, offset: 50, count: 5 });
640 let deadline = tokio::time::Instant::now() + WAIT;
642 let records = loop {
643 assert!(tokio::time::Instant::now() < deadline, "No fetch request sent");
644 let records = consumer.poll(Duration::from_millis(1)).await.expect("poll");
645 if !records.is_empty() {
646 break records;
647 }
648 tokio::time::sleep(Duration::from_millis(10)).await;
649 };
650 assert_eq!(5, records.count());
651 assert_eq!(55, consumer.position(&tp0()).await.expect("position"));
652 assert_eq!(1, records.next_offsets().len());
653 assert_eq!(
654 Some(&OffsetAndMetadata::new(55).expect("valid offset")),
655 records.next_offsets().get(&tp0())
656 );
657
658 assert_eq!(Some(45), consumer.current_lag(&tp0()).await.expect("lag"));
660
661 close_and_assert_current_lag_fails(consumer).await;
662 }
663
664 #[tokio::test(flavor = "multi_thread")]
667 async fn test_current_lag_prevents_multiple_in_flight_requests() {
668 let broker = FakeBroker::start("fake-cluster", TOPIC).await;
669 let mut consumer = set_up_consumer_for_current_lag(&broker).await;
670
671 assert_eq!(None, consumer.current_lag(&tp0()).await.expect("lag"));
672 assert_eq!(None, consumer.current_lag(&tp0()).await.expect("lag"));
673 wait_for_latest_list_offsets(&mut consumer, &broker, 1, false).await;
674 tokio::time::sleep(Duration::from_millis(200)).await;
676
677 let count = latest_list_offsets(&broker);
678 assert_eq!(
679 1, count,
680 "Expected only one in-flight LIST_OFFSETS request for consumerLag(), but consumer submitted {count} requests"
681 );
682 close_and_assert_current_lag_fails(consumer).await;
683 }
684
685 async fn assert_current_lag_clears_flag_on_partition_error(error: Errors) {
691 let broker = FakeBroker::start("fake-cluster", TOPIC).await;
692 let mut consumer = set_up_consumer_for_current_lag(&broker).await;
693
694 assert_eq!(None, consumer.current_lag(&tp0()).await.expect("lag"));
695 wait_for_latest_list_offsets(&mut consumer, &broker, 1, false).await;
696
697 assert_eq!(None, consumer.current_lag(&tp0()).await.expect("lag"));
700 tokio::time::sleep(Duration::from_millis(200)).await;
701 assert_eq!(1, latest_list_offsets(&broker));
702
703 broker.answer_list_offsets(Some(ListOffsetsAnswer { error, offset: -1 }));
705
706 let deadline = tokio::time::Instant::now() + WAIT;
708 while latest_list_offsets(&broker) < 2 {
709 assert!(
710 tokio::time::Instant::now() < deadline,
711 "endOffsetRequested flag was not cleared within allotted timeout"
712 );
713 assert_eq!(None, consumer.current_lag(&tp0()).await.expect("lag"));
714 tokio::time::sleep(Duration::from_millis(10)).await;
715 }
716 close_and_assert_current_lag_fails(consumer).await;
717 }
718
719 #[tokio::test(flavor = "multi_thread")]
722 async fn test_current_lag_clears_flag_on_fatal_partition_error() {
723 assert_current_lag_clears_flag_on_partition_error(Errors::TopicAuthorizationFailed).await;
724 }
725
726 #[tokio::test(flavor = "multi_thread")]
729 async fn test_current_lag_clears_flag_on_retriable_partition_error() {
730 assert_current_lag_clears_flag_on_partition_error(Errors::OffsetNotAvailable).await;
731 }
732
733 #[tokio::test(flavor = "multi_thread")]
737 async fn test_list_offset_should_update_subscriptions() {
738 let broker = FakeBroker::start("fake-cluster", TOPIC).await;
739 let mut consumer = new_consumer(&broker);
740 consumer.assign(vec![tp0()]).await.expect("assign");
741 consumer.seek_with_offset(tp0(), 50).await.expect("seek");
742
743 broker.answer_list_offsets(Some(ListOffsetsAnswer { error: Errors::None, offset: 90 }));
744 assert_eq!(
745 HashMap::from([(tp0(), 90)]),
746 consumer.end_offsets(&[tp0()]).await.expect("end offsets")
747 );
748 assert_eq!(Some(40), consumer.current_lag(&tp0()).await.expect("lag"));
750 close_and_assert_current_lag_fails(consumer).await;
751 }
752
753 #[tokio::test(flavor = "multi_thread")]
760 async fn test_offsets_query_completes_without_waiting_for_poll_timeout() {
761 let broker = FakeBroker::start("fake-cluster", TOPIC).await;
762 let mut consumer = new_consumer(&broker);
763 consumer.assign(vec![tp0()]).await.expect("assign");
764
765 broker.answer_list_offsets(Some(ListOffsetsAnswer { error: Errors::None, offset: 90 }));
766 assert_eq!(
767 HashMap::from([(tp0(), 90)]),
768 consumer
769 .end_offsets_with_timeout(&[tp0()], Duration::from_secs(1))
770 .await
771 .expect("end offsets")
772 );
773 assert_eq!(
774 HashMap::from([(tp0(), 90)]),
775 consumer
776 .beginning_offsets_with_timeout(&[tp0()], Duration::from_secs(1))
777 .await
778 .expect("beginning offsets")
779 );
780 consumer.close().await.expect("close");
781 }
782
783 async fn time_to_first_records_after_assign(retry_backoff_ms: Option<&str>) -> Duration {
786 let broker = FakeBroker::start("fake-cluster", TOPIC).await;
787 broker.set_committed_offset(50);
788 broker.answer_fetch(FetchAnswer { log_first_offset: 1, log_last_offset: 99, offset: 50, count: 5 });
789 let mut props = HashMap::from([
790 ("bootstrap.servers".to_string(), broker.bootstrap()),
791 ("group.protocol".to_string(), "consumer".to_string()),
792 ("group.id".to_string(), "first-poll-group".to_string()),
793 ("auto.offset.reset".to_string(), "earliest".to_string()),
794 ("enable.auto.commit".to_string(), "false".to_string()),
797 ]);
798 if let Some(backoff) = retry_backoff_ms {
799 props.insert("retry.backoff.ms".to_string(), backoff.to_string());
800 props.insert("retry.backoff.max.ms".to_string(), backoff.to_string());
801 }
802 let mut consumer = KafkaConsumer::new(
803 ConsumerConfig::new(&props).expect("config validates"),
804 StringDeserializer,
805 StringDeserializer,
806 )
807 .expect("constructs against the fake broker");
808 consumer.assign(vec![tp0()]).await.expect("assign");
809 tokio::time::sleep(Duration::from_millis(500)).await;
812
813 let start = std::time::Instant::now();
814 let records = consumer.poll(Duration::from_secs(15)).await.expect("poll");
815 let took = start.elapsed();
816 assert_eq!(1, broker.offset_fetch_requests(), "one OffsetFetch for the committed offset");
817 assert_eq!(
818 5,
819 records.count(),
820 "the first poll() returns the records at the committed offset ({took:?})"
821 );
822 assert_eq!(55, consumer.position(&tp0()).await.expect("position"));
823 consumer
824 .close_with_options(CloseOptions::new_with_timeout(Duration::ZERO))
825 .await
826 .expect("close");
827 took
828 }
829
830 #[tokio::test(flavor = "multi_thread")]
844 async fn first_poll_after_assign_does_not_wait_for_unrelated_wakes() {
845 let with_default = time_to_first_records_after_assign(None).await;
846 let with_slow_backoff = time_to_first_records_after_assign(Some("2000")).await;
847 eprintln!(
848 "time to first records: default retry.backoff.ms {with_default:?}, retry.backoff.ms=2000 {with_slow_backoff:?}"
849 );
850 assert!(
851 with_slow_backoff < Duration::from_secs(1),
852 "with retry.backoff.ms=2000 the first poll() after assign took {with_slow_backoff:?} to return records \
853 (default retry.backoff.ms: {with_default:?}): the OffsetFetch and the FETCH queued by the AsyncPoll \
854 task waited for unrelated wakes of the background loop. Java sends each in the ConsumerNetworkThread \
855 runOnce that queues it, so the time is a few round trips whatever retry.backoff.ms is"
856 );
857 }
858
859 #[tokio::test(flavor = "multi_thread")]
866 async fn test_closed_consumer_is_freed() {
867 for query_offsets in [false, true] {
868 let broker = FakeBroker::start("fake-cluster", TOPIC).await;
869 let mut consumer = new_consumer(&broker);
870 let (metadata, subscriptions) = consumer.delegate.metadata_and_subscriptions_for_test();
871 if query_offsets {
872 consumer.assign(vec![tp0()]).await.expect("assign");
873 broker.answer_list_offsets(Some(ListOffsetsAnswer { error: Errors::None, offset: 90 }));
874 consumer.end_offsets(&[tp0()]).await.expect("end offsets");
875 }
876 consumer.close().await.expect("close");
877 drop(consumer);
878
879 let deadline = std::time::Instant::now() + Duration::from_secs(5);
881 while (metadata.strong_count() > 0 || subscriptions.strong_count() > 0)
882 && std::time::Instant::now() < deadline
883 {
884 tokio::time::sleep(Duration::from_millis(10)).await;
885 }
886 assert_eq!(
887 (0, 0),
888 (metadata.strong_count(), subscriptions.strong_count()),
889 "ConsumerMetadata / SubscriptionState still referenced (query_offsets = {query_offsets})"
890 );
891 }
892 }
893
894 #[tokio::test(flavor = "multi_thread")]
900 async fn test_closed_group_consumer_is_freed() {
901 let broker = FakeBroker::start("fake-cluster", TOPIC).await;
902 let props = HashMap::from([
903 ("bootstrap.servers".to_string(), broker.bootstrap()),
904 ("group.protocol".to_string(), "consumer".to_string()),
905 ("group.id".to_string(), "freed-group".to_string()),
906 ]);
907 let mut consumer: KafkaConsumer<String, String> = KafkaConsumer::new(
908 ConsumerConfig::new(&props).expect("config validates"),
909 StringDeserializer,
910 StringDeserializer,
911 )
912 .expect("constructs against the fake broker");
913 let (metadata, subscriptions) = consumer.delegate.metadata_and_subscriptions_for_test();
914 consumer
915 .close_with_options(CloseOptions::new_with_timeout(Duration::from_millis(500)))
916 .await
917 .expect("close");
918 drop(consumer);
919
920 let deadline = std::time::Instant::now() + Duration::from_secs(5);
922 while (metadata.strong_count() > 0 || subscriptions.strong_count() > 0) && std::time::Instant::now() < deadline
923 {
924 tokio::time::sleep(Duration::from_millis(10)).await;
925 }
926 assert_eq!(
927 (0, 0),
928 (metadata.strong_count(), subscriptions.strong_count()),
929 "ConsumerMetadata / SubscriptionState still referenced"
930 );
931 }
932
933 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
939 async fn test_closed_consumer_frees_a_listener_capturing_the_handle() {
940 use std::sync::atomic::{AtomicBool, Ordering};
941
942 use async_trait::async_trait;
943
944 use crate::consumer::{ConsumerHandle, ConsumerRebalanceListener};
945
946 struct Sentinel(Arc<AtomicBool>);
947 impl Drop for Sentinel {
948 fn drop(&mut self) {
949 self.0.store(true, Ordering::SeqCst);
950 }
951 }
952 struct HandleListener {
953 _handle: ConsumerHandle,
954 _sentinel: Sentinel,
955 }
956 #[async_trait]
957 impl ConsumerRebalanceListener for HandleListener {
958 async fn on_partitions_revoked(&self, _partitions: &[TopicPartition]) -> Result<(), Error> {
959 Ok(())
960 }
961 async fn on_partitions_assigned(&self, _partitions: &[TopicPartition]) -> Result<(), Error> {
962 Ok(())
963 }
964 }
965
966 let broker = FakeBroker::start("fake-cluster", TOPIC).await;
967 let props = HashMap::from([
968 ("bootstrap.servers".to_string(), broker.bootstrap()),
969 ("group.protocol".to_string(), "consumer".to_string()),
970 ("group.id".to_string(), "listener-group".to_string()),
971 ]);
972 let mut consumer: KafkaConsumer<String, String> = KafkaConsumer::new(
973 ConsumerConfig::new(&props).expect("config validates"),
974 StringDeserializer,
975 StringDeserializer,
976 )
977 .expect("constructs against the fake broker");
978 let dropped = Arc::new(AtomicBool::new(false));
979 let listener = HandleListener { _handle: consumer.handle(), _sentinel: Sentinel(Arc::clone(&dropped)) };
980 consumer
981 .subscribe_with_topics_listener(vec![TOPIC.to_string()], Arc::new(listener))
982 .await
983 .expect("subscribe");
984 let (_, subscriptions) = consumer.delegate.metadata_and_subscriptions_for_test();
985 consumer
986 .close_with_options(CloseOptions::new_with_timeout(Duration::from_millis(500)))
987 .await
988 .expect("close");
989 drop(consumer);
990
991 let deadline = std::time::Instant::now() + Duration::from_secs(5);
993 while (!dropped.load(Ordering::SeqCst) || subscriptions.strong_count() > 0)
994 && std::time::Instant::now() < deadline
995 {
996 tokio::time::sleep(Duration::from_millis(10)).await;
997 }
998 assert!(dropped.load(Ordering::SeqCst), "listener still referenced");
999 assert_eq!(0, subscriptions.strong_count(), "SubscriptionState still referenced");
1000 }
1001
1002 async fn new_assigned_consumer(broker: &FakeBroker, extra: &[(&str, &str)]) -> KafkaConsumer<String, String> {
1005 let mut props = HashMap::from([
1006 ("bootstrap.servers".to_string(), broker.bootstrap()),
1007 ("group.protocol".to_string(), "consumer".to_string()),
1008 ]);
1009 props.extend(extra.iter().map(|(k, v)| (k.to_string(), v.to_string())));
1010 let mut consumer: KafkaConsumer<String, String> = KafkaConsumer::new(
1011 ConsumerConfig::new(&props).expect("config validates"),
1012 StringDeserializer,
1013 StringDeserializer,
1014 )
1015 .expect("constructs against the fake broker");
1016 consumer.assign(vec![tp0()]).await.expect("assign");
1017 consumer.seek_with_offset(tp0(), 0).await.expect("seek");
1018 consumer
1019 }
1020
1021 async fn poll_for(consumer: &mut KafkaConsumer<String, String>, window: Duration) -> Vec<Error> {
1024 let mut errors = Vec::new();
1025 let until = tokio::time::Instant::now() + window;
1026 while tokio::time::Instant::now() < until {
1027 if let Err(e) = consumer.poll(Duration::from_millis(100)).await {
1028 errors.push(e);
1029 }
1030 }
1031 errors
1032 }
1033
1034 async fn kill_leader_after_first_fetch(consumer: &mut KafkaConsumer<String, String>, broker: &FakeBroker) {
1037 let deadline = tokio::time::Instant::now() + WAIT;
1038 while broker.fetch_requests() < 1 {
1039 assert!(tokio::time::Instant::now() < deadline, "No FETCH request sent");
1040 let _ = consumer.poll(Duration::from_millis(10)).await;
1041 }
1042 broker.kill();
1043 poll_for(consumer, Duration::from_millis(300)).await;
1044 }
1045
1046 #[tokio::test(flavor = "multi_thread")]
1057 async fn fetch_skips_dead_leader_in_reconnect_backoff() {
1058 let broker = FakeBroker::start("fake-cluster", TOPIC).await;
1059 let mut consumer = new_assigned_consumer(
1060 &broker,
1061 &[("reconnect.backoff.ms", "10000"), ("reconnect.backoff.max.ms", "10000")],
1062 )
1063 .await;
1064 kill_leader_after_first_fetch(&mut consumer, &broker).await;
1065
1066 let before = consumer
1067 .delegate
1068 .fetch_requests_built_for_test(crate::test_fake_broker::FAKE_BROKER_NODE_ID);
1069 poll_for(&mut consumer, Duration::from_millis(1000)).await;
1070 let built = consumer
1071 .delegate
1072 .fetch_requests_built_for_test(crate::test_fake_broker::FAKE_BROKER_NODE_ID)
1073 - before;
1074 consumer
1075 .close_with_options(CloseOptions::new_with_timeout(Duration::ZERO))
1076 .await
1077 .expect("close");
1078 assert!(
1079 built <= 1,
1080 "{built} FETCH requests were built in 1 s for node 0, which is disconnected and inside its 10 s \
1081 reconnect backoff; Java's AbstractFetch.prepareFetchRequests skips it because \
1082 NetworkClientDelegate.isUnavailable(node) = connectionFailed(node) && connectionDelay(node, now) > 0"
1083 );
1084 }
1085
1086 #[tokio::test(flavor = "multi_thread")]
1092 async fn fetch_retries_dead_leader_once_reconnect_backoff_elapses() {
1093 let broker = FakeBroker::start("fake-cluster", TOPIC).await;
1094 let mut consumer =
1095 new_assigned_consumer(&broker, &[("reconnect.backoff.ms", "200"), ("reconnect.backoff.max.ms", "200")])
1096 .await;
1097 kill_leader_after_first_fetch(&mut consumer, &broker).await;
1098
1099 let before = consumer
1100 .delegate
1101 .fetch_requests_built_for_test(crate::test_fake_broker::FAKE_BROKER_NODE_ID);
1102 poll_for(&mut consumer, Duration::from_millis(2000)).await;
1103 let built = consumer
1104 .delegate
1105 .fetch_requests_built_for_test(crate::test_fake_broker::FAKE_BROKER_NODE_ID)
1106 - before;
1107 consumer
1108 .close_with_options(CloseOptions::new_with_timeout(Duration::ZERO))
1109 .await
1110 .expect("close");
1111 assert!(
1112 (2..=40).contains(&built),
1113 "{built} FETCH requests were built in 2 s for node 0 with a 200 ms reconnect backoff; Java builds about \
1114 one per elapsed backoff (isUnavailable(node) is false once connectionDelay(node, now) is 0)"
1115 );
1116 }
1117
1118 #[tokio::test(flavor = "multi_thread")]
1131 async fn poll_reports_cached_auth_failure_of_leader() {
1132 let broker = FakeBroker::start("fake-cluster", TOPIC).await;
1133 broker.accept_sasl_handshakes(1);
1134 let mut consumer = new_assigned_consumer(
1135 &broker,
1136 &[
1137 ("security.protocol", "SASL_PLAINTEXT"),
1138 ("sasl.mechanism", "PLAIN"),
1139 (
1140 "sasl.jaas.config",
1141 "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"u\" password=\"p\";",
1142 ),
1143 ("reconnect.backoff.ms", "10000"),
1144 ("reconnect.backoff.max.ms", "10000"),
1145 ],
1146 )
1147 .await;
1148
1149 let errors = poll_for(&mut consumer, Duration::from_millis(3000)).await;
1150 consumer
1151 .close_with_options(CloseOptions::new_with_timeout(Duration::ZERO))
1152 .await
1153 .expect("close");
1154 println!("poll errors: {} (first: {:?})", errors.len(), errors.first());
1155 assert_eq!(
1156 0,
1157 broker.fetch_requests(),
1158 "the leader never authenticated, so no FETCH reached it"
1159 );
1160 assert!(
1161 errors.iter().all(|e| e.is_authentication_error()),
1162 "every poll error is the leader's authentication failure: {errors:?}"
1163 );
1164 assert!(
1165 errors.len() >= 10,
1166 "poll() reported the leader's cached authentication failure {} times in 3 s of poll(100 ms); Java's \
1167 AbstractFetch.prepareFetchRequests rethrows NetworkClient.authenticationException(node) on every \
1168 fetch round, so 4.3.1 threw it from about every other poll (26 of 55)",
1169 errors.len()
1170 );
1171 }
1172
1173 #[test]
1178 fn kafka_consumer_is_send_and_not_sync() {
1179 assert!(is_send!(KafkaConsumer<String, String>));
1180 assert!(!is_sync!(KafkaConsumer<String, String>));
1181 }
1182
1183 #[test]
1186 fn consumer_handle_is_send_and_sync() {
1187 assert!(is_send!(ConsumerHandle));
1188 assert!(is_sync!(ConsumerHandle));
1189 }
1190}