confluent_kafka/consumer/internals/async_kafka_consumer.rs
1// Copyright 2025 Confluent Inc.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! User-facing async consumer implementing the KIP-848 (consumer-group)
16//! protocol.
17//!
18//! Translated from
19//! `org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer`. This
20//! file lands the struct, constructor, and the synchronous state-read
21//! methods (Phase 11 commit (2/N)). Subsequent commits (3-7) add
22//! subscribe / poll / commit / position / committed / close.
23//!
24//! # Generic-over-`KafkaClient`
25//!
26//! Java's `AsyncKafkaConsumer` carries the `NetworkClient` indirectly via
27//! `ApplicationEventHandler` and `ConsumerNetworkThread`. The Rust
28//! `ConsumerNetworkThread<K>` is generic over `K: KafkaClient` so tests
29//! can plug in `MockClient`. To keep the user-facing `Consumer<K, V>`
30//! trait clean of the network-client type, the spawned task is wrapped in
31//! a type-erased [`NetworkThreadCloseHandle`] that exposes only the
32//! lifecycle hooks (`signal_close`, `wakeup`, `await_join`).
33//!
34//! # Reentrancy guard (not translated)
35//!
36//! Java guards against multi-thread access via the
37//! `acquire()` / `release()` pair throwing
38//! `ConcurrentModificationException`. Rust's `&mut self` on the
39//! `Consumer` trait makes single-caller exclusivity a *compile-time*
40//! guarantee — the runtime guard is redundant. See Phase 11 PLAN.md
41//! deferral #4.
42//!
43//! # Metrics (deferred)
44//!
45//! All `kafkaConsumerMetrics.record*` / `asyncConsumerMetrics.record*`
46//! calls in the Java source are NO-OPs in this translation, marked
47//! `// METRICS-DEFERRED: …`. Per Phase 11 PLAN.md deferral #1.
48//!
49//! # Telemetry (deferred)
50//!
51//! `ClientTelemetryReporter` / `ClientTelemetryUtils` are NOT translated;
52//! the corresponding fields are `None`. Per Phase 11 PLAN.md deferral #6.
53
54// Phase 11 commits 5-7 wire commit / state-query / close. Which items are
55// dead differs between the library and its test build (the tests use some,
56// others are reached only from non-test code), so `expect` holds for neither
57// alone: expect it in the library, allow it in the test build.
58#![cfg_attr(not(test), expect(dead_code))]
59#![cfg_attr(test, allow(dead_code))]
60
61use std::collections::{HashMap, HashSet};
62use std::sync::atomic::{AtomicBool, AtomicI64, AtomicUsize, Ordering};
63use std::sync::{Arc, Mutex};
64use std::time::Duration;
65
66use tokio::sync::mpsc;
67use tokio::task::JoinHandle;
68
69use crate::common::metrics::{MetricConfig, Metrics, MetricsView, RecordingLevel};
70use crate::common::utils::Time;
71use crate::common::utils::Utils;
72use crate::common::utils::internals::LogContext;
73use crate::common::{Error, IsolationLevel, PartitionInfo, TopicPartition};
74use crate::consumer::ConsumerConfig;
75use crate::consumer::ConsumerRebalanceListener;
76use crate::consumer::ConsumerRecords;
77use crate::consumer::OffsetAndMetadata;
78use crate::consumer::OffsetAndTimestamp;
79use crate::consumer::SubscriptionPattern;
80use crate::consumer::internals::ConsumerInterceptors;
81use crate::consumer::internals::ConsumerMetadata;
82use crate::consumer::internals::ConsumerRebalanceListenerInvoker;
83use crate::consumer::internals::ConsumerUtils;
84use crate::consumer::internals::Deserializers;
85use crate::consumer::internals::FetchBuffer;
86use crate::consumer::internals::FetchCollector;
87use crate::consumer::internals::FetchMetricsManager;
88use crate::consumer::internals::FetchMetricsRegistry;
89use crate::consumer::internals::MemberState;
90use crate::consumer::internals::MemberStateListener;
91use crate::consumer::internals::OffsetAndTimestampInternal;
92use crate::consumer::internals::OffsetCommitCallbackInvoker;
93use crate::consumer::internals::OffsetCommitCallbackQueue;
94use crate::consumer::internals::PositionsValidator;
95use crate::consumer::internals::RequestManagers;
96use crate::consumer::internals::SubscriptionState;
97use crate::consumer::internals::WakeupTrigger;
98use crate::consumer::internals::events::ApplicationEventHandler;
99use crate::consumer::internals::events::BackgroundEventHandler;
100use crate::consumer::internals::events::CompletableEvent;
101use crate::consumer::internals::events::CompletableEventReaper;
102use crate::consumer::internals::events::{ApplicationEvent, AsyncPollState};
103use crate::consumer::internals::events::{BackgroundEvent, BackgroundEventEnvelope};
104use crate::consumer::internals::metrics::AsyncConsumerMetrics;
105use crate::consumer::internals::metrics::KafkaConsumerMetrics;
106use crate::consumer::internals::metrics::RebalanceCallbackMetricsManager;
107#[cfg(test)]
108use crate::consumer::internals::offset_commit_callback_invoker::CallbackInProgress;
109use crate::consumer::{ConsumerGroupMetadata, ConsumerGroupMetadataImpl};
110
111/// Backing join mechanism for the consumer background task.
112///
113/// The bg task can run in one of two execution strategies, both fully
114/// behavior-equivalent at the channel / shutdown level:
115///
116/// - [`BgJoin::Spawned`] — a `tokio::spawn`ed task on the caller's
117/// runtime. Used by unit tests and `with_components` (no-op
118/// handle). Joined on close via `JoinHandle::await`.
119/// - [`BgJoin::Dedicated`] — the production strategy (Phase 21): the
120/// bg loop runs on its own dedicated `std::thread` hosting a
121/// `current_thread` tokio runtime, removing the multi-thread
122/// scheduler park/unpark churn (~54% of consumer CPU per profiling).
123/// The bg loop signals completion on the `done` oneshot when it has
124/// finished `cleanup()`, after which the OS thread is reaped.
125enum BgJoin {
126 /// `tokio::spawn`ed task. Awaitable on close.
127 Spawned(Option<JoinHandle<()>>),
128 /// Dedicated OS thread hosting a `current_thread` runtime.
129 Dedicated {
130 /// Resolves when the bg loop has exited and `cleanup()` has
131 /// completed (sent from inside the dedicated thread).
132 done: Option<tokio::sync::oneshot::Receiver<()>>,
133 /// Handle to the OS thread; reaped (joined) after `done`.
134 thread: Option<std::thread::JoinHandle<()>>,
135 },
136}
137
138/// A `Clone + Send + Sync` handle to a consumer that exposes
139/// [`Consumer::wakeup`](crate::consumer::Consumer::wakeup), callable from any task or thread, **and** the
140/// reentrant-safe consumer operations, callable from inside a callback the consumer is running.
141///
142/// **No Java class counterpart — it recovers a Java capability.** Java's
143/// `Consumer` reference is itself a freely-shareable, thread-safe
144/// reference. Application code relies on this in two ways that a bare
145/// Rust `&mut self` consumer cannot express:
146///
147/// 1. **Cross-task `wakeup()`** — `consumer.wakeup()` is called from
148/// another thread while the owning thread blocks in `poll()` /
149/// `position()` (e.g.
150/// `CompletableFuture.runAsync(() -> consumer.wakeup())`,
151/// `org.apache.kafka.clients.consumer.PlaintextConsumerTest`).
152/// 2. **In-callback reentrancy** — a `ConsumerRebalanceListener` calls
153/// `consumer.assign/seek/pause/resume/position/committed/
154/// beginningOffsets/commit` from *inside*
155/// `onPartitionsAssigned` / `onPartitionsRevoked` by capturing the
156/// `consumer` variable in the (anonymous-inner-class) listener
157/// (`PlaintextConsumerCallbackTest.java`).
158///
159/// In Rust the consumer is owned via `&mut self` for the duration of a
160/// blocking call, and a consumer is not `Clone`, so neither
161/// pattern is expressible with a bare reference. This handle captures
162/// only the already-`Arc`-shared, internally-synchronized consumer state,
163/// so both patterns are expressible **without `unsafe`**. The user
164/// captures the handle into their listener struct — the Rust equivalent
165/// of Java capturing the `consumer` variable.
166///
167/// Obtain one via [`Consumer::handle`](crate::consumer::Consumer::handle). Cheap to clone — clones share the
168/// same underlying state.
169///
170/// # Operations
171///
172/// Sync: [`wakeup`](Self::wakeup), [`assignment`](Self::assignment),
173/// [`paused`](Self::paused), [`group_metadata`](Self::group_metadata),
174/// [`current_lag`](Self::current_lag).
175///
176/// Async (reentrant-safe consumer ops):
177/// [`seek_with_offset`](Self::seek_with_offset), [`seek_to_beginning`](Self::seek_to_beginning),
178/// [`seek_to_end`](Self::seek_to_end), [`pause`](Self::pause),
179/// [`resume`](Self::resume), [`position`](Self::position),
180/// [`committed`](Self::committed),
181/// [`beginning_offsets`](Self::beginning_offsets),
182/// [`end_offsets`](Self::end_offsets),
183/// [`offsets_for_times`](Self::offsets_for_times),
184/// [`partitions_for`](Self::partitions_for), [`list_topics`](Self::list_topics),
185/// [`commit_sync`](Self::commit_sync),
186/// [`commit_sync_with_offsets`](Self::commit_sync_with_offsets),
187/// [`commit_async`](Self::commit_async),
188/// [`commit_async_with_callback`](Self::commit_async_with_callback),
189/// [`commit_async_with_offsets_callback`](Self::commit_async_with_offsets_callback),
190/// and the `_with_timeout` form of each timed query.
191///
192/// The commits run the owning consumer's own commit path, as a Java callback
193/// calling `consumer.commitSync()` / `commitAsync()` does: they run the
194/// consumer's queued commit callbacks first, an async commit is one a later
195/// `commit_sync` or `close` waits for, and `interceptors.onCommit` fires.
196///
197/// # Surface: KIP-1306's `RebalanceConsumer`, plus `wakeup`
198///
199/// The method set is the one KIP-1306 (`RebalanceConsumer`, Kafka 4.5) hands a
200/// rebalance listener, under the Rust names of the
201/// [`Consumer`](crate::consumer::Consumer)
202/// methods: every `Consumer` method is here with `&self`, except the ones a
203/// listener must not call while its rebalance is in progress — `poll`, `close`,
204/// `subscribe*`, `unsubscribe`, `assign`, `enforce_rebalance*` — and the two
205/// reads it does not carry, `subscription` and `metrics`. `wakeup` is the one
206/// addition: Java's `Consumer.wakeup()` is callable on the consumer from any
207/// thread, while the Rust consumer's methods take `&mut self`, so the handle
208/// is the only object through which that cross-thread contract is honoured.
209/// `cargo xtask lint-custom` (`check-consumer-handle`) keeps the set aligned
210/// with the trait.
211///
212/// # Threads: the async operations work only inside a callback
213///
214/// Java's `KafkaConsumer` is not thread-safe: `acquire()` rejects a call from
215/// a thread other than the one inside the consumer with
216/// `ConcurrentModificationException`, and a listener may call back only
217/// because the thread running it is the one that holds the consumer. The
218/// Rust consumer turns that runtime guard into `&mut self`; the handle, which
219/// has no borrow of the consumer, carries the guard's other half: an async
220/// operation is admitted only **while the consumer is running a user
221/// callback** on the caller's task — a `ConsumerRebalanceListener` method or
222/// an `OffsetCommitCallback` — and otherwise returns the same
223/// `ConcurrentModificationError` (predicate
224/// `is_local_concurrent_modification_error`, message
225/// `HANDLE_OUTSIDE_CALLBACK_MSG`) before submitting anything. So a `seek`
226/// or `commit_sync` through the handle from a second thread while the owner
227/// is in `poll()` is rejected, as Java rejects it. The check is made when
228/// the operation's future runs, so a future created inside a callback and
229/// awaited after it returned is rejected too.
230///
231/// What is admitted is exactly the Java use: the listener or commit
232/// callback, on the task inside the owner's blocking call, calling back in
233/// while the owner waits for it. The background task keeps spinning during
234/// the callback, so those operations complete (consumer-threading.md §31).
235/// The scope is the consumer's, not the caller's: during a callback, a call
236/// from a third thread passes the check as well and is serialised by the
237/// background task like any other; the handle does not identify its caller.
238///
239/// [`wakeup`](Self::wakeup) and the sync reads (`assignment`, `paused`,
240/// `group_metadata`) are not gated: Java's
241/// `wakeup()` is callable from any thread by contract, and the reads take
242/// the same short locks the owning consumer's getters do.
243///
244/// # Closed consumer
245///
246/// Once the consumer's `close()` has returned, the handle refuses it as the
247/// consumer does: the async operations return its `IllegalStateError`
248/// (predicate `is_local_illegal_state_error`) ahead of the callback check,
249/// and the sync reads panic with the same message. `close()` marks the
250/// consumer closed only when it returns, so a listener it runs still calls
251/// back in, as Java's listener does from `close()`. `wakeup` stays callable.
252///
253/// A handle from a `MockConsumer` is not gated either: Java's `MockConsumer`
254/// has no `acquire()`, and the mock handle's commits commit into the mock
255/// from any thread as the mock's own methods do.
256///
257/// # Concrete `async fn`, no `#[async_trait]`
258///
259/// `ConsumerHandle` is a concrete struct, so its async methods are
260/// concrete `async fn` returning an anonymous future (no
261/// `Pin<Box<dyn Future>>`). None of its methods are on a per-record hot
262/// path.
263#[derive(Clone)]
264// a listener reenters the consumer through it (consumer-threading.md §31, §41), as a Java listener captures the consumer
265#[doc(alias = "rust-only")]
266pub struct ConsumerHandle {
267 inner: ConsumerHandleInner,
268}
269
270impl std::fmt::Debug for ConsumerHandle {
271 // A view of a consumer; it has no identifier of its own.
272 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
273 f.debug_struct("ConsumerHandle").finish_non_exhaustive()
274 }
275}
276
277/// Shared state captured by a [`ConsumerHandle`] for an
278/// [`AsyncKafkaConsumer`]. Every field is already `Arc`-shared on the
279/// consumer; the handle holds cheap clones.
280#[derive(Clone)]
281pub(crate) struct AsyncConsumerHandleState {
282 /// Rotating wakeup token + bg-task `select!` poke (see
283 /// [`AsyncKafkaConsumer::wakeup`]).
284 wakeup_trigger: WakeupTrigger,
285 bg_wakeup: Arc<dyn Fn() + Send + Sync>,
286 /// Submits `ApplicationEvent`s to the bg task.
287 application_event_handler: Arc<ApplicationEventHandler>,
288 /// Subscription / assignment state (sync getters + `position` /
289 /// `seek` pre-checks).
290 subscriptions: Arc<Mutex<SubscriptionState>>,
291 /// The consumer's metadata, whose cached cluster answers
292 /// [`ConsumerHandle::partitions_for`] before any request is made.
293 metadata: Arc<ConsumerMetadata>,
294 /// The consumer's group metadata slot, read by
295 /// [`ConsumerHandle::group_metadata`].
296 group_metadata: Arc<Mutex<Option<Arc<ConsumerGroupMetadataImpl>>>>,
297 /// Time source for deadline computation.
298 time: Arc<dyn Time>,
299 /// Cached `default.api.timeout.ms`.
300 default_api_timeout_ms: i64,
301 /// The consumer's `isolation.level`, carried by the handle's `CurrentLag`
302 /// event as the consumer's own.
303 isolation_level: IsolationLevel,
304 /// The consumer's metrics registry, read by [`ConsumerHandle::metrics`].
305 metrics: Arc<Metrics>,
306 /// The consumer's commit bookkeeping, so the handle's commits run the
307 /// consumer's own commit path.
308 commit_state: Arc<AsyncCommitState>,
309 /// The consumer's `group.id`, checked by the handle's commits.
310 group_id: Option<String>,
311 /// The consumer's metrics, whose `commit-sync` sensor the handle's
312 /// `commit_sync` records.
313 kafka_consumer_metrics: Arc<KafkaConsumerMetrics>,
314 /// How many user callbacks the consumer is running on the caller's task
315 /// right now (`OffsetCommitCallbackInvoker::callbacks_in_progress`). An
316 /// async handle operation is admitted only while it is non-zero: see
317 /// [`ConsumerHandle`]'s `Threads` section.
318 callbacks_in_progress: Arc<AtomicUsize>,
319 /// The consumer's closed flag, so the handle refuses a closed consumer as
320 /// the consumer's own methods do.
321 closed: Arc<AtomicBool>,
322}
323
324/// The message of the `ConcurrentModificationError` a [`ConsumerHandle`]
325/// operation returns outside a callback. Its first sentence is the one Java's
326/// `KafkaConsumer.acquire()` throws with.
327pub(crate) const HANDLE_OUTSIDE_CALLBACK_MSG: &str = "KafkaConsumer is not safe for multi-threaded access. \
328 ConsumerHandle operations are allowed only from inside a callback the consumer is running \
329 (ConsumerRebalanceListener, OffsetCommitCallback); from another thread call wakeup().";
330
331#[derive(Clone)]
332enum ConsumerHandleInner {
333 /// `AsyncKafkaConsumer`: full reentrant-safe op surface backed by the
334 /// shared `Arc` state.
335 Async(AsyncConsumerHandleState),
336 /// `MockConsumer`: `wakeup()` sets the shared wakeup flag observed by the
337 /// next `poll()`, the sync getters read the mock's shared
338 /// `SubscriptionState`, paused set and end offsets, and every commit
339 /// commits into the mock's shared committed offsets as Java's
340 /// `MockConsumer.commitAsync` / `commitSync` do inside a callback — the
341 /// ones without offsets commit the mock's consumed positions
342 /// (`commitSync(subscriptions.allConsumed())`).
343 /// The mock has no bg task / event pipeline, so the other async ops are
344 /// not wired (they return a `local_unsupported_operation` error — the mock test
345 /// surface drives the concrete `MockConsumer` directly). `closed`,
346 /// `should_rebalance` and `last_poll_timeout` are the mock's own, read by
347 /// the C binding while an operation borrows the mock.
348 Mock {
349 flag: Arc<AtomicBool>,
350 closed: Arc<AtomicBool>,
351 should_rebalance: Arc<AtomicBool>,
352 last_poll_timeout: Arc<Mutex<Option<Duration>>>,
353 /// The mock's committed offsets, which the handle's commits merge
354 /// into as the mock's own do.
355 committed: Arc<Mutex<HashMap<TopicPartition, OffsetAndMetadata>>>,
356 /// The mock's `SubscriptionState`, locked only for a read and never
357 /// across an `.await` (`consumer-threading.md` §16).
358 subscriptions: Arc<Mutex<SubscriptionState>>,
359 /// The mock's paused partitions (Java's `MockConsumer.paused`).
360 paused: Arc<Mutex<HashSet<TopicPartition>>>,
361 /// The mock's end offsets (Java's `MockConsumer.endOffsets`), read by
362 /// the handle's `current_lag` as the mock's own is.
363 end_offsets: Arc<Mutex<HashMap<TopicPartition, i64>>>,
364 },
365}
366
367impl ConsumerHandle {
368 /// Fires the consumer's `wakeup()` from this handle. Equivalent to
369 /// calling [`Consumer::wakeup`](crate::consumer::Consumer::wakeup) on the owning consumer, but callable
370 /// from any task / thread without holding a reference to the consumer.
371 pub fn wakeup(&self) {
372 match &self.inner {
373 ConsumerHandleInner::Async(state) => {
374 state.wakeup_trigger.wakeup();
375 (state.bg_wakeup)();
376 },
377 ConsumerHandleInner::Mock { flag, .. } => {
378 flag.store(true, Ordering::SeqCst);
379 },
380 }
381 }
382
383 // ── Sync getters ───────────────────────────────────────────────────
384
385 /// See [`Consumer::assignment`](crate::consumer::Consumer::assignment).
386 ///
387 /// Read from the consumer's shared `SubscriptionState`.
388 ///
389 /// # Returns
390 ///
391 /// As [`Consumer::assignment`](crate::consumer::Consumer::assignment).
392 ///
393 /// # Panics
394 ///
395 /// After the consumer's `close()` has returned, with Java's
396 /// `IllegalStateException` message, as the consumer's own `assignment`.
397 pub fn assignment(&self) -> HashSet<TopicPartition> {
398 match &self.inner {
399 ConsumerHandleInner::Async(state) => {
400 state.panic_if_closed();
401 state.subscriptions.lock().unwrap().assigned_partitions()
402 },
403 ConsumerHandleInner::Mock { subscriptions, .. } => lock_mock(subscriptions).assigned_partitions(),
404 }
405 }
406
407 /// See [`Consumer::paused`](crate::consumer::Consumer::paused).
408 ///
409 /// Read from the consumer's shared `SubscriptionState`; on a `MockConsumer`
410 /// handle, the mock's own paused set.
411 ///
412 /// # Returns
413 ///
414 /// As [`Consumer::paused`](crate::consumer::Consumer::paused).
415 ///
416 /// # Panics
417 ///
418 /// After the consumer's `close()` has returned, with Java's
419 /// `IllegalStateException` message, as the consumer's own `paused`.
420 pub fn paused(&self) -> HashSet<TopicPartition> {
421 match &self.inner {
422 ConsumerHandleInner::Async(state) => {
423 state.panic_if_closed();
424 state.subscriptions.lock().unwrap().paused_partitions()
425 },
426 // Java's `MockConsumer.paused()` returns its own `paused` set.
427 ConsumerHandleInner::Mock { paused, .. } => lock_mock(paused).clone(),
428 }
429 }
430
431 /// See [`Consumer::group_metadata`](crate::consumer::Consumer::group_metadata).
432 ///
433 /// Read from the consumer's shared group-metadata slot; on a `MockConsumer`
434 /// handle, Java's `MockConsumer.groupMetadata()` sentinel values.
435 ///
436 /// # Returns
437 ///
438 /// As [`Consumer::group_metadata`](crate::consumer::Consumer::group_metadata).
439 ///
440 /// # Panics
441 ///
442 /// As the consumer's own `group_metadata`: after its `close()` has
443 /// returned, with Java's `IllegalStateException` message, and on a
444 /// consumer without `group.id`, with Java's `InvalidGroupIdException`
445 /// message
446 /// (`org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer#throwIfGroupIdNotDefined`).
447 pub fn group_metadata(&self) -> Arc<dyn crate::consumer::ConsumerGroupMetadata> {
448 match &self.inner {
449 ConsumerHandleInner::Async(state) => {
450 state.panic_if_closed();
451 panic_if_group_id_not_defined(state.group_id.as_deref());
452 group_metadata_from(&state.group_metadata)
453 },
454 ConsumerHandleInner::Mock { .. } => crate::consumer::mock_consumer::mock_group_metadata(),
455 }
456 }
457
458 /// See [`Consumer::current_lag`](crate::consumer::Consumer::current_lag).
459 ///
460 /// Submits the consumer's `CurrentLag` event and waits for the background
461 /// task's answer, as the consumer's own does.
462 ///
463 /// On a `MockConsumer` handle, Java's `MockConsumer.currentLag` over the
464 /// mock's shared end offsets and positions, answered without waiting. The
465 /// handle cannot seed a missing position as the mock's own `position()`
466 /// does (it holds no reset offsets), so a partition without a position
467 /// answers `Ok(None)`, the lag being unknown.
468 ///
469 /// # Arguments
470 ///
471 /// - `topic_partition` - as [`Consumer::current_lag`](crate::consumer::Consumer::current_lag).
472 ///
473 /// # Returns
474 ///
475 /// As [`Consumer::current_lag`](crate::consumer::Consumer::current_lag).
476 ///
477 /// # Errors
478 ///
479 /// - as [`Consumer::current_lag`](crate::consumer::Consumer::current_lag);
480 /// - `Error::is_local_concurrent_modification_error` outside a callback the
481 /// consumer is running (see [`ConsumerHandle`]'s `Threads` section).
482 pub fn current_lag(
483 &self,
484 topic_partition: &TopicPartition,
485 ) -> impl Future<Output = Result<Option<i64>, Error>> + Send {
486 async move {
487 match &self.inner {
488 ConsumerHandleInner::Async(_) => self.async_state()?.current_lag(topic_partition).await,
489 ConsumerHandleInner::Mock { closed, subscriptions, end_offsets, .. } => {
490 crate::consumer::mock_consumer::mock_current_lag(
491 closed,
492 subscriptions,
493 end_offsets,
494 topic_partition,
495 )
496 },
497 }
498 }
499 }
500
501 /// [`Consumer::metrics`](crate::consumer::Consumer::metrics) via the shared metrics
502 /// registry; an empty view for a `MockConsumer`, whose `metrics()` is one.
503 ///
504 /// Crate-private: the C binding's `kafka_consumer_Consumer_metrics` reads
505 /// through it so that, as Java's `AsyncKafkaConsumer.metrics()` (which takes
506 /// no `acquire()`), it answers while another call holds the consumer.
507 pub(crate) fn metrics(&self) -> MetricsView {
508 match &self.inner {
509 ConsumerHandleInner::Async(state) => state.metrics.metrics(),
510 ConsumerHandleInner::Mock { .. } => MetricsView::default(),
511 }
512 }
513
514 /// `MockConsumer::closed` through the mock's shared flag, `None` for an
515 /// `AsyncKafkaConsumer` handle.
516 ///
517 /// Crate-private, like the two below: `kafka_consumer_MockConsumer_closed`,
518 /// `_should_rebalance` and `_last_poll_timeout` read through them so they
519 /// answer while another call holds the mock. Java's `MockConsumer` never
520 /// throws for them: `closed()` is `synchronized` and waits, the other two
521 /// take no lock at all.
522 pub(crate) fn mock_closed(&self) -> Option<bool> {
523 match &self.inner {
524 ConsumerHandleInner::Async(_) => None,
525 ConsumerHandleInner::Mock { closed, .. } => Some(closed.load(Ordering::SeqCst)),
526 }
527 }
528
529 /// `MockConsumer::should_rebalance` through the mock's shared flag,
530 /// `None` for an `AsyncKafkaConsumer` handle.
531 pub(crate) fn mock_should_rebalance(&self) -> Option<bool> {
532 match &self.inner {
533 ConsumerHandleInner::Async(_) => None,
534 ConsumerHandleInner::Mock { should_rebalance, .. } => Some(should_rebalance.load(Ordering::SeqCst)),
535 }
536 }
537
538 /// `MockConsumer::last_poll_timeout` through the mock's shared slot,
539 /// `None` for an `AsyncKafkaConsumer` handle.
540 pub(crate) fn mock_last_poll_timeout(&self) -> Option<Option<Duration>> {
541 match &self.inner {
542 ConsumerHandleInner::Async(_) => None,
543 ConsumerHandleInner::Mock { last_poll_timeout, .. } => {
544 Some(*last_poll_timeout.lock().unwrap_or_else(std::sync::PoisonError::into_inner))
545 },
546 }
547 }
548
549 // ── Async reentrant-safe consumer ops ───────────────────────────────
550
551 /// See [`Consumer::seek_with_offset`](crate::consumer::Consumer::seek_with_offset).
552 ///
553 /// # Arguments
554 ///
555 /// - `partition` - as [`Consumer::seek_with_offset`](crate::consumer::Consumer::seek_with_offset).
556 /// - `offset` - as [`Consumer::seek_with_offset`](crate::consumer::Consumer::seek_with_offset).
557 ///
558 /// # Errors
559 ///
560 /// - as [`Consumer::seek_with_offset`](crate::consumer::Consumer::seek_with_offset);
561 /// - `Error::is_local_concurrent_modification_error` outside a callback the
562 /// consumer is running (see [`ConsumerHandle`]'s `Threads` section);
563 /// - `Error::is_local_unsupported_operation_error` on a `MockConsumer` handle.
564 pub fn seek_with_offset(
565 &self,
566 partition: TopicPartition,
567 offset: i64,
568 ) -> impl Future<Output = Result<(), Error>> + Send {
569 async move { self.async_state()?.seek(partition, offset, None).await }
570 }
571
572 /// See [`Consumer::seek_with_offset_and_metadata`](crate::consumer::Consumer::seek_with_offset_and_metadata).
573 ///
574 /// # Arguments
575 ///
576 /// - `partition` - as [`Consumer::seek_with_offset_and_metadata`](crate::consumer::Consumer::seek_with_offset_and_metadata).
577 /// - `offset_and_metadata` - as [`Consumer::seek_with_offset_and_metadata`](crate::consumer::Consumer::seek_with_offset_and_metadata).
578 ///
579 /// # Errors
580 ///
581 /// - as [`Consumer::seek_with_offset_and_metadata`](crate::consumer::Consumer::seek_with_offset_and_metadata);
582 /// - `Error::is_local_concurrent_modification_error` outside a callback the
583 /// consumer is running (see [`ConsumerHandle`]'s `Threads` section);
584 /// - `Error::is_local_unsupported_operation_error` on a `MockConsumer` handle.
585 pub fn seek_with_offset_and_metadata(
586 &self,
587 partition: TopicPartition,
588 offset_and_metadata: OffsetAndMetadata,
589 ) -> impl Future<Output = Result<(), Error>> + Send {
590 async move {
591 let offset = offset_and_metadata.offset();
592 let epoch = offset_and_metadata.leader_epoch();
593 self.async_state()?.seek(partition, offset, epoch).await
594 }
595 }
596
597 /// See [`Consumer::seek_to_beginning`](crate::consumer::Consumer::seek_to_beginning).
598 ///
599 /// # Arguments
600 ///
601 /// - `partitions` - as [`Consumer::seek_to_beginning`](crate::consumer::Consumer::seek_to_beginning).
602 ///
603 /// # Errors
604 ///
605 /// - as [`Consumer::seek_to_beginning`](crate::consumer::Consumer::seek_to_beginning);
606 /// - `Error::is_local_concurrent_modification_error` outside a callback the
607 /// consumer is running (see [`ConsumerHandle`]'s `Threads` section);
608 /// - `Error::is_local_unsupported_operation_error` on a `MockConsumer` handle.
609 pub fn seek_to_beginning(&self, partitions: &[TopicPartition]) -> impl Future<Output = Result<(), Error>> + Send {
610 async move {
611 self.async_state()?
612 .seek_with_reset_strategy(partitions, crate::consumer::internals::AutoOffsetResetStrategy::EARLIEST)
613 .await
614 }
615 }
616
617 /// See [`Consumer::seek_to_end`](crate::consumer::Consumer::seek_to_end).
618 ///
619 /// # Arguments
620 ///
621 /// - `partitions` - as [`Consumer::seek_to_end`](crate::consumer::Consumer::seek_to_end).
622 ///
623 /// # Errors
624 ///
625 /// - as [`Consumer::seek_to_end`](crate::consumer::Consumer::seek_to_end);
626 /// - `Error::is_local_concurrent_modification_error` outside a callback the
627 /// consumer is running (see [`ConsumerHandle`]'s `Threads` section);
628 /// - `Error::is_local_unsupported_operation_error` on a `MockConsumer` handle.
629 pub fn seek_to_end(&self, partitions: &[TopicPartition]) -> impl Future<Output = Result<(), Error>> + Send {
630 async move {
631 self.async_state()?
632 .seek_with_reset_strategy(partitions, crate::consumer::internals::AutoOffsetResetStrategy::LATEST)
633 .await
634 }
635 }
636
637 /// See [`Consumer::pause`](crate::consumer::Consumer::pause).
638 ///
639 /// # Arguments
640 ///
641 /// - `partitions` - as [`Consumer::pause`](crate::consumer::Consumer::pause).
642 ///
643 /// # Errors
644 ///
645 /// - as [`Consumer::pause`](crate::consumer::Consumer::pause);
646 /// - `Error::is_local_concurrent_modification_error` outside a callback the
647 /// consumer is running (see [`ConsumerHandle`]'s `Threads` section);
648 /// - `Error::is_local_unsupported_operation_error` on a `MockConsumer` handle.
649 pub fn pause(&self, partitions: &[TopicPartition]) -> impl Future<Output = Result<(), Error>> + Send {
650 async move { self.async_state()?.pause(partitions).await }
651 }
652
653 /// See [`Consumer::resume`](crate::consumer::Consumer::resume).
654 ///
655 /// # Arguments
656 ///
657 /// - `partitions` - as [`Consumer::resume`](crate::consumer::Consumer::resume).
658 ///
659 /// # Errors
660 ///
661 /// - as [`Consumer::resume`](crate::consumer::Consumer::resume);
662 /// - `Error::is_local_concurrent_modification_error` outside a callback the
663 /// consumer is running (see [`ConsumerHandle`]'s `Threads` section);
664 /// - `Error::is_local_unsupported_operation_error` on a `MockConsumer` handle.
665 pub fn resume(&self, partitions: &[TopicPartition]) -> impl Future<Output = Result<(), Error>> + Send {
666 async move { self.async_state()?.resume(partitions).await }
667 }
668
669 /// See [`Consumer::position`](crate::consumer::Consumer::position).
670 ///
671 /// # Arguments
672 ///
673 /// - `partition` - as [`Consumer::position`](crate::consumer::Consumer::position).
674 ///
675 /// # Returns
676 ///
677 /// As [`Consumer::position`](crate::consumer::Consumer::position).
678 ///
679 /// # Errors
680 ///
681 /// - as [`Consumer::position`](crate::consumer::Consumer::position);
682 /// - `Error::is_local_concurrent_modification_error` outside a callback the
683 /// consumer is running (see [`ConsumerHandle`]'s `Threads` section);
684 /// - `Error::is_local_unsupported_operation_error` on a `MockConsumer` handle.
685 pub fn position(&self, partition: &TopicPartition) -> impl Future<Output = Result<i64, Error>> + Send {
686 async move {
687 let state = self.async_state()?;
688 let timeout = Duration::from_millis(state.default_api_timeout_ms as u64);
689 state.position(partition, timeout).await
690 }
691 }
692
693 /// See [`Consumer::position_with_timeout`](crate::consumer::Consumer::position_with_timeout).
694 ///
695 /// # Arguments
696 ///
697 /// - `partition` - as [`Consumer::position_with_timeout`](crate::consumer::Consumer::position_with_timeout).
698 /// - `timeout` - as [`Consumer::position_with_timeout`](crate::consumer::Consumer::position_with_timeout).
699 ///
700 /// # Returns
701 ///
702 /// As [`Consumer::position_with_timeout`](crate::consumer::Consumer::position_with_timeout).
703 ///
704 /// # Errors
705 ///
706 /// - as [`Consumer::position_with_timeout`](crate::consumer::Consumer::position_with_timeout);
707 /// - `Error::is_local_concurrent_modification_error` outside a callback the
708 /// consumer is running (see [`ConsumerHandle`]'s `Threads` section);
709 /// - `Error::is_local_unsupported_operation_error` on a `MockConsumer` handle.
710 pub fn position_with_timeout(
711 &self,
712 partition: &TopicPartition,
713 timeout: Duration,
714 ) -> impl Future<Output = Result<i64, Error>> + Send {
715 async move { self.async_state()?.position(partition, timeout).await }
716 }
717
718 /// See [`Consumer::committed`](crate::consumer::Consumer::committed).
719 ///
720 /// # Arguments
721 ///
722 /// - `partitions` - as [`Consumer::committed`](crate::consumer::Consumer::committed).
723 ///
724 /// # Returns
725 ///
726 /// As [`Consumer::committed`](crate::consumer::Consumer::committed).
727 ///
728 /// # Errors
729 ///
730 /// - as [`Consumer::committed`](crate::consumer::Consumer::committed);
731 /// - `Error::is_local_concurrent_modification_error` outside a callback the
732 /// consumer is running (see [`ConsumerHandle`]'s `Threads` section);
733 /// - `Error::is_local_unsupported_operation_error` on a `MockConsumer` handle.
734 pub fn committed(
735 &self,
736 partitions: &[TopicPartition],
737 ) -> impl Future<Output = Result<HashMap<TopicPartition, Option<OffsetAndMetadata>>, Error>> + Send {
738 async move {
739 let state = self.async_state()?;
740 let timeout = Duration::from_millis(state.default_api_timeout_ms as u64);
741 state.committed(partitions, timeout).await
742 }
743 }
744
745 /// See [`Consumer::committed_with_timeout`](crate::consumer::Consumer::committed_with_timeout).
746 ///
747 /// # Arguments
748 ///
749 /// - `partitions` - as [`Consumer::committed_with_timeout`](crate::consumer::Consumer::committed_with_timeout).
750 /// - `timeout` - as [`Consumer::committed_with_timeout`](crate::consumer::Consumer::committed_with_timeout).
751 ///
752 /// # Returns
753 ///
754 /// As [`Consumer::committed_with_timeout`](crate::consumer::Consumer::committed_with_timeout).
755 ///
756 /// # Errors
757 ///
758 /// - as [`Consumer::committed_with_timeout`](crate::consumer::Consumer::committed_with_timeout);
759 /// - `Error::is_local_concurrent_modification_error` outside a callback the
760 /// consumer is running (see [`ConsumerHandle`]'s `Threads` section);
761 /// - `Error::is_local_unsupported_operation_error` on a `MockConsumer` handle.
762 pub fn committed_with_timeout(
763 &self,
764 partitions: &[TopicPartition],
765 timeout: Duration,
766 ) -> impl Future<Output = Result<HashMap<TopicPartition, Option<OffsetAndMetadata>>, Error>> + Send {
767 async move { self.async_state()?.committed(partitions, timeout).await }
768 }
769
770 /// See [`Consumer::partitions_for`](crate::consumer::Consumer::partitions_for).
771 ///
772 /// # Arguments
773 ///
774 /// - `topic` - as [`Consumer::partitions_for`](crate::consumer::Consumer::partitions_for).
775 ///
776 /// # Returns
777 ///
778 /// As [`Consumer::partitions_for`](crate::consumer::Consumer::partitions_for).
779 ///
780 /// # Errors
781 ///
782 /// - as [`Consumer::partitions_for`](crate::consumer::Consumer::partitions_for);
783 /// - `Error::is_local_concurrent_modification_error` outside a callback the
784 /// consumer is running (see [`ConsumerHandle`]'s `Threads` section);
785 /// - `Error::is_local_unsupported_operation_error` on a `MockConsumer` handle.
786 pub fn partitions_for(&self, topic: &str) -> impl Future<Output = Result<Vec<PartitionInfo>, Error>> + Send {
787 async move {
788 let state = self.async_state()?;
789 let timeout = Duration::from_millis(state.default_api_timeout_ms as u64);
790 state.partitions_for(topic, timeout).await
791 }
792 }
793
794 /// See [`Consumer::partitions_for_with_timeout`](crate::consumer::Consumer::partitions_for_with_timeout).
795 ///
796 /// # Arguments
797 ///
798 /// - `topic` - as [`Consumer::partitions_for_with_timeout`](crate::consumer::Consumer::partitions_for_with_timeout).
799 /// - `timeout` - as [`Consumer::partitions_for_with_timeout`](crate::consumer::Consumer::partitions_for_with_timeout).
800 ///
801 /// # Returns
802 ///
803 /// As [`Consumer::partitions_for_with_timeout`](crate::consumer::Consumer::partitions_for_with_timeout).
804 ///
805 /// # Errors
806 ///
807 /// - as [`Consumer::partitions_for_with_timeout`](crate::consumer::Consumer::partitions_for_with_timeout);
808 /// - `Error::is_local_concurrent_modification_error` outside a callback the
809 /// consumer is running (see [`ConsumerHandle`]'s `Threads` section);
810 /// - `Error::is_local_unsupported_operation_error` on a `MockConsumer` handle.
811 pub fn partitions_for_with_timeout(
812 &self,
813 topic: &str,
814 timeout: Duration,
815 ) -> impl Future<Output = Result<Vec<PartitionInfo>, Error>> + Send {
816 async move { self.async_state()?.partitions_for(topic, timeout).await }
817 }
818
819 /// See [`Consumer::list_topics`](crate::consumer::Consumer::list_topics).
820 ///
821 /// # Returns
822 ///
823 /// As [`Consumer::list_topics`](crate::consumer::Consumer::list_topics).
824 ///
825 /// # Errors
826 ///
827 /// - as [`Consumer::list_topics`](crate::consumer::Consumer::list_topics);
828 /// - `Error::is_local_concurrent_modification_error` outside a callback the
829 /// consumer is running (see [`ConsumerHandle`]'s `Threads` section);
830 /// - `Error::is_local_unsupported_operation_error` on a `MockConsumer` handle.
831 pub fn list_topics(&self) -> impl Future<Output = Result<HashMap<String, Vec<PartitionInfo>>, Error>> + Send {
832 async move {
833 let state = self.async_state()?;
834 let timeout = Duration::from_millis(state.default_api_timeout_ms as u64);
835 state.list_topics(timeout).await
836 }
837 }
838
839 /// See [`Consumer::list_topics_with_timeout`](crate::consumer::Consumer::list_topics_with_timeout).
840 ///
841 /// # Arguments
842 ///
843 /// - `timeout` - as [`Consumer::list_topics_with_timeout`](crate::consumer::Consumer::list_topics_with_timeout).
844 ///
845 /// # Returns
846 ///
847 /// As [`Consumer::list_topics_with_timeout`](crate::consumer::Consumer::list_topics_with_timeout).
848 ///
849 /// # Errors
850 ///
851 /// - as [`Consumer::list_topics_with_timeout`](crate::consumer::Consumer::list_topics_with_timeout);
852 /// - `Error::is_local_concurrent_modification_error` outside a callback the
853 /// consumer is running (see [`ConsumerHandle`]'s `Threads` section);
854 /// - `Error::is_local_unsupported_operation_error` on a `MockConsumer` handle.
855 pub fn list_topics_with_timeout(
856 &self,
857 timeout: Duration,
858 ) -> impl Future<Output = Result<HashMap<String, Vec<PartitionInfo>>, Error>> + Send {
859 async move { self.async_state()?.list_topics(timeout).await }
860 }
861
862 /// See [`Consumer::beginning_offsets`](crate::consumer::Consumer::beginning_offsets).
863 ///
864 /// # Arguments
865 ///
866 /// - `partitions` - as [`Consumer::beginning_offsets`](crate::consumer::Consumer::beginning_offsets).
867 ///
868 /// # Returns
869 ///
870 /// As [`Consumer::beginning_offsets`](crate::consumer::Consumer::beginning_offsets).
871 ///
872 /// # Errors
873 ///
874 /// - as [`Consumer::beginning_offsets`](crate::consumer::Consumer::beginning_offsets);
875 /// - `Error::is_local_concurrent_modification_error` outside a callback the
876 /// consumer is running (see [`ConsumerHandle`]'s `Threads` section);
877 /// - `Error::is_local_unsupported_operation_error` on a `MockConsumer` handle.
878 pub fn beginning_offsets(
879 &self,
880 partitions: &[TopicPartition],
881 ) -> impl Future<Output = Result<HashMap<TopicPartition, i64>, Error>> + Send {
882 async move {
883 let state = self.async_state()?;
884 let timeout = Duration::from_millis(state.default_api_timeout_ms as u64);
885 // Java's `ListOffsetsRequest.EARLIEST_TIMESTAMP = -2L`.
886 state.beginning_or_end_offsets(partitions, -2, timeout).await
887 }
888 }
889
890 /// See [`Consumer::beginning_offsets_with_timeout`](crate::consumer::Consumer::beginning_offsets_with_timeout).
891 ///
892 /// # Arguments
893 ///
894 /// - `partitions` - as [`Consumer::beginning_offsets_with_timeout`](crate::consumer::Consumer::beginning_offsets_with_timeout).
895 /// - `timeout` - as [`Consumer::beginning_offsets_with_timeout`](crate::consumer::Consumer::beginning_offsets_with_timeout).
896 ///
897 /// # Returns
898 ///
899 /// As [`Consumer::beginning_offsets_with_timeout`](crate::consumer::Consumer::beginning_offsets_with_timeout).
900 ///
901 /// # Errors
902 ///
903 /// - as [`Consumer::beginning_offsets_with_timeout`](crate::consumer::Consumer::beginning_offsets_with_timeout);
904 /// - `Error::is_local_concurrent_modification_error` outside a callback the
905 /// consumer is running (see [`ConsumerHandle`]'s `Threads` section);
906 /// - `Error::is_local_unsupported_operation_error` on a `MockConsumer` handle.
907 pub fn beginning_offsets_with_timeout(
908 &self,
909 partitions: &[TopicPartition],
910 timeout: Duration,
911 ) -> impl Future<Output = Result<HashMap<TopicPartition, i64>, Error>> + Send {
912 // Java's `ListOffsetsRequest.EARLIEST_TIMESTAMP = -2L`.
913 async move { self.async_state()?.beginning_or_end_offsets(partitions, -2, timeout).await }
914 }
915
916 /// See [`Consumer::end_offsets`](crate::consumer::Consumer::end_offsets).
917 ///
918 /// # Arguments
919 ///
920 /// - `partitions` - as [`Consumer::end_offsets`](crate::consumer::Consumer::end_offsets).
921 ///
922 /// # Returns
923 ///
924 /// As [`Consumer::end_offsets`](crate::consumer::Consumer::end_offsets).
925 ///
926 /// # Errors
927 ///
928 /// - as [`Consumer::end_offsets`](crate::consumer::Consumer::end_offsets);
929 /// - `Error::is_local_concurrent_modification_error` outside a callback the
930 /// consumer is running (see [`ConsumerHandle`]'s `Threads` section);
931 /// - `Error::is_local_unsupported_operation_error` on a `MockConsumer` handle.
932 pub fn end_offsets(
933 &self,
934 partitions: &[TopicPartition],
935 ) -> impl Future<Output = Result<HashMap<TopicPartition, i64>, Error>> + Send {
936 async move {
937 let state = self.async_state()?;
938 let timeout = Duration::from_millis(state.default_api_timeout_ms as u64);
939 // Java's `ListOffsetsRequest.LATEST_TIMESTAMP = -1L`.
940 state.beginning_or_end_offsets(partitions, -1, timeout).await
941 }
942 }
943
944 /// See [`Consumer::end_offsets_with_timeout`](crate::consumer::Consumer::end_offsets_with_timeout).
945 ///
946 /// # Arguments
947 ///
948 /// - `partitions` - as [`Consumer::end_offsets_with_timeout`](crate::consumer::Consumer::end_offsets_with_timeout).
949 /// - `timeout` - as [`Consumer::end_offsets_with_timeout`](crate::consumer::Consumer::end_offsets_with_timeout).
950 ///
951 /// # Returns
952 ///
953 /// As [`Consumer::end_offsets_with_timeout`](crate::consumer::Consumer::end_offsets_with_timeout).
954 ///
955 /// # Errors
956 ///
957 /// - as [`Consumer::end_offsets_with_timeout`](crate::consumer::Consumer::end_offsets_with_timeout);
958 /// - `Error::is_local_concurrent_modification_error` outside a callback the
959 /// consumer is running (see [`ConsumerHandle`]'s `Threads` section);
960 /// - `Error::is_local_unsupported_operation_error` on a `MockConsumer` handle.
961 pub fn end_offsets_with_timeout(
962 &self,
963 partitions: &[TopicPartition],
964 timeout: Duration,
965 ) -> impl Future<Output = Result<HashMap<TopicPartition, i64>, Error>> + Send {
966 // Java's `ListOffsetsRequest.LATEST_TIMESTAMP = -1L`.
967 async move { self.async_state()?.beginning_or_end_offsets(partitions, -1, timeout).await }
968 }
969
970 /// See [`Consumer::offsets_for_times`](crate::consumer::Consumer::offsets_for_times).
971 ///
972 /// # Arguments
973 ///
974 /// - `timestamps_to_search` - as [`Consumer::offsets_for_times`](crate::consumer::Consumer::offsets_for_times).
975 ///
976 /// # Returns
977 ///
978 /// As [`Consumer::offsets_for_times`](crate::consumer::Consumer::offsets_for_times).
979 ///
980 /// # Errors
981 ///
982 /// - as [`Consumer::offsets_for_times`](crate::consumer::Consumer::offsets_for_times);
983 /// - `Error::is_local_concurrent_modification_error` outside a callback the
984 /// consumer is running (see [`ConsumerHandle`]'s `Threads` section);
985 /// - `Error::is_local_unsupported_operation_error` on a `MockConsumer` handle.
986 pub fn offsets_for_times(
987 &self,
988 timestamps_to_search: HashMap<TopicPartition, i64>,
989 ) -> impl Future<Output = Result<HashMap<TopicPartition, Option<OffsetAndTimestamp>>, Error>> + Send {
990 async move {
991 let state = self.async_state()?;
992 let timeout = Duration::from_millis(state.default_api_timeout_ms as u64);
993 state.offsets_for_times(timestamps_to_search, timeout).await
994 }
995 }
996
997 /// See [`Consumer::offsets_for_times_with_timeout`](crate::consumer::Consumer::offsets_for_times_with_timeout).
998 ///
999 /// # Arguments
1000 ///
1001 /// - `timestamps_to_search` - as [`Consumer::offsets_for_times_with_timeout`](crate::consumer::Consumer::offsets_for_times_with_timeout).
1002 /// - `timeout` - as [`Consumer::offsets_for_times_with_timeout`](crate::consumer::Consumer::offsets_for_times_with_timeout).
1003 ///
1004 /// # Returns
1005 ///
1006 /// As [`Consumer::offsets_for_times_with_timeout`](crate::consumer::Consumer::offsets_for_times_with_timeout).
1007 ///
1008 /// # Errors
1009 ///
1010 /// - as [`Consumer::offsets_for_times_with_timeout`](crate::consumer::Consumer::offsets_for_times_with_timeout);
1011 /// - `Error::is_local_concurrent_modification_error` outside a callback the
1012 /// consumer is running (see [`ConsumerHandle`]'s `Threads` section);
1013 /// - `Error::is_local_unsupported_operation_error` on a `MockConsumer` handle.
1014 pub fn offsets_for_times_with_timeout(
1015 &self,
1016 timestamps_to_search: HashMap<TopicPartition, i64>,
1017 timeout: Duration,
1018 ) -> impl Future<Output = Result<HashMap<TopicPartition, Option<OffsetAndTimestamp>>, Error>> + Send {
1019 async move { self.async_state()?.offsets_for_times(timestamps_to_search, timeout).await }
1020 }
1021
1022 /// See [`Consumer::commit_sync`](crate::consumer::Consumer::commit_sync).
1023 ///
1024 /// Commits the offsets the background task has consumed.
1025 ///
1026 /// On a `MockConsumer` handle, commits the mock's consumed positions into
1027 /// its committed offsets, as Java's `MockConsumer.commitSync()`.
1028 ///
1029 /// # Errors
1030 ///
1031 /// - as [`Consumer::commit_sync`](crate::consumer::Consumer::commit_sync);
1032 /// - `Error::is_local_concurrent_modification_error` outside a callback the
1033 /// consumer is running (see [`ConsumerHandle`]'s `Threads` section).
1034 pub fn commit_sync(&self) -> impl Future<Output = Result<(), Error>> + Send {
1035 async move {
1036 match &self.inner {
1037 ConsumerHandleInner::Async(state) => {
1038 let timeout = Duration::from_millis(state.default_api_timeout_ms as u64);
1039 state.commit_sync(None, timeout).await
1040 },
1041 ConsumerHandleInner::Mock { closed, committed, subscriptions, .. } => {
1042 crate::consumer::mock_consumer::mock_commit_all_consumed(closed, committed, subscriptions, None)
1043 .await
1044 },
1045 }
1046 }
1047 }
1048
1049 /// See [`Consumer::commit_sync_with_timeout`](crate::consumer::Consumer::commit_sync_with_timeout).
1050 ///
1051 /// On a `MockConsumer` handle, as [`commit_sync`](Self::commit_sync):
1052 /// Java's `MockConsumer.commitSync(Duration)` ignores the timeout.
1053 ///
1054 /// # Arguments
1055 ///
1056 /// - `timeout` - as [`Consumer::commit_sync_with_timeout`](crate::consumer::Consumer::commit_sync_with_timeout).
1057 ///
1058 /// # Errors
1059 ///
1060 /// - as [`Consumer::commit_sync_with_timeout`](crate::consumer::Consumer::commit_sync_with_timeout);
1061 /// - `Error::is_local_concurrent_modification_error` outside a callback the
1062 /// consumer is running (see [`ConsumerHandle`]'s `Threads` section).
1063 pub fn commit_sync_with_timeout(&self, timeout: Duration) -> impl Future<Output = Result<(), Error>> + Send {
1064 async move {
1065 match &self.inner {
1066 ConsumerHandleInner::Async(state) => state.commit_sync(None, timeout).await,
1067 ConsumerHandleInner::Mock { closed, committed, subscriptions, .. } => {
1068 crate::consumer::mock_consumer::mock_commit_all_consumed(closed, committed, subscriptions, None)
1069 .await
1070 },
1071 }
1072 }
1073 }
1074
1075 /// See [`Consumer::commit_sync_with_offsets`](crate::consumer::Consumer::commit_sync_with_offsets).
1076 ///
1077 /// On a `MockConsumer` handle, commits into the mock's committed offsets,
1078 /// as Java's `MockConsumer.commitSync(Map)`.
1079 ///
1080 /// # Arguments
1081 ///
1082 /// - `offsets` - as [`Consumer::commit_sync_with_offsets`](crate::consumer::Consumer::commit_sync_with_offsets).
1083 ///
1084 /// # Errors
1085 ///
1086 /// - as [`Consumer::commit_sync_with_offsets`](crate::consumer::Consumer::commit_sync_with_offsets);
1087 /// - `Error::is_local_concurrent_modification_error` outside a callback the
1088 /// consumer is running (see [`ConsumerHandle`]'s `Threads` section).
1089 pub fn commit_sync_with_offsets(
1090 &self,
1091 offsets: HashMap<TopicPartition, OffsetAndMetadata>,
1092 ) -> impl Future<Output = Result<(), Error>> + Send {
1093 async move {
1094 match &self.inner {
1095 ConsumerHandleInner::Async(state) => {
1096 let timeout = Duration::from_millis(state.default_api_timeout_ms as u64);
1097 state.commit_sync(Some(offsets), timeout).await
1098 },
1099 ConsumerHandleInner::Mock { closed, committed, .. } => {
1100 crate::consumer::mock_consumer::mock_commit_async(closed, committed, offsets, None).await
1101 },
1102 }
1103 }
1104 }
1105
1106 /// See [`Consumer::commit_sync_with_offsets_timeout`](crate::consumer::Consumer::commit_sync_with_offsets_timeout).
1107 ///
1108 /// On a `MockConsumer` handle, as
1109 /// [`commit_sync_with_offsets`](Self::commit_sync_with_offsets): Java's
1110 /// `MockConsumer.commitSync(Map, Duration)` ignores the timeout.
1111 ///
1112 /// # Arguments
1113 ///
1114 /// - `offsets` - as [`Consumer::commit_sync_with_offsets_timeout`](crate::consumer::Consumer::commit_sync_with_offsets_timeout).
1115 /// - `timeout` - as [`Consumer::commit_sync_with_offsets_timeout`](crate::consumer::Consumer::commit_sync_with_offsets_timeout).
1116 ///
1117 /// # Errors
1118 ///
1119 /// - as [`Consumer::commit_sync_with_offsets_timeout`](crate::consumer::Consumer::commit_sync_with_offsets_timeout);
1120 /// - `Error::is_local_concurrent_modification_error` outside a callback the
1121 /// consumer is running (see [`ConsumerHandle`]'s `Threads` section).
1122 pub fn commit_sync_with_offsets_timeout(
1123 &self,
1124 offsets: HashMap<TopicPartition, OffsetAndMetadata>,
1125 timeout: Duration,
1126 ) -> impl Future<Output = Result<(), Error>> + Send {
1127 async move {
1128 match &self.inner {
1129 ConsumerHandleInner::Async(state) => state.commit_sync(Some(offsets), timeout).await,
1130 ConsumerHandleInner::Mock { closed, committed, .. } => {
1131 crate::consumer::mock_consumer::mock_commit_async(closed, committed, offsets, None).await
1132 },
1133 }
1134 }
1135 }
1136
1137 /// See [`Consumer::commit_async`](crate::consumer::Consumer::commit_async).
1138 ///
1139 /// Commits the offsets the background task has consumed without waiting
1140 /// for the result; a later `commit_sync` or `close` waits for it.
1141 ///
1142 /// On a `MockConsumer` handle, commits the mock's consumed positions into
1143 /// its committed offsets, as Java's `MockConsumer.commitAsync()`.
1144 ///
1145 /// # Errors
1146 ///
1147 /// - as [`Consumer::commit_async`](crate::consumer::Consumer::commit_async);
1148 /// - `Error::is_local_concurrent_modification_error` outside a callback the
1149 /// consumer is running (see [`ConsumerHandle`]'s `Threads` section).
1150 pub fn commit_async(&self) -> impl Future<Output = Result<(), Error>> + Send {
1151 async move {
1152 match &self.inner {
1153 ConsumerHandleInner::Async(state) => state.commit_async(None, None).await,
1154 ConsumerHandleInner::Mock { closed, committed, subscriptions, .. } => {
1155 crate::consumer::mock_consumer::mock_commit_all_consumed(closed, committed, subscriptions, None)
1156 .await
1157 },
1158 }
1159 }
1160 }
1161
1162 /// See [`Consumer::commit_async_with_callback`](crate::consumer::Consumer::commit_async_with_callback).
1163 ///
1164 /// As [`commit_async`](Self::commit_async), then `callback` runs on the
1165 /// consumer's caller task, in commit order with the consumer's other
1166 /// commit callbacks.
1167 ///
1168 /// On a `MockConsumer` handle, commits the mock's consumed positions and
1169 /// invokes `callback` inline before returning, as Java's
1170 /// `MockConsumer.commitAsync(OffsetCommitCallback)`.
1171 ///
1172 /// # Arguments
1173 ///
1174 /// - `callback` - as [`Consumer::commit_async_with_callback`](crate::consumer::Consumer::commit_async_with_callback).
1175 ///
1176 /// # Errors
1177 ///
1178 /// - as [`Consumer::commit_async_with_callback`](crate::consumer::Consumer::commit_async_with_callback);
1179 /// - `Error::is_local_concurrent_modification_error` outside a callback the
1180 /// consumer is running (see [`ConsumerHandle`]'s `Threads` section).
1181 pub fn commit_async_with_callback(
1182 &self,
1183 callback: Arc<dyn crate::consumer::OffsetCommitCallback>,
1184 ) -> impl Future<Output = Result<(), Error>> + Send {
1185 async move {
1186 match &self.inner {
1187 ConsumerHandleInner::Async(state) => state.commit_async(None, Some(callback)).await,
1188 ConsumerHandleInner::Mock { closed, committed, subscriptions, .. } => {
1189 crate::consumer::mock_consumer::mock_commit_all_consumed(
1190 closed,
1191 committed,
1192 subscriptions,
1193 Some(callback),
1194 )
1195 .await
1196 },
1197 }
1198 }
1199 }
1200
1201 /// See [`Consumer::commit_async_with_offsets_callback`](crate::consumer::Consumer::commit_async_with_offsets_callback).
1202 ///
1203 /// Commits `offsets`, then `callback` runs as for
1204 /// [`commit_async_with_callback`](Self::commit_async_with_callback).
1205 ///
1206 /// On a `MockConsumer` handle, commits into the mock's committed offsets and
1207 /// invokes `callback` inline before returning, as Java's
1208 /// `MockConsumer.commitAsync(Map, OffsetCommitCallback)`.
1209 ///
1210 /// # Arguments
1211 ///
1212 /// - `offsets` - as [`Consumer::commit_async_with_offsets_callback`](crate::consumer::Consumer::commit_async_with_offsets_callback).
1213 /// - `callback` - as [`Consumer::commit_async_with_offsets_callback`](crate::consumer::Consumer::commit_async_with_offsets_callback).
1214 ///
1215 /// # Errors
1216 ///
1217 /// - as [`Consumer::commit_async_with_offsets_callback`](crate::consumer::Consumer::commit_async_with_offsets_callback);
1218 /// - `Error::is_local_concurrent_modification_error` outside a callback the
1219 /// consumer is running (see [`ConsumerHandle`]'s `Threads` section).
1220 pub fn commit_async_with_offsets_callback(
1221 &self,
1222 offsets: HashMap<TopicPartition, OffsetAndMetadata>,
1223 callback: Arc<dyn crate::consumer::OffsetCommitCallback>,
1224 ) -> impl Future<Output = Result<(), Error>> + Send {
1225 async move {
1226 match &self.inner {
1227 ConsumerHandleInner::Async(state) => state.commit_async(Some(offsets), Some(callback)).await,
1228 ConsumerHandleInner::Mock { closed, committed, .. } => {
1229 crate::consumer::mock_consumer::mock_commit_async(closed, committed, offsets, Some(callback)).await
1230 },
1231 }
1232 }
1233 }
1234
1235 /// Returns the shared async state, or an error if this handle was
1236 /// obtained from a `MockConsumer` (which has no event pipeline). The
1237 /// mock surface drives the concrete `MockConsumer` directly, so this
1238 /// path is never hit by faithful mock tests.
1239 ///
1240 /// On an `AsyncKafkaConsumer` handle the state is handed out only while
1241 /// the consumer is running a callback ([`Self`]'s `Threads` section);
1242 /// outside one this is the `ConcurrentModificationError` Java's
1243 /// `acquire()` throws, before any event is submitted.
1244 fn async_state(&self) -> Result<&AsyncConsumerHandleState, Error> {
1245 match &self.inner {
1246 ConsumerHandleInner::Async(state) => {
1247 state.ensure_in_callback()?;
1248 Ok(state)
1249 },
1250 ConsumerHandleInner::Mock { .. } => Err(Error::local_unsupported_operation(
1251 "ConsumerHandle async operations are not supported on a MockConsumer handle; \
1252 drive the MockConsumer directly.",
1253 )),
1254 }
1255 }
1256
1257 /// Builds an async-consumer handle from its shared state.
1258 pub(crate) fn for_async(state: AsyncConsumerHandleState) -> Self {
1259 Self { inner: ConsumerHandleInner::Async(state) }
1260 }
1261
1262 /// Builds a mock-consumer handle from its shared wakeup flag and the
1263 /// mock's shared `closed`, `should_rebalance`, `last_poll_timeout`,
1264 /// committed offsets, `SubscriptionState`, paused set and end offsets.
1265 #[expect(clippy::too_many_arguments)]
1266 pub(crate) fn for_mock(
1267 flag: Arc<AtomicBool>,
1268 closed: Arc<AtomicBool>,
1269 should_rebalance: Arc<AtomicBool>,
1270 last_poll_timeout: Arc<Mutex<Option<Duration>>>,
1271 committed: Arc<Mutex<HashMap<TopicPartition, OffsetAndMetadata>>>,
1272 subscriptions: Arc<Mutex<SubscriptionState>>,
1273 paused: Arc<Mutex<HashSet<TopicPartition>>>,
1274 end_offsets: Arc<Mutex<HashMap<TopicPartition, i64>>>,
1275 ) -> Self {
1276 Self {
1277 inner: ConsumerHandleInner::Mock {
1278 flag,
1279 closed,
1280 should_rebalance,
1281 last_poll_timeout,
1282 committed,
1283 subscriptions,
1284 paused,
1285 end_offsets,
1286 },
1287 }
1288 }
1289}
1290
1291/// Java's `AsyncKafkaConsumer#groupMetadata()` body over
1292/// the consumer's shared slot, shared by [`AsyncKafkaConsumer::group_metadata`]
1293/// and [`ConsumerHandle::group_metadata`]. Both first run
1294/// [`panic_if_group_id_not_defined`], and the slot is seeded at construction
1295/// whenever `group.id` is set (`initialize_group_metadata`, in Java's constructor), so
1296/// it is never empty here.
1297fn group_metadata_from(
1298 slot: &Mutex<Option<Arc<ConsumerGroupMetadataImpl>>>,
1299) -> Arc<dyn crate::consumer::ConsumerGroupMetadata> {
1300 let guard = slot.lock().unwrap();
1301 let meta = guard
1302 .as_ref()
1303 .expect("the group-metadata slot is seeded at construction whenever group.id is set");
1304 Arc::clone(meta) as Arc<dyn crate::consumer::ConsumerGroupMetadata>
1305}
1306
1307/// Locks a `MockConsumer`'s shared state as the mock itself does, recovering
1308/// a poisoned guard.
1309fn lock_mock<T>(state: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
1310 state.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
1311}
1312
1313impl AsyncConsumerHandleState {
1314 /// The closed half of Java's `acquireAndEnsureOpen()` for the handle's
1315 /// sync reads, as [`AsyncKafkaConsumer`]'s own `panic_if_closed`.
1316 fn panic_if_closed(&self) {
1317 if self.closed.load(Ordering::Acquire) {
1318 panic!("{CONSUMER_CLOSED_MSG}");
1319 }
1320 }
1321
1322 /// The handle's translation of the thread check in Java's
1323 /// `KafkaConsumer.acquire()`: `Ok` while the consumer is running a user
1324 /// callback on the caller's task (a rebalance-listener method or a
1325 /// commit callback), the `ConcurrentModificationError` Java throws
1326 /// otherwise. Every async handle operation calls this before it touches
1327 /// the consumer: [`ConsumerHandle::async_state`] for the event-routed
1328 /// ones, [`Self::commit_sync`] and [`Self::commit_async`] for the
1329 /// commits.
1330 ///
1331 /// A closed consumer is reported first, with the consumer's own
1332 /// `IllegalStateError`: Java's `acquireAndEnsureOpen()` passes `acquire()`
1333 /// when no thread is inside the consumer, as after `close()`, and then
1334 /// throws from `ensureOpen()`. The flag is set only when `close()`
1335 /// returns, so a listener `close()` runs still calls back in.
1336 fn ensure_in_callback(&self) -> Result<(), Error> {
1337 if self.closed.load(Ordering::Acquire) {
1338 return Err(Error::local_illegal_state(CONSUMER_CLOSED_MSG));
1339 }
1340 if self.callbacks_in_progress.load(Ordering::Acquire) == 0 {
1341 return Err(Error::local_concurrent_modification(HANDLE_OUTSIDE_CALLBACK_MSG));
1342 }
1343 Ok(())
1344 }
1345
1346 /// Shared submit + wakeup-aware-await core, the handle's
1347 /// [`AsyncKafkaConsumer::submit_and_await`]: Java's
1348 /// `applicationEventHandler.addAndGet(event)`.
1349 ///
1350 /// Like every blocking consumer call except `poll()` and
1351 /// `unsubscribe()`, it does NOT drain background events while waiting
1352 /// (§31). It is only ever called reentrantly from *inside* a user
1353 /// callback, by which point (after Phase 41b) the background task is no
1354 /// longer frozen on the callback ack — it keeps spinning and services
1355 /// this event.
1356 ///
1357 /// The wait honors `wakeup()` through [`await_event_completion`]: it
1358 /// races the receiver against the rotating wakeup token's cancellation
1359 /// and the absolute `deadline_ms`.
1360 ///
1361 /// As [`AsyncKafkaConsumer::submit_and_await`], a wakeup pending when the
1362 /// call starts is raised before the event is submitted, Java's
1363 /// `setActiveTask` before `addAndGet`.
1364 async fn submit_and_await<T: Send + 'static>(
1365 &self,
1366 event: ApplicationEvent,
1367 receiver: tokio::sync::oneshot::Receiver<Result<T, Error>>,
1368 deadline_ms: i64,
1369 timeout_msg: impl AsRef<str>,
1370 enable_wakeup: bool,
1371 ) -> Result<T, Error> {
1372 if enable_wakeup && let Err(err) = self.wakeup_trigger.maybe_trigger_wakeup() {
1373 self.wakeup_trigger.rotate();
1374 return Err(err);
1375 }
1376 let now_ms = self.time.milliseconds();
1377 self.application_event_handler.add(event, now_ms)?;
1378 self.await_completion_inner(receiver, deadline_ms, timeout_msg, enable_wakeup, false)
1379 .await
1380 }
1381
1382 fn default_api_timeout_deadline_ms(&self) -> i64 {
1383 CompletableEvent::calculate_deadline_ms(self.time.milliseconds(), self.default_api_timeout_ms)
1384 }
1385
1386 /// Reentrant-safe [`AsyncKafkaConsumer::partitions_for_with_timeout`]
1387 /// (`AsyncKafkaConsumer#partitionsFor(String,Duration)`).
1388 async fn partitions_for(&self, topic: &str, timeout: Duration) -> Result<Vec<PartitionInfo>, Error> {
1389 // Java: `Cluster cluster = this.metadata.fetch();
1390 // List<PartitionInfo> parts = cluster.partitionsForTopic(topic);
1391 // if (!parts.isEmpty()) return parts;`
1392 {
1393 let cluster = self.metadata.metadata_arc().fetch();
1394 let parts = cluster.partitions_for_topic(topic);
1395 if !parts.is_empty() {
1396 return Ok(parts.to_vec());
1397 }
1398 }
1399
1400 if timeout.is_zero() {
1401 // Java: `throw new TimeoutException();` — the same recorded
1402 // deviation as `AsyncKafkaConsumer::partitions_for_with_timeout`.
1403 return Err(Error::timeout(format!(
1404 "Timeout of {}ms expired before partitions for topic {topic} could be determined",
1405 ConsumerUtils::duration_to_millis(timeout)
1406 )));
1407 }
1408
1409 let now_ms = self.time.milliseconds();
1410 let deadline_ms = CompletableEvent::calculate_deadline_ms(now_ms, ConsumerUtils::duration_to_millis(timeout));
1411 let (handle, receiver, _erased) =
1412 CompletableEvent::make_completable_event::<HashMap<String, Vec<PartitionInfo>>>(deadline_ms);
1413 // Java's `partitionsFor` calls `setActiveTask(future)` — `enable_wakeup=true`.
1414 let map = self
1415 .submit_and_await::<HashMap<String, Vec<PartitionInfo>>>(
1416 ApplicationEvent::TopicMetadata { handle, topic: topic.to_string() },
1417 receiver,
1418 deadline_ms,
1419 "Timeout expired while waiting for TopicMetadata",
1420 true,
1421 )
1422 .await?;
1423 Ok(map.get(topic).cloned().unwrap_or_default())
1424 }
1425
1426 /// Reentrant-safe [`AsyncKafkaConsumer::list_topics_with_timeout`]
1427 /// (`AsyncKafkaConsumer#listTopics(Duration)`).
1428 async fn list_topics(&self, timeout: Duration) -> Result<HashMap<String, Vec<PartitionInfo>>, Error> {
1429 if timeout.is_zero() {
1430 // Java: `throw new TimeoutException();` — the same recorded
1431 // deviation as `AsyncKafkaConsumer::list_topics_with_timeout`.
1432 return Err(Error::timeout(format!(
1433 "Timeout of {}ms expired before all topics' metadata could be listed",
1434 ConsumerUtils::duration_to_millis(timeout)
1435 )));
1436 }
1437 let now_ms = self.time.milliseconds();
1438 let deadline_ms = CompletableEvent::calculate_deadline_ms(now_ms, ConsumerUtils::duration_to_millis(timeout));
1439 let (handle, receiver, _erased) =
1440 CompletableEvent::make_completable_event::<HashMap<String, Vec<PartitionInfo>>>(deadline_ms);
1441 // Java's `listTopics` calls `setActiveTask(future)` — `enable_wakeup=true`.
1442 self.submit_and_await::<HashMap<String, Vec<PartitionInfo>>>(
1443 ApplicationEvent::AllTopicsMetadata { handle },
1444 receiver,
1445 deadline_ms,
1446 "Timeout expired while waiting for AllTopicsMetadata",
1447 true,
1448 )
1449 .await
1450 }
1451
1452 /// Reentrant-safe [`AsyncKafkaConsumer::seek`] /
1453 /// [`AsyncKafkaConsumer::seek_with_offset_and_metadata`].
1454 async fn seek(&self, partition: TopicPartition, offset: i64, offset_epoch: Option<i32>) -> Result<(), Error> {
1455 if offset < 0 {
1456 return Err(Error::local_illegal_argument("seek offset must not be a negative number"));
1457 }
1458 log::info!("Seeking to offset {offset} for partition {partition}");
1459 let deadline_ms = self.default_api_timeout_deadline_ms();
1460 let (handle, receiver, _erased) = CompletableEvent::make_completable_event::<()>(deadline_ms);
1461 self.submit_and_await::<()>(
1462 ApplicationEvent::SeekUnvalidated { handle, partition, offset, offset_epoch },
1463 receiver,
1464 deadline_ms,
1465 "Timeout expired while waiting for the seek event to complete",
1466 false,
1467 )
1468 .await
1469 }
1470
1471 /// Reentrant-safe `seekToBeginning` / `seekToEnd`.
1472 async fn seek_with_reset_strategy(
1473 &self,
1474 partitions: &[TopicPartition],
1475 strategy: crate::consumer::internals::AutoOffsetResetStrategy,
1476 ) -> Result<(), Error> {
1477 let set: HashSet<TopicPartition> = partitions.iter().cloned().collect();
1478 let deadline_ms = self.default_api_timeout_deadline_ms();
1479 let (handle, receiver, _erased) = CompletableEvent::make_completable_event::<()>(deadline_ms);
1480 self.submit_and_await::<()>(
1481 ApplicationEvent::ResetOffset { handle, partitions: set, offset_reset_strategy: strategy },
1482 receiver,
1483 deadline_ms,
1484 "Timeout expired while waiting for the seek-with-reset-strategy event to complete",
1485 false,
1486 )
1487 .await
1488 }
1489
1490 /// Reentrant-safe [`AsyncKafkaConsumer::pause`].
1491 async fn pause(&self, partitions: &[TopicPartition]) -> Result<(), Error> {
1492 if partitions.is_empty() {
1493 return Ok(());
1494 }
1495 let deadline_ms = self.default_api_timeout_deadline_ms();
1496 let set: HashSet<TopicPartition> = partitions.iter().cloned().collect();
1497 let (handle, receiver, _erased) = CompletableEvent::make_completable_event::<()>(deadline_ms);
1498 self.submit_and_await::<()>(
1499 ApplicationEvent::PausePartitions { handle, partitions: set },
1500 receiver,
1501 deadline_ms,
1502 "Timeout expired while waiting for PausePartitions",
1503 false,
1504 )
1505 .await
1506 }
1507
1508 /// Reentrant-safe [`AsyncKafkaConsumer::resume`].
1509 async fn resume(&self, partitions: &[TopicPartition]) -> Result<(), Error> {
1510 if partitions.is_empty() {
1511 return Ok(());
1512 }
1513 let deadline_ms = self.default_api_timeout_deadline_ms();
1514 let set: HashSet<TopicPartition> = partitions.iter().cloned().collect();
1515 let (handle, receiver, _erased) = CompletableEvent::make_completable_event::<()>(deadline_ms);
1516 self.submit_and_await::<()>(
1517 ApplicationEvent::ResumePartitions { handle, partitions: set },
1518 receiver,
1519 deadline_ms,
1520 "Timeout expired while waiting for ResumePartitions",
1521 false,
1522 )
1523 .await
1524 }
1525
1526 /// Reentrant-safe [`AsyncKafkaConsumer::current_lag`].
1527 async fn current_lag(&self, topic_partition: &TopicPartition) -> Result<Option<i64>, Error> {
1528 let deadline_ms = self.default_api_timeout_deadline_ms();
1529 let (handle, receiver, _erased) = CompletableEvent::make_completable_event::<Option<i64>>(deadline_ms);
1530 // Java's `currentLag` does not call `setActiveTask`, so no wakeup.
1531 self.submit_and_await::<Option<i64>>(
1532 ApplicationEvent::CurrentLag {
1533 handle,
1534 partition: topic_partition.clone(),
1535 isolation_level: self.isolation_level,
1536 },
1537 receiver,
1538 deadline_ms,
1539 "Timeout expired while waiting for CurrentLag",
1540 false,
1541 )
1542 .await
1543 }
1544
1545 /// Reentrant-safe [`AsyncKafkaConsumer::position_with_timeout`].
1546 async fn position(&self, partition: &TopicPartition, timeout: Duration) -> Result<i64, Error> {
1547 {
1548 let subs = self.subscriptions.lock().unwrap();
1549 if !subs.is_assigned(partition) {
1550 return Err(Error::local_illegal_state(
1551 "You can only check the position for partitions assigned to this consumer.",
1552 ));
1553 }
1554 }
1555 let now_ms = self.time.milliseconds();
1556 let deadline_ms = CompletableEvent::calculate_deadline_ms(now_ms, ConsumerUtils::duration_to_millis(timeout));
1557
1558 loop {
1559 let position_offset = {
1560 let subs = self.subscriptions.lock().unwrap();
1561 subs.valid_position(partition)?.map(|fp| fp.offset)
1562 };
1563 if let Some(offset) = position_offset {
1564 return Ok(offset);
1565 }
1566
1567 let (handle, receiver, _erased) = CompletableEvent::make_completable_event::<()>(deadline_ms);
1568 let drain_result = self
1569 .submit_and_await::<()>(
1570 ApplicationEvent::CheckAndUpdatePositions { handle },
1571 receiver,
1572 deadline_ms,
1573 "Timeout expired while waiting for CheckAndUpdatePositions",
1574 true,
1575 )
1576 .await;
1577 match drain_result {
1578 Ok(()) => {},
1579 Err(Error::Timeout(_)) => {},
1580 Err(err) => return Err(err),
1581 }
1582
1583 // Java: `wakeupTrigger.maybeTriggerWakeup()` after every
1584 // `updateFetchPositions` in
1585 // `org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer#position(TopicPartition,Duration)`.
1586 if let Err(err) = self.wakeup_trigger.maybe_trigger_wakeup() {
1587 self.wakeup_trigger.rotate();
1588 return Err(err);
1589 }
1590 if self.time.milliseconds() >= deadline_ms {
1591 return Err(Error::timeout(format!(
1592 "Timeout of {}ms expired before the position for partition {} could be determined",
1593 ConsumerUtils::duration_to_millis(timeout),
1594 partition
1595 )));
1596 }
1597 }
1598 }
1599
1600 /// Reentrant-safe [`AsyncKafkaConsumer::committed_with_timeout`].
1601 async fn committed(
1602 &self,
1603 partitions: &[TopicPartition],
1604 timeout: Duration,
1605 ) -> Result<HashMap<TopicPartition, Option<OffsetAndMetadata>>, Error> {
1606 if partitions.is_empty() {
1607 return Ok(HashMap::new());
1608 }
1609 let now_ms = self.time.milliseconds();
1610 let deadline_ms = CompletableEvent::calculate_deadline_ms(now_ms, ConsumerUtils::duration_to_millis(timeout));
1611 let set: HashSet<TopicPartition> = partitions.iter().cloned().collect();
1612 let (handle, receiver, _erased) =
1613 CompletableEvent::make_completable_event::<HashMap<TopicPartition, Option<OffsetAndMetadata>>>(deadline_ms);
1614 let result = self
1615 .submit_and_await::<HashMap<TopicPartition, Option<OffsetAndMetadata>>>(
1616 ApplicationEvent::FetchCommittedOffsets { handle, partitions: set },
1617 receiver,
1618 deadline_ms,
1619 "Timeout expired while waiting for FetchCommittedOffsets",
1620 true,
1621 )
1622 .await;
1623 match result {
1624 Ok(map) => Ok(map),
1625 Err(Error::Timeout(_)) => Err(Error::timeout(format!(
1626 "Timeout of {}ms expired before the last committed offset for partitions {} could be determined. Try tuning default.api.timeout.ms larger to relax the threshold.",
1627 ConsumerUtils::duration_to_millis(timeout),
1628 format_partitions_for_display(partitions),
1629 ))),
1630 Err(err) => Err(err),
1631 }
1632 }
1633
1634 /// Reentrant-safe `beginningOffsets` / `endOffsets`.
1635 async fn beginning_or_end_offsets(
1636 &self,
1637 partitions: &[TopicPartition],
1638 timestamp: i64,
1639 timeout: Duration,
1640 ) -> Result<HashMap<TopicPartition, i64>, Error> {
1641 if partitions.is_empty() {
1642 return Ok(HashMap::new());
1643 }
1644 let mut timestamps_to_search: HashMap<TopicPartition, i64> = HashMap::new();
1645 for tp in partitions {
1646 timestamps_to_search.insert(tp.clone(), timestamp);
1647 }
1648 let now_ms = self.time.milliseconds();
1649 let deadline_ms = CompletableEvent::calculate_deadline_ms(now_ms, ConsumerUtils::duration_to_millis(timeout));
1650
1651 if timeout.is_zero() {
1652 let (handle, _receiver, _erased) = CompletableEvent::make_completable_event::<
1653 HashMap<TopicPartition, Option<OffsetAndTimestampInternal>>,
1654 >(deadline_ms);
1655 self.application_event_handler.add(
1656 ApplicationEvent::ListOffsets { handle, timestamps_to_search, require_timestamps: false },
1657 now_ms,
1658 )?;
1659 return Ok(HashMap::new());
1660 }
1661
1662 let (handle, receiver, _erased) = CompletableEvent::make_completable_event::<
1663 HashMap<TopicPartition, Option<OffsetAndTimestampInternal>>,
1664 >(deadline_ms);
1665 let result = self
1666 .submit_and_await::<HashMap<TopicPartition, Option<OffsetAndTimestampInternal>>>(
1667 ApplicationEvent::ListOffsets { handle, timestamps_to_search, require_timestamps: false },
1668 receiver,
1669 deadline_ms,
1670 "Timeout expired while waiting for ListOffsets",
1671 false,
1672 )
1673 .await;
1674 match result {
1675 Ok(offsets_map) => {
1676 let mut out = HashMap::with_capacity(offsets_map.len());
1677 for (tp, opt) in offsets_map {
1678 if let Some(oat) = opt {
1679 out.insert(tp, oat.offset());
1680 }
1681 }
1682 Ok(out)
1683 },
1684 Err(Error::Timeout(_)) => Err(Error::timeout(format!(
1685 "Failed to get offsets by times in {}ms",
1686 ConsumerUtils::duration_to_millis(timeout)
1687 ))),
1688 Err(err) => Err(err),
1689 }
1690 }
1691
1692 /// Reentrant-safe [`AsyncKafkaConsumer::offsets_for_times_with_timeout`].
1693 async fn offsets_for_times(
1694 &self,
1695 timestamps_to_search: HashMap<TopicPartition, i64>,
1696 timeout: Duration,
1697 ) -> Result<HashMap<TopicPartition, Option<OffsetAndTimestamp>>, Error> {
1698 for (tp, ts) in ×tamps_to_search {
1699 if *ts < 0 {
1700 return Err(Error::local_illegal_argument(format!(
1701 "The target time for partition {tp} is {ts}. The target time cannot be negative."
1702 )));
1703 }
1704 }
1705 if timestamps_to_search.is_empty() {
1706 return Ok(HashMap::new());
1707 }
1708 let now_ms = self.time.milliseconds();
1709 let deadline_ms = CompletableEvent::calculate_deadline_ms(now_ms, ConsumerUtils::duration_to_millis(timeout));
1710
1711 if timeout.is_zero() {
1712 let (handle, _receiver, _erased) = CompletableEvent::make_completable_event::<
1713 HashMap<TopicPartition, Option<OffsetAndTimestampInternal>>,
1714 >(deadline_ms);
1715 // `emptyResults()`: every requested partition mapped to `null`.
1716 let empty_results = timestamps_to_search.keys().map(|tp| (tp.clone(), None)).collect();
1717 self.application_event_handler.add(
1718 ApplicationEvent::ListOffsets { handle, timestamps_to_search, require_timestamps: true },
1719 now_ms,
1720 )?;
1721 return Ok(empty_results);
1722 }
1723
1724 let (handle, receiver, _erased) = CompletableEvent::make_completable_event::<
1725 HashMap<TopicPartition, Option<OffsetAndTimestampInternal>>,
1726 >(deadline_ms);
1727 let result = self
1728 .submit_and_await::<HashMap<TopicPartition, Option<OffsetAndTimestampInternal>>>(
1729 ApplicationEvent::ListOffsets { handle, timestamps_to_search, require_timestamps: true },
1730 receiver,
1731 deadline_ms,
1732 "Timeout expired while waiting for ListOffsets",
1733 false,
1734 )
1735 .await;
1736 match result {
1737 Ok(offsets_map) => {
1738 let mut out = HashMap::with_capacity(offsets_map.len());
1739 for (tp, opt) in offsets_map {
1740 let value = match opt {
1741 Some(oat) => Some(oat.build_offset_and_timestamp()?),
1742 None => None,
1743 };
1744 out.insert(tp, value);
1745 }
1746 Ok(out)
1747 },
1748 Err(Error::Timeout(_)) => Err(Error::timeout(format!(
1749 "Failed to get offsets by times in {}ms",
1750 ConsumerUtils::duration_to_millis(timeout)
1751 ))),
1752 Err(err) => Err(err),
1753 }
1754 }
1755
1756 /// Reentrant-safe [`AsyncKafkaConsumer::commit_sync`]: Java's
1757 /// `AsyncKafkaConsumer#commitSync(Optional,Duration)`
1758 /// called back from inside a callback.
1759 ///
1760 /// Runs the owning consumer's commit path through the shared
1761 /// [`AsyncCommitState`]: it runs the queued commit callbacks, waits for
1762 /// the consumer's earlier async commits (made through the consumer or a
1763 /// handle) and runs their callbacks, fires `interceptors.onCommit` and
1764 /// records the `commit-sync` metric. The waits are the consumer's too:
1765 /// neither processes background events, which only `poll()` and
1766 /// `unsubscribe()` deliver, as in Java.
1767 async fn commit_sync(
1768 &self,
1769 offsets: Option<HashMap<TopicPartition, OffsetAndMetadata>>,
1770 timeout: Duration,
1771 ) -> Result<(), Error> {
1772 self.ensure_in_callback()?;
1773 // Java: `long commitStart = time.nanoseconds()`, recorded in `finally`.
1774 let commit_start_ns = self.time.nanoseconds();
1775 let result = self.commit_sync_inner(offsets, timeout).await;
1776 self.kafka_consumer_metrics
1777 .record_commit_sync(self.time.nanoseconds() - commit_start_ns);
1778 result
1779 }
1780
1781 /// The `try`-body of [`Self::commit_sync`], as
1782 /// [`AsyncKafkaConsumer::commit_sync_inner`] is the consumer's.
1783 async fn commit_sync_inner(
1784 &self,
1785 offsets: Option<HashMap<TopicPartition, OffsetAndMetadata>>,
1786 timeout: Duration,
1787 ) -> Result<(), Error> {
1788 let now_ms = self.time.milliseconds();
1789 let deadline_ms = CompletableEvent::calculate_deadline_ms(now_ms, ConsumerUtils::duration_to_millis(timeout));
1790 let receiver = self
1791 .commit(CommitEventKind::Sync { offsets, deadline_ms })
1792 .await?
1793 .unwrap_or_else(completed_commit_result);
1794
1795 // Java: `awaitPendingAsyncCommitsAndExecuteCommitCallbacks(requestTimer, true)`.
1796 self.commit_state
1797 .await_pending_async_commits_and_execute_commit_callbacks(
1798 &self.wakeup_trigger,
1799 self.time.as_ref(),
1800 deadline_ms,
1801 true,
1802 )
1803 .await?;
1804
1805 // Java: `ConsumerUtils.getResult(commitFuture, requestTimer)`.
1806 //
1807 // RECORDED DEVIATION (definition-of-done.md §7): Java's
1808 // `commitSync(Duration)` attaches no message of its own —
1809 // `ConsumerUtils.getResult` rethrows the underlying `TimeoutException`
1810 // as-is (`org.apache.kafka.clients.consumer.internals.ConsumerUtils#getResult(Future,long)`). The text below is
1811 // Rust-side diagnostics (it resembles
1812 // `org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer#commitSync(Map,Duration)`, which is out of
1813 // scope per `consumer-threading.md` §20). It is additive: the error class, code and retriable ancestry are the
1814 // ones Java produces, and it deliberately does NOT format the offsets map. See the same
1815 // note on `AsyncKafkaConsumer::commit_sync_with_offsets_timeout`.
1816 let committed = self
1817 .await_completion::<HashMap<TopicPartition, OffsetAndMetadata>>(
1818 receiver,
1819 deadline_ms,
1820 format!(
1821 "Timeout of {}ms expired before successfully committing offsets",
1822 ConsumerUtils::duration_to_millis(timeout)
1823 ),
1824 true,
1825 )
1826 .await?;
1827
1828 // Java: `interceptors.onCommit(committedOffsets)`.
1829 self.commit_state.on_commit(&committed);
1830 Ok(())
1831 }
1832
1833 /// Reentrant-safe [`AsyncKafkaConsumer::commit_async_with_offsets_callback`]
1834 /// and its siblings: Java's
1835 /// `AsyncKafkaConsumer#commitAsync(Optional,OffsetCommitCallback)`
1836 /// called back from inside a
1837 /// callback.
1838 ///
1839 /// The commit becomes the consumer's last pending async commit, which a
1840 /// later `commit_sync` or `close` waits for, through the consumer or a
1841 /// handle. Its interceptor `onCommit` and its `callback` are queued in
1842 /// commit order on the consumer's callback invoker, which runs them as
1843 /// it runs the consumer's own.
1844 async fn commit_async(
1845 &self,
1846 offsets: Option<HashMap<TopicPartition, OffsetAndMetadata>>,
1847 callback: Option<Arc<dyn crate::consumer::OffsetCommitCallback>>,
1848 ) -> Result<(), Error> {
1849 self.ensure_in_callback()?;
1850 // Java's commitAsync is non-blocking and never throws Wakeup: the
1851 // offsets-ready wait runs with wakeup disabled.
1852 let receiver = self.commit(CommitEventKind::Async { offsets }).await?;
1853 self.commit_state.register_async_commit(receiver, callback);
1854 Ok(())
1855 }
1856
1857 /// Java's `AsyncKafkaConsumer#commit` for
1858 /// a handle: [`AsyncCommitState::submit_commit`], then the offsets-ready
1859 /// wait, bounded by `default.api.timeout.ms` as the consumer's.
1860 async fn commit(&self, commit_event: CommitEventKind) -> Result<Option<CommitResultReceiver>, Error> {
1861 let Some((offsets_ready_rx, receiver)) = self
1862 .commit_state
1863 .submit_commit(
1864 self.group_id.as_deref(),
1865 commit_event,
1866 &self.application_event_handler,
1867 self.time.as_ref(),
1868 )
1869 .await?
1870 else {
1871 return Ok(None);
1872 };
1873 // Not wakeable, as Java's `offsetsReady` wait
1874 // (`org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer#commit`): see
1875 // `AsyncKafkaConsumer::commit_inner`.
1876 self.await_completion::<()>(
1877 offsets_ready_rx,
1878 self.default_api_timeout_deadline_ms(),
1879 OFFSETS_READY_TIMEOUT_MSG,
1880 /* enable_wakeup = */ false,
1881 )
1882 .await?;
1883 Ok(Some(receiver))
1884 }
1885
1886 /// The shared wakeup-aware await core (see
1887 /// [`Self::submit_and_await`]). Awaits an already-submitted event's
1888 /// receiver, racing it against the rotating wakeup token's
1889 /// cancellation (when `enable_wakeup`) and the absolute `deadline_ms`.
1890 /// All handle ops funnel their wait through this one helper so the
1891 /// deadline / timeout logic is not duplicated. The commit paths submit
1892 /// one event but await two receivers (offsets-ready + commit result),
1893 /// hence the submit and the await are separated here.
1894 async fn await_completion<T: Send + 'static>(
1895 &self,
1896 receiver: tokio::sync::oneshot::Receiver<Result<T, Error>>,
1897 deadline_ms: i64,
1898 timeout_msg: impl AsRef<str>,
1899 enable_wakeup: bool,
1900 ) -> Result<T, Error> {
1901 self.await_completion_inner(receiver, deadline_ms, timeout_msg, enable_wakeup, true)
1902 .await
1903 }
1904
1905 /// [`Self::await_completion`], with `result_first_on_entry` as
1906 /// [`await_event_completion`] takes it: `false` where the wakeup was
1907 /// checked before the event was submitted ([`Self::submit_and_await`]).
1908 async fn await_completion_inner<T: Send + 'static>(
1909 &self,
1910 receiver: tokio::sync::oneshot::Receiver<Result<T, Error>>,
1911 deadline_ms: i64,
1912 timeout_msg: impl AsRef<str>,
1913 enable_wakeup: bool,
1914 result_first_on_entry: bool,
1915 ) -> Result<T, Error> {
1916 await_event_completion(
1917 &self.wakeup_trigger,
1918 self.time.as_ref(),
1919 receiver,
1920 deadline_ms,
1921 timeout_msg,
1922 enable_wakeup,
1923 result_first_on_entry,
1924 )
1925 .await
1926 }
1927}
1928
1929/// Type-erased handle to the consumer background task.
1930///
1931/// Owns the join mechanism ([`BgJoin`]) for the bg loop and the
1932/// `Box<dyn Fn>` closures that close / wakeup the underlying
1933/// `ConsumerNetworkThread<K>` regardless of its concrete `K`.
1934///
1935/// A type-erased, thread-safe lifecycle hook (`signal_close` / `wakeup`).
1936/// Erased so the close handle does not carry the consumer's `K`/`V` types.
1937type LifecycleFn = Box<dyn Fn() + Send + Sync>;
1938
1939/// The type-erased close hook: Java's `ConsumerNetworkThread.close(Duration)`
1940/// state change (`ConsumerNetworkThread#closeInternal`), which
1941/// carries the close timeout the bg task's `cleanup()` is bounded by.
1942type SignalCloseFn = Box<dyn Fn(Duration) + Send + Sync>;
1943
1944/// Builds the two lifecycle closures a [`NetworkThreadCloseHandle`] needs:
1945/// `signal_close_fn` (publish the close timeout, clear the running flag, then
1946/// nudge the bg task) and `wakeup_fn` (nudge the bg task).
1947///
1948/// `signal_close_fn` stores its timeout into `close_timeout_ms` — the bg
1949/// task's [`ConsumerNetworkThread::close_timeout_handle`] — mirroring Java's
1950/// `closeTimeout = timeout` (`ConsumerNetworkThread#closeInternal`). Without it the
1951/// bg task's `cleanup()` kept its 30 s default and a `close(CloseOptions
1952/// .timeout(100ms))` waited out any in-flight fetch (up to
1953/// `fetch.max.wait.ms`), which `PlaintextConsumerCloseTest
1954/// .testAsyncConsumerCloseWithTimeoutIgnoresFetchMaxWaitMs` forbids. The
1955/// timeout is stored *before* the running flag is cleared (Java assigns it
1956/// after `running = false`), so the bg loop can never observe the shutdown
1957/// and enter `cleanup()` ahead of the new timeout.
1958///
1959/// Both nudge the **transport** primitive — the notify the bg loop's poll
1960/// `select!` waits on — and NOT the [`WakeupTrigger`]. That distinction is the
1961/// whole point of this function existing:
1962///
1963/// - `WakeupTrigger::wakeup()` is a no-op once `disable()` has run, and
1964/// `close()` calls `disable()` as its very first step. Routing the close
1965/// nudge through the trigger therefore made BOTH `signal_close()` and
1966/// `wakeup()` dead by the time close used them: the bg task only noticed
1967/// `running == false` after its in-flight poll drained naturally, so every
1968/// `close()` paid a full `poll_wait_time_ms`.
1969/// - Cancelling the token also arms a user-visible `Error::Wakeup` that
1970/// the next public API call raises (§11), which a lifecycle nudge must not
1971/// do.
1972///
1973/// Java routes the same way: `ConsumerNetworkThread.close(timeout)` sets the
1974/// timeout then calls `wakeup()` -> `networkClientDelegate.wakeup()` ->
1975/// `Selector.wakeup()` (`ConsumerNetworkThread#closeInternal`). The user-facing
1976/// `WakeupTrigger` is never involved in shutdown.
1977///
1978/// Returned as a pair from one function so production and tests share the
1979/// wiring. The fixture previously stubbed these closures, which is exactly why
1980/// the dead-nudge bug survived: the stub set a flag, so it could not reproduce
1981/// the trigger's `disabled` short-circuit.
1982fn build_close_handle_fns(
1983 running: Arc<std::sync::atomic::AtomicBool>,
1984 close_timeout_ms: Arc<AtomicI64>,
1985 notify: Arc<tokio::sync::Notify>,
1986) -> (SignalCloseFn, LifecycleFn) {
1987 let close_notify = Arc::clone(¬ify);
1988 let signal_close_fn: SignalCloseFn = Box::new(move |timeout: Duration| {
1989 close_timeout_ms.store(ConsumerUtils::duration_to_millis(timeout), Ordering::Release);
1990 running.store(false, Ordering::Release);
1991 close_notify.notify_one();
1992 });
1993 let wakeup_fn: Box<dyn Fn() + Send + Sync> = Box::new(move || {
1994 notify.notify_one();
1995 });
1996 (signal_close_fn, wakeup_fn)
1997}
1998
1999/// Held by [`AsyncKafkaConsumer`] for the lifetime of the consumer
2000/// instance; dropped (with `signal_close`) on close.
2001pub(crate) struct NetworkThreadCloseHandle {
2002 /// Publishes the close timeout, cancels the bg-task `run_once` loop and
2003 /// wakes it so the next iteration observes the shutdown.
2004 signal_close_fn: SignalCloseFn,
2005 /// Wakes the bg-task's `select!` on the wakeup token. Held as an
2006 /// `Arc` (not `Box`) so a clone can be handed to a shareable
2007 /// [`ConsumerHandle`] (so cross-task `wakeup()` — a first-class Java
2008 /// pattern — is expressible without `unsafe`); the bg-wakeup
2009 /// closure is `Send + Sync` and side-effect-idempotent.
2010 wakeup_fn: Arc<dyn Fn() + Send + Sync>,
2011 /// How the bg loop is joined on close (tokio task vs dedicated thread).
2012 join: BgJoin,
2013}
2014
2015impl NetworkThreadCloseHandle {
2016 /// Constructor used by `with_components` and unit tests. The
2017 /// closures capture the concrete `ConsumerNetworkThread<K>` clones
2018 /// of the close / wakeup state so the outer struct can stay
2019 /// non-generic over `K`. The bg loop runs as a `tokio::spawn`ed task.
2020 pub(crate) fn new(
2021 signal_close_fn: SignalCloseFn,
2022 wakeup_fn: Box<dyn Fn() + Send + Sync>,
2023 join_handle: JoinHandle<()>,
2024 ) -> Self {
2025 Self {
2026 signal_close_fn,
2027 wakeup_fn: Arc::from(wakeup_fn),
2028 join: BgJoin::Spawned(Some(join_handle)),
2029 }
2030 }
2031
2032 /// Constructor used by the production [`AsyncKafkaConsumer::new`]
2033 /// path (Phase 21). The bg loop runs on a dedicated `std::thread`
2034 /// hosting a `current_thread` tokio runtime; `done` resolves when
2035 /// the bg loop has finished `cleanup()`, and `thread` is the OS
2036 /// thread handle reaped afterwards.
2037 pub(crate) fn with_dedicated(
2038 signal_close_fn: SignalCloseFn,
2039 wakeup_fn: Box<dyn Fn() + Send + Sync>,
2040 done: tokio::sync::oneshot::Receiver<()>,
2041 thread: std::thread::JoinHandle<()>,
2042 ) -> Self {
2043 Self {
2044 signal_close_fn,
2045 wakeup_fn: Arc::from(wakeup_fn),
2046 join: BgJoin::Dedicated { done: Some(done), thread: Some(thread) },
2047 }
2048 }
2049
2050 /// Clones the bg-task wakeup closure as a shareable `Arc`. Used to
2051 /// build a [`ConsumerHandle`] that can fire the bg-task `select!` from
2052 /// another task without holding any reference to the consumer.
2053 pub(crate) fn wakeup_fn_clone(&self) -> Arc<dyn Fn() + Send + Sync> {
2054 Arc::clone(&self.wakeup_fn)
2055 }
2056
2057 /// Signals the bg task to exit and wakes it from its current
2058 /// `select!`, bounding its `cleanup()` by `timeout` — Java's
2059 /// `ConsumerNetworkThread.close(Duration timeout)` state change
2060 /// (`ConsumerNetworkThread#closeInternal`; the join is
2061 /// [`Self::await_join`]). Idempotent.
2062 pub(crate) fn signal_close(&self, timeout: Duration) {
2063 (self.signal_close_fn)(timeout);
2064 }
2065
2066 /// Wakes the bg task's `select!` without signalling shutdown. Used
2067 /// from `AsyncKafkaConsumer::wakeup`.
2068 pub(crate) fn wakeup(&self) {
2069 (self.wakeup_fn)();
2070 }
2071
2072 /// Awaits the bg loop to completion. Returns `Ok(())` on clean exit,
2073 /// or wraps a task / thread panic as a `Error::local_illegal_state`.
2074 ///
2075 /// For [`BgJoin::Spawned`] this awaits the tokio `JoinHandle` exactly
2076 /// as before. For [`BgJoin::Dedicated`] it first awaits the `done`
2077 /// oneshot (which the dedicated thread fires after the bg loop exits
2078 /// and `cleanup()` completes), then reaps the OS thread off the async
2079 /// runtime via `spawn_blocking` so it does not block the close
2080 /// future. Both paths are idempotent (a second call is a no-op) and
2081 /// never hang if the bg loop already exited (a closed/`None` receiver
2082 /// is treated as a clean exit).
2083 pub(crate) async fn await_join(&mut self) -> Result<(), Error> {
2084 match &mut self.join {
2085 BgJoin::Spawned(handle) => {
2086 if let Some(handle) = handle.take() {
2087 match handle.await {
2088 Ok(()) => Ok(()),
2089 Err(join_err) => Err(Error::local_illegal_state(format!(
2090 "Consumer network thread terminated with error: {join_err}"
2091 ))),
2092 }
2093 } else {
2094 Ok(())
2095 }
2096 },
2097 BgJoin::Dedicated { done, thread } => {
2098 // Wait for the bg loop to finish `cleanup()`. If the
2099 // sender was dropped without sending (the dedicated thread
2100 // already exited), `recv()` returns `Err(RecvError)` — we
2101 // treat that as a clean exit and proceed to reap the
2102 // thread, mirroring the `None` (already-joined) case.
2103 if let Some(done_rx) = done.take() {
2104 let _ = done_rx.await;
2105 }
2106 // Reap the OS thread off the async runtime so the close
2107 // future is not blocked on `JoinHandle::join`. A panic
2108 // inside the dedicated thread is mapped to the SAME
2109 // message shape as the `Spawned` panic path.
2110 if let Some(thread) = thread.take() {
2111 let join_result = tokio::task::spawn_blocking(move || thread.join()).await;
2112 match join_result {
2113 // Outer: the spawn_blocking task itself; inner: the
2114 // dedicated OS thread.
2115 Ok(Ok(())) => Ok(()),
2116 Ok(Err(_panic)) => Err(Error::local_illegal_state(
2117 "Consumer network thread terminated with error: panic".to_string(),
2118 )),
2119 Err(join_err) => Err(Error::local_illegal_state(format!(
2120 "Consumer network thread terminated with error: {join_err}"
2121 ))),
2122 }
2123 } else {
2124 Ok(())
2125 }
2126 },
2127 }
2128 }
2129}
2130
2131/// `AsyncKafkaConsumer<K, V>` — the production consumer for the KIP-848
2132/// group protocol. Implements [`crate::consumer::Consumer`], and is the
2133/// delegate held by [`crate::consumer::KafkaConsumer`].
2134///
2135/// The struct shape mirrors the Phase 11 PLAN.md layout:
2136/// - **Shared with the bg task**: `subscriptions`, `metadata`,
2137/// `request_managers`, `background_event_rx`,
2138/// `application_event_handler`, `max_time_to_wait_ms`,
2139/// `wakeup_trigger`, `network_thread_close` (the erased handle).
2140/// - **App-side only**: `client_id`, `group_id`, `group_metadata`,
2141/// `rebalance_listener_invoker`,
2142/// `offset_commit_callback_invoker`,
2143/// `interceptors`, `auto_commit_enabled`, `default_api_timeout_ms`,
2144/// `closed`, `rebalance_listener`.
2145///
2146/// The `current_thread` / `ref_count` Java fields are NOT translated —
2147/// `&mut self` on the trait already enforces single-caller exclusivity
2148/// (Phase 11 PLAN.md deferral #4).
2149#[doc(alias = "org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer")]
2150pub struct AsyncKafkaConsumer<K, V>
2151where
2152 K: Send + Sync + 'static,
2153 V: Send + Sync + 'static,
2154{
2155 // ── Shared with the bg task ────────────────────────────────────────
2156 /// Subscription / assignment state. Wrapped in `Arc<Mutex<…>>` per
2157 /// `consumer-threading.md` §16.
2158 subscriptions: Arc<Mutex<SubscriptionState>>,
2159 /// Consumer-side metadata cache shared with the bg task and
2160 /// `NetworkClient`.
2161 metadata: Arc<ConsumerMetadata>,
2162 /// Per-RM handles shared with the bg task. See §16 / Phase-10
2163 /// pattern #3 (`Arc<Mutex<RequestManagers>>` discipline).
2164 request_managers: Arc<std::sync::Mutex<RequestManagers>>,
2165 /// Receiver for `BackgroundEvent`s emitted by the bg task. Drained
2166 /// by `process_background_events` (commit (3)).
2167 background_event_rx: mpsc::UnboundedReceiver<BackgroundEventEnvelope>,
2168 /// App-side handle for enqueuing `ApplicationEvent`s — wraps the
2169 /// channel sender shared with the bg task.
2170 application_event_handler: Arc<ApplicationEventHandler>,
2171 /// Completable-event reaper, shared with the bg task.
2172 completable_event_reaper: Arc<std::sync::Mutex<CompletableEventReaper>>,
2173 /// Mirror of the bg task's `cached_max_time_to_wait_ms`. Exposed
2174 /// via `maximum_time_to_wait_ms()` and used by the poll loop.
2175 max_time_to_wait_ms: Arc<AtomicI64>,
2176 /// Wakeup primitive shared with the bg task.
2177 wakeup_trigger: WakeupTrigger,
2178 /// Type-erased handle to the spawned bg task. Drops the
2179 /// `JoinHandle` on consumer close (after `signal_close()` + the
2180 /// final `wakeup_trigger.wakeup()`).
2181 network_thread_close: NetworkThreadCloseHandle,
2182 /// Per `consumer-threading.md` §27: the consumer owns one
2183 /// `Arc<FetchBuffer>` shared with both the bg-side `FetchRequestManager`
2184 /// (which `add`s `CompletedFetch`es as fetch responses land) and the
2185 /// app-side [`FetchCollector`] (which drains them in `poll()`).
2186 fetch_buffer: Arc<FetchBuffer>,
2187 /// App-side fetch decoder. Owned by `Arc` so the consumer can hand a
2188 /// shared reference to per-poll helpers without re-construction.
2189 fetch_collector: Arc<FetchCollector<K, V>>,
2190
2191 /// The metrics registry (Java `private final Metrics metrics`). Kept so
2192 /// Phase M7 can expose the public `metrics()` accessor over the same
2193 /// registry the fetch path records into. The fetch managers hold
2194 /// `Arc<FetchMetricsManager>` clones that reference this same registry.
2195 metrics: Arc<Metrics>,
2196
2197 /// Consumer-level poll/commit timing metrics (`KafkaConsumerMetrics`,
2198 /// `AsyncKafkaConsumer#kafkaConsumerMetrics`). Records `time-between-poll`,
2199 /// `poll-idle-ratio-avg`, `last-poll-seconds-ago`,
2200 /// `commit-sync-time-ns-total`, `committed-time-ns-total` into the same
2201 /// `metrics` registry. Wired in `poll`/`commit_sync`/`committed`/`close`.
2202 kafka_consumer_metrics: Arc<KafkaConsumerMetrics>,
2203
2204 /// Async-consumer background-task / event-queue metrics
2205 /// (`AsyncConsumerMetrics`, `AsyncKafkaConsumer.java`). Records into the
2206 /// same `metrics` registry; wired into the bg task, the event handlers,
2207 /// and the network client delegate. Used app-side by
2208 /// `process_background_events` (bg-event queue/processing time) and
2209 /// removed in `close`.
2210 async_consumer_metrics: Arc<AsyncConsumerMetrics>,
2211
2212 /// The fetch metrics manager (Java `private final FetchMetricsManager
2213 /// fetchMetricsManager`, a field since KAFKA-19542). The fetch path holds
2214 /// clones of the same `Arc`; the consumer keeps one to close it.
2215 fetch_metrics_manager: Arc<FetchMetricsManager>,
2216
2217 /// The rebalance-callback metrics manager (Java `private final
2218 /// RebalanceCallbackMetricsManager rebalanceCallbackMetricsManager`, a field
2219 /// since KAFKA-19542). Shared with the rebalance listener invoker; the
2220 /// consumer keeps one to close it.
2221 rebalance_callback_metrics_manager: Arc<RebalanceCallbackMetricsManager>,
2222
2223 /// Shared mirror of the background-event queue depth (Java reads
2224 /// `backgroundEventQueue.size()`; tokio mpsc has no `len()`). Bumped by
2225 /// `BackgroundEventHandler::add` on the bg task, reset to 0 by
2226 /// `process_background_events` (Java's `drainEvents`).
2227 background_event_queue_size: Arc<AtomicI64>,
2228
2229 // ── App-side only ─────────────────────────────────────────────────
2230 /// `client.id`, as a cheap-to-clone `Arc<str>` per CLAUDE.md §13.
2231 client_id: Arc<str>,
2232 /// `group.id`, if any.
2233 group_id: Option<String>,
2234 /// Group metadata cached by the `MemberStateListener` callback;
2235 /// returned by [`Self::group_metadata`]. `None` while uninitialized
2236 /// or for assignment-only consumers. Updated by
2237 /// [`ConsumerStateNotifier::on_member_epoch_updated`] which is the
2238 /// `MemberStateListener` registered with the membership manager
2239 /// (production wire-up in Phase 12; for tests, callers register
2240 /// [`Self::state_notifier`] directly on the membership manager).
2241 group_metadata: Arc<Mutex<Option<Arc<ConsumerGroupMetadataImpl>>>>,
2242 /// Java: `private final AtomicReference<Set<TopicPartition>> groupAssignmentSnapshot`
2243 /// (`AsyncKafkaConsumer#groupAssignmentSnapshot`).
2244 ///
2245 /// Snapshot of the partitions assigned to this consumer through the
2246 /// **group-management** path (not `assign(...)` — manually-assigned
2247 /// partitions never appear here). Updated by
2248 /// [`ConsumerStateNotifier::on_group_assignment_updated`] from the
2249 /// membership manager's reconciliation step (Java
2250 /// `AsyncKafkaConsumer#setGroupAssignmentSnapshot`). Read by
2251 /// [`Self::run_rebalance_callbacks_on_close`] to determine which
2252 /// partitions are passed to the user's `on_partitions_revoked` /
2253 /// `on_partitions_lost` callback on close.
2254 group_assignment_snapshot: Arc<Mutex<HashSet<TopicPartition>>>,
2255 /// `MemberStateListener` impl that updates `group_metadata` and
2256 /// `group_assignment_snapshot` when the membership manager fires a
2257 /// state-change notification. Cloned out of [`Self::state_notifier`]
2258 /// at construction time and exposed for production wire-up
2259 /// (Phase 12).
2260 state_notifier: Arc<ConsumerStateNotifier>,
2261 /// Invoker for the user-supplied [`ConsumerRebalanceListener`].
2262 rebalance_listener_invoker: ConsumerRebalanceListenerInvoker,
2263 /// Drains pending `OffsetCommitCallback` invocations at the top of
2264 /// every blocking-style API.
2265 offset_commit_callback_invoker: Arc<OffsetCommitCallbackInvoker<K, V>>,
2266 /// User-supplied interceptors. Wrapped in `Mutex` because
2267 /// `on_consume` takes `&mut self` per Phase 2.
2268 interceptors: Arc<Mutex<ConsumerInterceptors<K, V>>>,
2269 /// Cached `enable.auto.commit`.
2270 auto_commit_enabled: bool,
2271 /// Cached `default.api.timeout.ms`.
2272 default_api_timeout_ms: i64,
2273 /// Cached `retry.backoff.ms` — used by [`Self::poll_for_fetches`] to
2274 /// throttle the poll wait when no positions are valid yet (Java
2275 /// `AsyncKafkaConsumer.pollForFetches`).
2276 retry_backoff_ms: i64,
2277 /// Cached `isolation.level`. Currently used by the (Phase 11 commit
2278 /// 6/N) `current_lag` event and reserved for future fetch-path
2279 /// callers; kept on the consumer struct because Java reads it from
2280 /// the same source.
2281 isolation_level: IsolationLevel,
2282 /// `true` after [`Self::close`] has run. Subsequent calls return
2283 /// `Error::local_illegal_state`. Shared with every [`ConsumerHandle`],
2284 /// whose calls check it as the consumer's own do.
2285 closed: Arc<AtomicBool>,
2286 /// Listener registered via `subscribe_with_topics_listener` /
2287 /// `subscribe_with_pattern_listener`. Wrapped in `Mutex<Option<…>>`
2288 /// so it can be swapped without invalidating
2289 /// `&self.rebalance_listener_invoker` references.
2290 rebalance_listener: Mutex<Option<Arc<dyn ConsumerRebalanceListener>>>,
2291 /// Java: `private AsyncPollEvent inflightPoll`.
2292 ///
2293 /// Stores the `Arc<AsyncPollState>` of the currently-inflight
2294 /// `AsyncPoll` event (if any). Recycled across `poll()` calls per
2295 /// Java semantics. Held in a plain `Option<...>` instead of
2296 /// `Mutex<Option<...>>` because `poll()` takes `&mut self` and is the
2297 /// only caller.
2298 inflight_poll: Option<InflightPoll>,
2299 /// Java: `private volatile boolean hasPendingReconciliation` (AK 4.3.1,
2300 /// KAFKA-20106). Set on the bg task by the `memberStateListener`'s
2301 /// `on_member_state_change` (`true` iff the member is `RECONCILING`);
2302 /// read by the app-side [`Self::collect_fetch`] to decide whether to wait
2303 /// for the in-flight poll's reconciliation check before returning
2304 /// buffered records. Shared with [`ConsumerStateNotifier`] via `Arc`.
2305 has_pending_reconciliation: Arc<AtomicBool>,
2306 /// Java: `private final PositionsValidator positionsValidator`
2307 /// (`AsyncKafkaConsumer#positionsValidator`). Read-only from this side: the app
2308 /// task calls [`PositionsValidator::can_skip_update_fetch_positions`]
2309 /// on the `poll()` critical path, while the background task owns every
2310 /// mutation through the `OffsetsRequestManager` that shares the `Arc`.
2311 positions_validator: Arc<PositionsValidator>,
2312 /// Cached `ConsumerConfig` for late-bound config lookups (e.g.
2313 /// inside `close`).
2314 config: ConsumerConfig,
2315 /// Time source used for `current_time_ms` arguments to events.
2316 time: Arc<dyn Time>,
2317 /// The commit bookkeeping shared with every [`ConsumerHandle`]: Java's
2318 /// `lastPendingAsyncCommit`, plus type-erased views of
2319 /// `offset_commit_callback_invoker` and `interceptors` (see
2320 /// [`AsyncCommitState`]).
2321 commit_state: Arc<AsyncCommitState>,
2322}
2323
2324impl<K, V> std::fmt::Debug for AsyncKafkaConsumer<K, V>
2325where
2326 K: Send + Sync + 'static,
2327 V: Send + Sync + 'static,
2328{
2329 /// The identifiers Java's log context prints, `clientId` and `groupId`.
2330 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2331 f.debug_struct("AsyncKafkaConsumer")
2332 .field("client_id", &self.client_id)
2333 .field("group_id", &self.group_id)
2334 .finish_non_exhaustive()
2335 }
2336}
2337
2338/// Tracks the state of the currently-inflight `AsyncPoll` event.
2339///
2340/// Mirrors Java's `AsyncPollEvent` field: it carries the event's
2341/// `deadline_ms` and the shared `Arc<AsyncPollState>` that the bg task
2342/// completes asynchronously. Stored as a separate small struct so the
2343/// `is_expired` check stays close to the data it operates on.
2344pub(crate) struct InflightPoll {
2345 deadline_ms: i64,
2346 state: Arc<AsyncPollState>,
2347}
2348
2349impl InflightPoll {
2350 fn is_expired(&self, current_time_ms: i64) -> bool {
2351 current_time_ms >= self.deadline_ms
2352 }
2353}
2354
2355/// Discriminates between the async / sync commit forms when calling the
2356/// shared [`AsyncKafkaConsumer::commit_inner`] helper. Mirrors the
2357/// `CommitEvent` Java base class — both `AsyncCommitEvent` and
2358/// `SyncCommitEvent` carry the same shape (offsets + deadline) and
2359/// differ only in the bg-side dispatch.
2360enum CommitEventKind {
2361 Async {
2362 offsets: Option<HashMap<TopicPartition, OffsetAndMetadata>>,
2363 },
2364 Sync {
2365 offsets: Option<HashMap<TopicPartition, OffsetAndMetadata>>,
2366 deadline_ms: i64,
2367 },
2368}
2369
2370impl CommitEventKind {
2371 fn offsets(&self) -> Option<&HashMap<TopicPartition, OffsetAndMetadata>> {
2372 match self {
2373 Self::Async { offsets } => offsets.as_ref(),
2374 Self::Sync { offsets, .. } => offsets.as_ref(),
2375 }
2376 }
2377
2378 /// For [`Self::Sync`], the user-supplied timeout-converted deadline;
2379 /// for [`Self::Async`], the consumer's default API timeout (Java's
2380 /// `AsyncCommitEvent` carries no explicit deadline — it uses the
2381 /// default).
2382 fn deadline_ms(&self) -> i64 {
2383 match self {
2384 Self::Async { .. } => i64::MAX,
2385 Self::Sync { deadline_ms, .. } => *deadline_ms,
2386 }
2387 }
2388}
2389
2390/// The consumer's commit-sync `interceptors.onCommit`, shared with its handles.
2391type OnCommitHook = Arc<dyn Fn(&HashMap<TopicPartition, OffsetAndMetadata>) + Send + Sync>;
2392
2393/// The result receiver of a submitted commit event: resolves with the
2394/// committed offsets, Java's `CommitEvent.future()`.
2395type CommitResultReceiver = tokio::sync::oneshot::Receiver<Result<HashMap<TopicPartition, OffsetAndMetadata>, Error>>;
2396
2397/// The offsets-ready receiver of a submitted commit event, Java's
2398/// `CommitEvent.offsetsReady()`.
2399type OffsetsReadyReceiver = tokio::sync::oneshot::Receiver<Result<(), Error>>;
2400
2401/// Timeout message of the offsets-ready wait in Java's `commit(CommitEvent)`,
2402/// shared by the owning consumer and its [`ConsumerHandle`].
2403const OFFSETS_READY_TIMEOUT_MSG: &str = "Timed out waiting for offsetsReady on commit event";
2404
2405/// The message of the `IllegalStateException` Java's
2406/// `AsyncKafkaConsumer.acquireAndEnsureOpen()` throws on a closed consumer.
2407const CONSUMER_CLOSED_MSG: &str = "This consumer has already been closed.";
2408
2409/// Timeout message of the wait for the last pending async commit in Java's
2410/// `awaitPendingAsyncCommitsAndExecuteCommitCallbacks`.
2411const PENDING_ASYNC_COMMIT_TIMEOUT_MSG: &str = "Timed out waiting for last pending async commit to complete";
2412
2413/// Java's `InvalidGroupIdException` message from `throwIfGroupIdNotDefined`.
2414const GROUP_ID_NOT_DEFINED_MSG: &str = "To use the group management or offset commit APIs, you must provide a \
2415 valid group.id in the consumer configuration.";
2416
2417/// Translates Java's `AsyncKafkaConsumer#throwIfGroupIdNotDefined`
2418/// over a borrowed `group.id`, so the
2419/// owning consumer and its [`ConsumerHandle`] run the same check.
2420fn return_error_if_group_id_not_defined(group_id: Option<&str>) -> Result<(), Error> {
2421 if group_id.map(str::is_empty).unwrap_or(true) {
2422 return Err(Error::invalid_group_id(GROUP_ID_NOT_DEFINED_MSG));
2423 }
2424 Ok(())
2425}
2426
2427/// The panicking form of [`return_error_if_group_id_not_defined`] for the
2428/// sync `group_metadata` getters of the consumer and its [`ConsumerHandle`],
2429/// which have no error channel: a missing `group.id` is a programming error
2430/// (CLAUDE.md §12.5), reported with Java's `InvalidGroupIdException` message.
2431fn panic_if_group_id_not_defined(group_id: Option<&str>) {
2432 if let Err(error) = return_error_if_group_id_not_defined(group_id) {
2433 panic!("{}", error.message());
2434 }
2435}
2436
2437/// Takes an event's result if the background task has already delivered it,
2438/// as Java's `future.isDone()` followed by `getResult(future)`. A sender
2439/// dropped without a result is reported as the error every completion wait
2440/// gives for it. `None` while the result is still pending.
2441fn try_take_completion<T>(receiver: &mut tokio::sync::oneshot::Receiver<Result<T, Error>>) -> Option<Result<T, Error>> {
2442 match receiver.try_recv() {
2443 Ok(result) => Some(result),
2444 Err(tokio::sync::oneshot::error::TryRecvError::Closed) => Some(Err(Error::local_illegal_state(
2445 "Background task dropped the completion sender without completing it",
2446 ))),
2447 Err(tokio::sync::oneshot::error::TryRecvError::Empty) => None,
2448 }
2449}
2450
2451/// Waits for an already-submitted event's result: Java's
2452/// `ConsumerUtils.getResult(future, timer)` behind
2453/// `applicationEventHandler.addAndGet(event)`, with
2454/// `wakeupTrigger.setActiveTask(future)` when `enable_wakeup`.
2455///
2456/// This is the wait of every blocking consumer call except `poll()` and
2457/// `unsubscribe()`, the owning consumer's and the [`ConsumerHandle`]'s
2458/// alike. It does NOT process background events: Java runs
2459/// `processBackgroundEvents()` only from `poll()` and `unsubscribe()`, so a
2460/// queued rebalance callback or `ErrorEvent` stays queued for the next
2461/// `poll()` (`consumer-threading.md` §31).
2462///
2463/// The receiver is raced against the rotating wakeup token's cancellation
2464/// (when `enable_wakeup`) and the absolute `deadline_ms`, in 100 ms slices
2465/// so the deadline follows the mock-clock-safe `time`.
2466///
2467/// `result_first_on_entry`: whether the event's result may already be there
2468/// when its wakeup is registered, Java's `setActiveTask` on a done future,
2469/// which keeps the result and leaves the wakeup pending. `false` where the
2470/// wakeup was checked before the event was submitted (Java's
2471/// `setActiveTask` before `addAndGet`): a wakeup seen afterwards found the
2472/// event's future pending, so it wins.
2473async fn await_event_completion<T: Send + 'static>(
2474 wakeup_trigger: &WakeupTrigger,
2475 time: &dyn Time,
2476 mut receiver: tokio::sync::oneshot::Receiver<Result<T, Error>>,
2477 deadline_ms: i64,
2478 timeout_msg: impl AsRef<str>,
2479 enable_wakeup: bool,
2480 result_first_on_entry: bool,
2481) -> Result<T, Error> {
2482 // Java's `setActiveTask(future)` on a future that is already done
2483 // keeps its result and leaves a pending wakeup armed for the next
2484 // call (`WakeupTrigger.java` `setActiveTask`), so on entry the result
2485 // is looked at before the wakeup, where the future can be done then
2486 // (`result_first_on_entry`). Afterwards the wakeup goes first: it
2487 // was not there when the result was last seen missing, and where both
2488 // arrived since, the `biased` wait below also prefers it.
2489 if result_first_on_entry && let Some(result) = try_take_completion(&mut receiver) {
2490 return result;
2491 }
2492 loop {
2493 if enable_wakeup && let Err(err) = wakeup_trigger.maybe_trigger_wakeup() {
2494 wakeup_trigger.rotate();
2495 return Err(err);
2496 }
2497 match receiver.try_recv() {
2498 Ok(Ok(value)) => return Ok(value),
2499 Ok(Err(err)) => return Err(err),
2500 Err(tokio::sync::oneshot::error::TryRecvError::Closed) => {
2501 return Err(Error::local_illegal_state(
2502 "Background task dropped the completion sender without completing it",
2503 ));
2504 },
2505 Err(tokio::sync::oneshot::error::TryRecvError::Empty) => {
2506 let remaining = deadline_ms.saturating_sub(time.milliseconds()).max(0);
2507 if remaining <= 0 {
2508 return Err(Error::timeout(timeout_msg.as_ref().to_string()));
2509 }
2510 let wait = std::cmp::min(remaining, 100) as u64;
2511 let token = if enable_wakeup {
2512 Some(wakeup_trigger.current_token())
2513 } else {
2514 None
2515 };
2516 let recv_fut = &mut receiver;
2517 match token {
2518 Some(tok) => {
2519 tokio::select! {
2520 biased;
2521 _ = tok.cancelled() => {
2522 // The wakeup came while the result was
2523 // pending: raise it now, ahead of the
2524 // deadline check. A trigger disabled by
2525 // close() reports none; keep waiting.
2526 if let Err(err) = wakeup_trigger.maybe_trigger_wakeup() {
2527 wakeup_trigger.rotate();
2528 return Err(err);
2529 }
2530 },
2531 res = tokio::time::timeout(Duration::from_millis(wait), recv_fut) => {
2532 match res {
2533 Ok(Ok(Ok(value))) => return Ok(value),
2534 Ok(Ok(Err(err))) => return Err(err),
2535 Ok(Err(_recv_err)) => {
2536 return Err(Error::local_illegal_state(
2537 "Background task dropped the completion sender without completing it",
2538 ));
2539 },
2540 Err(_elapsed) => {},
2541 }
2542 },
2543 }
2544 },
2545 None => match tokio::time::timeout(Duration::from_millis(wait), recv_fut).await {
2546 Ok(Ok(Ok(value))) => return Ok(value),
2547 Ok(Ok(Err(err))) => return Err(err),
2548 Ok(Err(_recv_err)) => {
2549 return Err(Error::local_illegal_state(
2550 "Background task dropped the completion sender without completing it",
2551 ));
2552 },
2553 Err(_elapsed) => {},
2554 },
2555 }
2556 },
2557 }
2558 if deadline_ms.saturating_sub(time.milliseconds()).max(0) <= 0 {
2559 return Err(Error::timeout(timeout_msg.as_ref().to_string()));
2560 }
2561 }
2562}
2563
2564/// An already-completed commit result, Java's `completedFuture(null)` for
2565/// empty explicit offsets, read by `commitSync` as no committed offsets.
2566fn completed_commit_result() -> CommitResultReceiver {
2567 let (tx, rx) = tokio::sync::oneshot::channel();
2568 let _ = tx.send(Ok(HashMap::new()));
2569 rx
2570}
2571
2572/// A pending async commit: done once the interceptor and user-callback
2573/// invocations of its commit, and of every earlier async commit, have been
2574/// enqueued on the callback invoker.
2575///
2576/// A `watch` receiver rather than a `oneshot`, so the owning consumer and
2577/// every [`ConsumerHandle`] can wait for the same commit without taking it
2578/// from the shared slot: Java's `lastPendingAsyncCommit` is a
2579/// `CompletableFuture`, which any number of callers can wait for. A dropped
2580/// sender counts as done, as Java's `whenComplete` wrapper completes on
2581/// either outcome.
2582#[derive(Clone)]
2583struct PendingAsyncCommit(tokio::sync::watch::Receiver<bool>);
2584
2585impl PendingAsyncCommit {
2586 /// An incomplete pending commit and the sender that completes it.
2587 fn incomplete() -> (tokio::sync::watch::Sender<bool>, Self) {
2588 let (tx, rx) = tokio::sync::watch::channel(false);
2589 (tx, Self(rx))
2590 }
2591
2592 /// An already-completed pending commit, Java's `completedFuture(null)`.
2593 fn completed() -> Self {
2594 let (_tx, rx) = tokio::sync::watch::channel(true);
2595 Self(rx)
2596 }
2597
2598 /// Waits until the commit is done (or its sender is dropped).
2599 async fn wait(&mut self) {
2600 let _ = self.0.wait_for(|done| *done).await;
2601 }
2602
2603 /// Java's `isDone()`: the pending commit has completed.
2604 fn is_done(&self) -> bool {
2605 *self.0.borrow()
2606 }
2607
2608 /// `true` when both refer to the same pending commit.
2609 fn is_same(&self, other: &Self) -> bool {
2610 self.0.same_channel(&other.0)
2611 }
2612}
2613
2614/// The two pending-commit slots of [`AsyncCommitState`].
2615#[derive(Default)]
2616struct PendingAsyncCommits {
2617 /// Java: `private CompletableFuture<...> lastPendingAsyncCommit`.
2618 ///
2619 /// Tracks the most-recently-submitted async commit so that
2620 /// `commit_sync` and `close` can wait for in-flight async commits to
2621 /// complete before continuing (mirrors Java's
2622 /// `awaitPendingAsyncCommitsAndExecuteCommitCallbacks`). It is done only
2623 /// after the interceptor / callback invocations of this commit (success
2624 /// OR failure) and of every earlier async commit have been enqueued —
2625 /// the actual commit result is delivered via the registered
2626 /// [`crate::consumer::OffsetCommitCallback`], not via this slot.
2627 ///
2628 /// Exception: after an empty-offsets `commit_async` this is an
2629 /// already-completed commit covering only that commit, as Java's
2630 /// `completedFuture(null)` replaces `lastPendingAsyncCommit`. The earlier
2631 /// in-flight commits it displaced are kept in
2632 /// [`ordering_tail`](Self::ordering_tail).
2633 last: Option<PendingAsyncCommit>,
2634 /// Rust-specific, no Java counterpart. The in-flight async commit that an
2635 /// empty-offsets `commit_async` displaced from [`last`](Self::last), which
2636 /// the next non-empty `commit_async` chains behind instead.
2637 ///
2638 /// Needed because Rust enqueues each async commit's callbacks from its own
2639 /// spawned task, which the runtime may schedule in any order, so each
2640 /// continuation awaits its predecessor to keep callbacks in commit order
2641 /// (`KafkaConsumer#commitAsync(OffsetCommitCallback)`). Java needs no equivalent: its
2642 /// `whenComplete` actions run on the single background thread in
2643 /// completion order. Without this, a non-empty commit issued after an
2644 /// empty one would chain behind the already-completed empty commit, and
2645 /// its callbacks could be enqueued before those of an earlier, still
2646 /// in-flight commit.
2647 ordering_tail: Option<PendingAsyncCommit>,
2648}
2649
2650/// The commit bookkeeping an [`AsyncKafkaConsumer`] shares with every
2651/// [`ConsumerHandle`] it hands out: the callback invoker, the interceptor
2652/// `onCommit` hook and the pending async commits.
2653///
2654/// Rust-only (DoD #7). In Java these are fields of `AsyncKafkaConsumer`
2655/// (`offsetCommitCallbackInvoker`, `interceptors`, `lastPendingAsyncCommit`),
2656/// and a callback that commits reaches them by calling back into the
2657/// `consumer` it captured. A Rust callback calls back through a
2658/// `ConsumerHandle`, which cannot borrow the consumer (it is `&mut`-borrowed
2659/// by the call that runs the callback) and is not generic over `<K, V>`. So
2660/// the fields move into this `Arc`-shared struct, type-erased where they
2661/// mention `K` / `V`, and both the consumer and its handles run the commit
2662/// path through its methods: one implementation of `commit`, the `commitAsync`
2663/// continuation and `awaitPendingAsyncCommitsAndExecuteCommitCallbacks`.
2664pub(crate) struct AsyncCommitState {
2665 /// The consumer's `OffsetCommitCallbackInvoker`.
2666 invoker: Arc<dyn OffsetCommitCallbackQueue>,
2667 /// Java's `interceptors.onCommit(committedOffsets)` in `commitSync`, over
2668 /// the consumer's interceptor chain.
2669 on_commit: OnCommitHook,
2670 /// Java's `lastPendingAsyncCommit` plus the Rust ordering tail. Locked
2671 /// only to read or replace a slot, never across an `.await`.
2672 pending: Mutex<PendingAsyncCommits>,
2673}
2674
2675impl AsyncCommitState {
2676 fn new(invoker: Arc<dyn OffsetCommitCallbackQueue>, on_commit: OnCommitHook) -> Self {
2677 Self { invoker, on_commit, pending: Mutex::new(PendingAsyncCommits::default()) }
2678 }
2679
2680 fn lock_pending(&self) -> std::sync::MutexGuard<'_, PendingAsyncCommits> {
2681 self.pending.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
2682 }
2683
2684 /// Java's `offsetCommitCallbackInvoker.executeCallbacks()`; returns the
2685 /// first error a commit callback returned, the callbacks behind it
2686 /// staying queued.
2687 async fn invoke_pending_callbacks(&self) -> Result<(), Error> {
2688 self.invoker.invoke_pending_callbacks().await
2689 }
2690
2691 /// Java's `interceptors.onCommit(committedOffsets)`.
2692 fn on_commit(&self, offsets: &HashMap<TopicPartition, OffsetAndMetadata>) {
2693 (self.on_commit)(offsets);
2694 }
2695
2696 /// Translates Java's
2697 /// `private CompletableFuture<Map<TopicPartition, OffsetAndMetadata>> commit(CommitEvent)`
2698 /// up to the offsets-ready wait,
2699 /// which the caller runs. Neither the owning consumer nor a handle
2700 /// processes background events during that wait: as in Java, only
2701 /// `poll()` and `unsubscribe()` deliver them.
2702 ///
2703 /// Checks `group.id`, runs the queued commit callbacks, and returns
2704 /// `None` for empty explicit offsets (Java's `completedFuture(null)`);
2705 /// otherwise submits the event and returns its offsets-ready and result
2706 /// receivers.
2707 async fn submit_commit(
2708 &self,
2709 group_id: Option<&str>,
2710 commit_event: CommitEventKind,
2711 application_event_handler: &ApplicationEventHandler,
2712 time: &dyn Time,
2713 ) -> Result<Option<(OffsetsReadyReceiver, CommitResultReceiver)>, Error> {
2714 return_error_if_group_id_not_defined(group_id)?;
2715 // Java's `executeCallbacks()` has no try/catch: an exception a
2716 // callback throws propagates out of `commit()`, and so out of
2717 // `commitSync` / `commitAsync`.
2718 self.invoke_pending_callbacks().await?;
2719
2720 // Java's `if (event.offsets().isPresent() && event.offsets().get().isEmpty())`
2721 // short-circuits with `completedFuture(null)`.
2722 if let Some(map) = commit_event.offsets()
2723 && map.is_empty()
2724 {
2725 return Ok(None);
2726 }
2727
2728 let now_ms = time.milliseconds();
2729 let deadline_ms = commit_event.deadline_ms();
2730 let (handle, receiver, _erased) =
2731 CompletableEvent::make_completable_event::<HashMap<TopicPartition, OffsetAndMetadata>>(deadline_ms);
2732 let (offsets_ready_handle, offsets_ready_rx, _erased_or) =
2733 CompletableEvent::make_completable_event::<()>(deadline_ms);
2734
2735 // Build and enqueue the matching event variant.
2736 let event = match commit_event {
2737 CommitEventKind::Async { offsets } => {
2738 ApplicationEvent::CommitAsync { handle, offsets_ready: offsets_ready_handle, offsets }
2739 },
2740 CommitEventKind::Sync { offsets, .. } => {
2741 ApplicationEvent::CommitSync { handle, offsets_ready: offsets_ready_handle, offsets }
2742 },
2743 };
2744 application_event_handler.add(event, now_ms)?;
2745 Ok(Some((offsets_ready_rx, receiver)))
2746 }
2747
2748 /// Translates Java's `lastPendingAsyncCommit = commit(asyncCommitEvent).whenComplete(...)`
2749 /// in `AsyncKafkaConsumer#commitAsync(Optional,OffsetCommitCallback)`, after
2750 /// `commit(...)` returned `receiver` (`None` for the empty explicit
2751 /// offsets Java completes at once).
2752 fn register_async_commit(
2753 &self,
2754 receiver: Option<CommitResultReceiver>,
2755 callback: Option<Arc<dyn crate::consumer::OffsetCommitCallback>>,
2756 ) {
2757 let Some(receiver) = receiver else {
2758 // Empty explicit offsets: Java's `commit()` returns an
2759 // already-completed future (`org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer#commit`),
2760 // so `whenComplete` runs inline on the calling thread and the
2761 // result replaces `lastPendingAsyncCommit`
2762 // (`AsyncKafkaConsumer#commitAsync(Optional,OffsetCommitCallback)`). Enqueue the callbacks here and store
2763 // an already-completed pending commit, without chaining behind earlier in-flight commits: a later
2764 // `commitSync` / `close` does not wait for them, and they still
2765 // enqueue their own callbacks when they complete.
2766 //
2767 // The displaced in-flight commit moves to `ordering_tail` rather
2768 // than being dropped, so the next non-empty commit still chains
2769 // behind it. If the tail is already set, an earlier empty commit
2770 // has displaced it and the pending commit is an already-completed
2771 // one.
2772 self.invoker.enqueue_interceptor_invocation(HashMap::new());
2773 if let Some(cb) = callback {
2774 self.invoker.enqueue_user_callback_invocation(cb, HashMap::new(), None);
2775 }
2776 let mut pending = self.lock_pending();
2777 if pending.ordering_tail.is_none() {
2778 pending.ordering_tail = pending.last.take();
2779 }
2780 pending.last = Some(PendingAsyncCommit::completed());
2781 return;
2782 };
2783
2784 // Java: `lastPendingAsyncCommit = commit(asyncCommitEvent).whenComplete(...)`
2785 // — the resulting future is stored so a later `commitSync` / `close`
2786 // can wait for it to complete. The Rust analog spawns the
2787 // continuation, which invokes the callback chain and then completes
2788 // the pending commit.
2789 //
2790 // Each continuation is a separate task that the runtime may schedule
2791 // in any order, so it awaits its predecessor before enqueueing. Hence
2792 // callbacks are enqueued in commit order, as the `commitAsync` javadoc
2793 // requires (`org.apache.kafka.clients.consumer.KafkaConsumer#commitAsync(OffsetCommitCallback)`), and the
2794 // pending commit completing implies every earlier callback is enqueued, which is why
2795 // `commit_sync` and `close` await only the most recent commit. The
2796 // predecessor is the commit an empty-offsets commit displaced, if any
2797 // (the empty commit itself has already enqueued its callbacks).
2798 let (pending_tx, pending_commit) = PendingAsyncCommit::incomplete();
2799 let previous_pending = {
2800 let mut pending = self.lock_pending();
2801 let previous = pending.ordering_tail.take().or_else(|| pending.last.take());
2802 pending.last = Some(pending_commit);
2803 previous
2804 };
2805 let invoker = Arc::clone(&self.invoker);
2806 tokio::spawn(async move {
2807 let result = receiver.await;
2808 if let Some(mut previous) = previous_pending {
2809 // Done or dropped: the predecessor has enqueued its callbacks,
2810 // or its task is gone. Proceed in either case.
2811 previous.wait().await;
2812 }
2813 match result {
2814 Ok(Ok(committed)) => {
2815 // Java: `if (throwable == null)
2816 // offsetCommitCallbackInvoker.enqueueInterceptorInvocation(committedOffsets)`.
2817 invoker.enqueue_interceptor_invocation(committed.clone());
2818 if let Some(cb) = callback {
2819 invoker.enqueue_user_callback_invocation(cb, committed, None);
2820 }
2821 },
2822 Ok(Err(err)) => {
2823 if let Some(cb) = callback {
2824 invoker.enqueue_user_callback_invocation(cb, HashMap::new(), Some(err.clone()));
2825 } else {
2826 log::error!("Offset commit failed: {err}");
2827 }
2828 },
2829 Err(_recv_err) => {
2830 log::error!("commit_async receiver dropped without completion");
2831 },
2832 }
2833 pending_tx.send_replace(true);
2834 });
2835 }
2836
2837 /// Translates Java's
2838 /// `private void awaitPendingAsyncCommitsAndExecuteCommitCallbacks(Timer timer, boolean enableWakeup)`
2839 /// (`AsyncKafkaConsumer#awaitPendingAsyncCommitsAndExecuteCommitCallbacks`).
2840 ///
2841 /// If there is a pending async commit, await it (bounded by the
2842 /// deadline) and then drain the callback invoker queue. The
2843 /// `enable_wakeup` flag mirrors Java's
2844 /// `wakeupTrigger.setActiveTask(futureToAwait)` — `true` makes a
2845 /// concurrent `wakeup()` interrupt the wait.
2846 ///
2847 /// The wait is a plain deadline-bounded await on the pending commit — it
2848 /// does NOT drain or process background events. Java only waits on the
2849 /// commit future via `ConsumerUtils.getResult(futureToAwait, timer)` and
2850 /// never calls `processBackgroundEvents`, so it never invokes
2851 /// rebalance-listener callbacks here. Routing this wait through
2852 /// `process_background_events_until` (which invokes pending
2853 /// `RebalanceListenerCallbackNeeded` callbacks) is NOT faithful: a
2854 /// failing rebalance listener would propagate its error out of
2855 /// `close()`, which Java never does.
2856 async fn await_pending_async_commits_and_execute_commit_callbacks(
2857 &self,
2858 wakeup_trigger: &WakeupTrigger,
2859 time: &dyn Time,
2860 deadline_ms: i64,
2861 enable_wakeup: bool,
2862 ) -> Result<(), Error> {
2863 // Java clears `lastPendingAsyncCommit` only *after*
2864 // `getResult(futureToAwait, timer)` returns normally; its `finally`
2865 // deliberately does NOT clear it, so a timeout or a wakeup leaves the
2866 // pending commit in place and a later `commit_sync` still waits for
2867 // it. Waiting on a clone leaves the slot untouched on every exit; the
2868 // success exit clears it unless a newer commit replaced it meanwhile.
2869 let awaited = self.lock_pending().last.clone();
2870 if let Some(mut pending_commit) = awaited.clone() {
2871 // A deadline-bounded await in short `tokio::time::timeout` slices,
2872 // so the deadline is driven by the mock-clock-safe `time`.
2873 let mut entering = true;
2874 loop {
2875 // Java's `whenComplete` completes `futureToAwait` at once for a
2876 // commit already done, before `setActiveTask(futureToAwait)`,
2877 // which then keeps the result and leaves a pending wakeup
2878 // armed for the next call (`WakeupTrigger.java`): on entry a
2879 // done commit wins over the wakeup.
2880 if entering && pending_commit.is_done() {
2881 break;
2882 }
2883 entering = false;
2884 // §11: observe a pending wakeup at the top of each iteration
2885 // when wakeups are enabled (Java's
2886 // `wakeupTrigger.setActiveTask(futureToAwait)`): a wakeup that
2887 // came while the commit was pending fails the wait.
2888 if enable_wakeup && let Err(err) = wakeup_trigger.maybe_trigger_wakeup() {
2889 wakeup_trigger.rotate();
2890 return Err(err);
2891 }
2892
2893 // Java's `getResult(future, timer)` is `future.get(remainingMs)`,
2894 // which returns a completed future's result even once the timer
2895 // has expired: check completion before the deadline.
2896 if pending_commit.is_done() {
2897 break;
2898 }
2899 let remaining = deadline_ms.saturating_sub(time.milliseconds()).max(0);
2900 if remaining <= 0 {
2901 return Err(Error::timeout(PENDING_ASYNC_COMMIT_TIMEOUT_MSG.to_string()));
2902 }
2903 let wait = std::cmp::min(remaining, 100) as u64;
2904
2905 // §11: race the wait against the wakeup token's cancellation so
2906 // `wakeup()` from another task interrupts it immediately.
2907 let wait_fut = pending_commit.wait();
2908 if enable_wakeup {
2909 let tok = wakeup_trigger.current_token();
2910 tokio::select! {
2911 biased;
2912 _ = tok.cancelled() => {
2913 // Loop top surfaces Error::Wakeup via
2914 // maybe_trigger_wakeup + rotate.
2915 },
2916 res = tokio::time::timeout(Duration::from_millis(wait), wait_fut) => {
2917 if res.is_ok() {
2918 break;
2919 }
2920 },
2921 }
2922 } else if tokio::time::timeout(Duration::from_millis(wait), wait_fut).await.is_ok() {
2923 break;
2924 }
2925 }
2926 // Java: `lastPendingAsyncCommit = null`.
2927 let mut pending = self.lock_pending();
2928 if let (Some(last), Some(awaited)) = (pending.last.as_ref(), awaited.as_ref())
2929 && last.is_same(awaited)
2930 {
2931 pending.last = None;
2932 }
2933 }
2934 // Java: `offsetCommitCallbackInvoker.executeCallbacks()`, whose
2935 // callback exception propagates to `commitSync` (and, swallowed into
2936 // its first error, to `close`).
2937 self.invoke_pending_callbacks().await
2938 }
2939
2940 /// Whether an async commit is pending, for tests.
2941 #[cfg(test)]
2942 fn has_last_pending_async_commit(&self) -> bool {
2943 self.lock_pending().last.is_some()
2944 }
2945
2946 /// The last pending async commit, for tests.
2947 #[cfg(test)]
2948 fn last_pending_async_commit(&self) -> Option<PendingAsyncCommit> {
2949 self.lock_pending().last.clone()
2950 }
2951}
2952
2953/// Return the result of a `PartitionsRemoved` callback to the background
2954/// task — the notify half of Java's
2955/// `AsyncKafkaConsumer.invokeRebalanceCallbackAndNotifyBackgroundThread`.
2956///
2957/// With `ack_through_application_events` (the callback a leave enqueued),
2958/// the ack goes back inside a `ConsumerRebalanceListenerCallbackCompleted`
2959/// application event, as Java's
2960/// `applicationEventHandler.add(new ConsumerRebalanceListenerCallbackCompletedEvent(...))`
2961/// does: the background task then applies it after every event enqueued
2962/// before it — including one the listener submitted and stopped waiting
2963/// for — and before every event enqueued after it. Sending on `ack`
2964/// directly would let the background task see the result before such an
2965/// earlier, still-queued event. `add` wakes the background task. If the
2966/// background task is gone, the event (and the ack in it) is dropped.
2967///
2968/// Otherwise (the reconcile and release callbacks) the result is sent on
2969/// `ack` directly and the background task is woken through the
2970/// application-event `Notify` (§31 step 4): never through the user-facing
2971/// wakeup trigger, which would make the caller's next blocking call return
2972/// a spurious `Wakeup`.
2973pub(crate) fn notify_background_task_of_rebalance_callback(
2974 application_event_handler: &ApplicationEventHandler,
2975 method_name: crate::consumer::internals::ConsumerRebalanceListenerMethodName,
2976 ack: tokio::sync::oneshot::Sender<Result<(), Error>>,
2977 ack_through_application_events: bool,
2978 result: Result<(), Error>,
2979 now_ms: i64,
2980) {
2981 if ack_through_application_events {
2982 let _ = application_event_handler.add(
2983 ApplicationEvent::ConsumerRebalanceListenerCallbackCompleted { method_name, error: result.err(), ack },
2984 now_ms,
2985 );
2986 } else {
2987 let _ = ack.send(result);
2988 application_event_handler.wake_background_task();
2989 }
2990}
2991
2992/// Owns a `PartitionsRemoved` callback's ack while the app side runs the
2993/// listener, so a leave's ack is still returned through the application-event
2994/// queue if the caller's future is dropped mid-listener.
2995///
2996/// DoD §7: no Java counterpart. A Java app thread cannot be cancelled in the
2997/// middle of a callback; the closest case is a listener cut short by a
2998/// `WakeupException` / `InterruptException`, which
2999/// `ConsumerRebalanceListenerInvoker` rethrows and
3000/// `AsyncKafkaConsumer.invokeRebalanceCallbacks` catches into the event's
3001/// error. `invokeRebalanceCallbackAndNotifyBackgroundThread` then still
3002/// enqueues the `ConsumerRebalanceListenerCallbackCompletedEvent` behind
3003/// everything the listener had enqueued. A Rust future dropped while the
3004/// listener awaits would otherwise drop the ack unsent, and the background
3005/// task would apply the leave ahead of the listener's still-queued ops (a
3006/// revocation `commit_sync()` would then commit nothing). On drop this guard
3007/// enqueues the completed event with an [`Error::Interrupt`], the error Java
3008/// carries for an interrupted callback. `ApplicationEventHandler::add` is
3009/// sync and never blocks, so it is safe to call from `Drop`.
3010///
3011/// Only a leave's ack (`ack_through_application_events`) is returned on
3012/// drop; a reconcile or release ack is dropped unsent, as before.
3013struct PartitionsRemovedAckGuard {
3014 application_event_handler: Arc<ApplicationEventHandler>,
3015 time: Arc<dyn Time>,
3016 method_name: crate::consumer::internals::ConsumerRebalanceListenerMethodName,
3017 ack: Option<tokio::sync::oneshot::Sender<Result<(), Error>>>,
3018 ack_through_application_events: bool,
3019}
3020
3021impl PartitionsRemovedAckGuard {
3022 /// Returns the listener's `result` through
3023 /// [`notify_background_task_of_rebalance_callback`].
3024 fn complete(mut self, result: Result<(), Error>) {
3025 if let Some(ack) = self.ack.take() {
3026 notify_background_task_of_rebalance_callback(
3027 &self.application_event_handler,
3028 self.method_name,
3029 ack,
3030 self.ack_through_application_events,
3031 result,
3032 self.time.milliseconds(),
3033 );
3034 }
3035 }
3036}
3037
3038impl Drop for PartitionsRemovedAckGuard {
3039 fn drop(&mut self) {
3040 if !self.ack_through_application_events {
3041 return;
3042 }
3043 if let Some(ack) = self.ack.take() {
3044 notify_background_task_of_rebalance_callback(
3045 &self.application_event_handler,
3046 self.method_name,
3047 ack,
3048 true,
3049 Err(Error::Interrupt(crate::common::errors::InterruptError::new(format!(
3050 "The {} callback was interrupted: the call running it was cancelled",
3051 self.method_name.fully_qualified_method_name()
3052 )))),
3053 self.time.milliseconds(),
3054 );
3055 }
3056 }
3057}
3058
3059/// Format a slice of [`TopicPartition`] in Java's `Set.toString()`
3060/// shape: `[topic-0, topic-1]`. Used in user-facing error messages
3061/// where Java would format a `Set<TopicPartition>` directly
3062/// (Issue 19 — DoD §3 exact-message contract).
3063///
3064/// Rust's `{:?}` produces `[TopicPartition { topic: "t", partition: 0 }]`
3065/// which diverges from Java's user-visible string.
3066fn format_partitions_for_display(partitions: &[TopicPartition]) -> String {
3067 let mut out = String::from("[");
3068 for (i, tp) in partitions.iter().enumerate() {
3069 if i > 0 {
3070 out.push_str(", ");
3071 }
3072 use std::fmt::Write as _;
3073 let _ = write!(out, "{tp}");
3074 }
3075 out.push(']');
3076 out
3077}
3078
3079/// Java: `private ConsumerGroupMetadata initializeConsumerGroupMetadata(final String groupId, final Optional<String> groupInstanceId)`
3080/// (`AsyncKafkaConsumer#initializeConsumerGroupMetadata`).
3081///
3082/// Builds the initial [`ConsumerGroupMetadata`] for a consumer that has a
3083/// `group.id`: `UNKNOWN_GENERATION_ID` (-1) / `UNKNOWN_MEMBER_ID` ("") — the
3084/// generation / member placeholders that stay in place until the first
3085/// heartbeat response updates them (see
3086/// [`ConsumerStateNotifier::update_group_metadata`]) — plus the configured
3087/// `group.instance.id`. Java calls the deprecated 4-arg
3088/// `ConsumerGroupMetadata` constructor here under
3089/// `@SuppressWarnings("removal")`; the Rust client has no public constructor —
3090/// the crate-internal [`ConsumerGroupMetadataImpl`] translates it — so there is
3091/// nothing to suppress.
3092fn initialize_consumer_group_metadata(group_id: &str, group_instance_id: Option<&str>) -> ConsumerGroupMetadataImpl {
3093 ConsumerGroupMetadataImpl::with_generation_id_member_id_group_instance_id(
3094 group_id,
3095 -1, // org.apache.kafka.common.requests.JoinGroupRequest#UNKNOWN_GENERATION_ID
3096 "", // org.apache.kafka.common.requests.JoinGroupRequest#UNKNOWN_MEMBER_ID
3097 group_instance_id.map(str::to_string),
3098 )
3099}
3100
3101/// Java: `private Optional<ConsumerGroupMetadata> initializeGroupMetadata(final String groupId, final Optional<String> groupInstanceId)`
3102/// (`AsyncKafkaConsumer#initializeGroupMetadata(String,Optional)`).
3103///
3104/// Seeds the group-metadata slot at construction: a `None` group id yields
3105/// `None` (an assignment-only consumer is never "in a group"); otherwise
3106/// `Some(initialize_consumer_group_metadata(group_id, group_instance_id))`.
3107/// Java also rejects a present-but-empty `group.id` with
3108/// `InvalidGroupIdException` here; the Rust constructor performs
3109/// that rejection earlier, when it computes `group_id: Option<String>` (the
3110/// `Some("") => Err(Error::invalid_group_id(...))` arm), so by the time this
3111/// helper runs the group id is already `None` or non-empty and no re-check is
3112/// needed.
3113fn initialize_group_metadata(
3114 group_id: Option<&str>,
3115 group_instance_id: Option<&str>,
3116) -> Option<Arc<ConsumerGroupMetadataImpl>> {
3117 group_id.map(|id| Arc::new(initialize_consumer_group_metadata(id, group_instance_id)))
3118}
3119
3120/// `MemberStateListener` implementation that bridges the membership
3121/// manager's state-change notifications back to the consumer's app-side
3122/// caches: `group_metadata` and `group_assignment_snapshot`.
3123///
3124/// Mirrors Java's anonymous-inner-class `memberStateListener` at
3125/// `AsyncKafkaConsumer#memberStateListener`. The callbacks are invoked
3126/// synchronously from the bg task (membership-manager reconciliation
3127/// step); they only touch their two `Arc<Mutex<…>>` fields and never
3128/// call back into the consumer or take other locks, so the brief
3129/// critical sections are §16-safe.
3130///
3131/// The listener does **not** own a `group_id` / `group_instance_id`: those
3132/// live in the [`ConsumerGroupMetadata`] already stored in `group_metadata`
3133/// (seeded at construction — see `initialize_group_metadata`). Java reads
3134/// them back off the old value inside `updateAndGet`
3135/// (`AsyncKafkaConsumer#updateGroupMetadata`) rather than off a field, and so
3136/// do we.
3137pub(crate) struct ConsumerStateNotifier {
3138 /// Shared with [`AsyncKafkaConsumer::group_metadata`].
3139 group_metadata: Arc<Mutex<Option<Arc<ConsumerGroupMetadataImpl>>>>,
3140 /// Shared with [`AsyncKafkaConsumer::group_assignment_snapshot`].
3141 group_assignment_snapshot: Arc<Mutex<HashSet<TopicPartition>>>,
3142 /// Shared with [`AsyncKafkaConsumer::has_pending_reconciliation`]
3143 /// (AK 4.3.1, KAFKA-20106). Set to `true` when the member enters
3144 /// `RECONCILING`, `false` otherwise, via
3145 /// [`MemberStateListener::on_member_state_change`] fired on the bg task.
3146 /// Read by the app-side `collect_fetch` to decide whether to wait for the
3147 /// reconciliation check before returning buffered records. Java:
3148 /// `setHasPendingReconciliation(memberState == MemberState.RECONCILING)`.
3149 has_pending_reconciliation: Arc<AtomicBool>,
3150}
3151
3152impl ConsumerStateNotifier {
3153 /// Constructor. The `group_metadata` / `group_assignment_snapshot`
3154 /// Arcs are owned by both the notifier and the consumer.
3155 pub(crate) fn new(
3156 group_metadata: Arc<Mutex<Option<Arc<ConsumerGroupMetadataImpl>>>>,
3157 group_assignment_snapshot: Arc<Mutex<HashSet<TopicPartition>>>,
3158 has_pending_reconciliation: Arc<AtomicBool>,
3159 ) -> Self {
3160 Self { group_metadata, group_assignment_snapshot, has_pending_reconciliation }
3161 }
3162
3163 /// Java: `private void updateGroupMetadata(Optional<Integer> memberEpoch, String memberId)`
3164 /// (`AsyncKafkaConsumer#updateGroupMetadata`).
3165 ///
3166 /// Updates the cached [`ConsumerGroupMetadata`] to carry the new
3167 /// epoch / member-id. Java is an `updateAndGet` over an
3168 /// `AtomicReference<Optional<ConsumerGroupMetadata>>` whose lambda
3169 /// `map`s the *old* value: it keeps the existing
3170 /// `group.id` and `group.instance.id` and overwrites
3171 /// only the generation and member id. A `None` slot stays `None`
3172 /// (Java's `Optional.map` short-circuits) — the slot is seeded once at
3173 /// construction, when `group.id` is present (`initialize_group_metadata`,
3174 /// in Java's constructor), and an assignment-only consumer never populates it.
3175 fn update_group_metadata(&self, member_epoch: Option<i32>, member_id: &str) {
3176 let Some(epoch) = member_epoch else {
3177 // Java's `memberEpoch.ifPresent(...)` short-circuits when
3178 // None — no metadata mutation.
3179 return;
3180 };
3181 let mut guard = self.group_metadata.lock().unwrap();
3182 if let Some(old) = guard.as_ref() {
3183 // Map over the old value: keep its group_id / group_instance_id,
3184 // overwrite generation + member id. Java's `updateGroupMetadata`.
3185 let next = ConsumerGroupMetadataImpl::with_generation_id_member_id_group_instance_id(
3186 old.group_id().to_string(),
3187 epoch,
3188 member_id.to_string(),
3189 old.group_instance_id().map(str::to_string),
3190 );
3191 *guard = Some(Arc::new(next));
3192 }
3193 // A `None` slot stays `None`: Java's `oldGroupMetadataOptional.map(...)`
3194 // short-circuits on empty, so an assignment-only consumer (no
3195 // `group.id`) is never given group metadata by a heartbeat.
3196 }
3197
3198 /// Returns a clone of the shared `has_pending_reconciliation` flag so
3199 /// the [`AsyncKafkaConsumer`] can read the same atomic the notifier
3200 /// writes from `on_member_state_change`.
3201 pub(crate) fn has_pending_reconciliation_handle(&self) -> Arc<AtomicBool> {
3202 Arc::clone(&self.has_pending_reconciliation)
3203 }
3204
3205 /// Java: `private void resetGroupMetadata()`
3206 /// (`AsyncKafkaConsumer#resetGroupMetadata`).
3207 ///
3208 /// Resets the cached [`ConsumerGroupMetadata`] to the
3209 /// `UNKNOWN_GENERATION_ID` / `UNKNOWN_MEMBER_ID` defaults,
3210 /// preserving the original `groupId` and `groupInstanceId`. Called
3211 /// by [`AsyncKafkaConsumer::unsubscribe`] after the unsubscribe
3212 /// event completes, matching Java's
3213 /// `processBackgroundEvents(...)` → `resetGroupMetadata()` sequence
3214 /// (in Java's `unsubscribe()`).
3215 ///
3216 /// Mirrors Java's `updateAndGet` over the Optional: the
3217 /// lambda `map`s the old value through `initializeConsumerGroupMetadata`,
3218 /// so a `Some` slot is rebuilt with UNKNOWN epoch / member but the same
3219 /// group id + instance id, and a `None` slot (assignment-only consumer
3220 /// that never populated the cache) stays `None`.
3221 pub(crate) fn reset_group_metadata(&self) {
3222 let mut guard = self.group_metadata.lock().unwrap();
3223 if let Some(old) = guard.as_ref() {
3224 // Java: `initializeConsumerGroupMetadata(old.groupId(),
3225 // old.groupInstanceId())` — fresh metadata with UNKNOWN epoch +
3226 // member, preserving the old group_id + group_instance_id.
3227 *guard = Some(Arc::new(initialize_consumer_group_metadata(
3228 old.group_id(),
3229 old.group_instance_id(),
3230 )));
3231 }
3232 // Java's `oldGroupMetadataOptional.map(...)` short-circuits when
3233 // the slot is empty (assignment-only consumer never populated the
3234 // cache); we mirror that by leaving the slot as None.
3235 }
3236}
3237
3238impl MemberStateListener for ConsumerStateNotifier {
3239 /// Java: `memberStateListener.onMemberEpochUpdated(memberEpoch, memberId)`
3240 /// (`AsyncKafkaConsumer#memberStateListener`).
3241 fn on_member_epoch_updated(&self, member_epoch: Option<i32>, member_id: &str) {
3242 self.update_group_metadata(member_epoch, member_id);
3243 }
3244
3245 /// Java: `memberStateListener.onGroupAssignmentUpdated(partitions)`
3246 /// (`AsyncKafkaConsumer#memberStateListener`) → `setGroupAssignmentSnapshot`
3247 /// (`AsyncKafkaConsumer#setGroupAssignmentSnapshot`). Snapshots the assignment so `runRebalanceCallbacksOnClose`
3248 /// can drive listener callbacks over the **group**-assigned partitions
3249 /// specifically (manual `assign(...)` partitions are intentionally
3250 /// excluded).
3251 fn on_group_assignment_updated(&self, partitions: &HashSet<TopicPartition>) {
3252 let mut guard = self.group_assignment_snapshot.lock().unwrap();
3253 *guard = partitions.clone();
3254 }
3255
3256 /// Java: `memberStateListener.onMemberStateChange(memberState)`
3257 /// (`AsyncKafkaConsumer.java`) →
3258 /// `setHasPendingReconciliation(memberState == MemberState.RECONCILING)`.
3259 fn on_member_state_change(&self, member_state: MemberState) {
3260 self.has_pending_reconciliation
3261 .store(member_state == MemberState::Reconciling, Ordering::Release);
3262 }
3263}
3264
3265/// Components handed to [`AsyncKafkaConsumer::new_with_thread`]: the
3266/// per-RM container, metadata, subscriptions, application-event handle,
3267/// reaper, wakeup trigger, etc. Constructed by the production factory
3268/// (commit (7)) and by tests directly.
3269///
3270/// Bundling these into a struct keeps the ctor signature manageable as
3271/// the Java ctor has 14 already-constructed dependencies.
3272pub(crate) struct AsyncKafkaConsumerComponents<K: Send + Sync + 'static, V: Send + Sync + 'static> {
3273 pub config: ConsumerConfig,
3274 pub client_id: Arc<str>,
3275 pub group_id: Option<String>,
3276 pub subscriptions: Arc<Mutex<SubscriptionState>>,
3277 pub metadata: Arc<ConsumerMetadata>,
3278 pub request_managers: Arc<std::sync::Mutex<RequestManagers>>,
3279 pub background_event_rx: mpsc::UnboundedReceiver<BackgroundEventEnvelope>,
3280 pub application_event_handler: Arc<ApplicationEventHandler>,
3281 pub completable_event_reaper: Arc<std::sync::Mutex<CompletableEventReaper>>,
3282 pub max_time_to_wait_ms: Arc<AtomicI64>,
3283 pub wakeup_trigger: WakeupTrigger,
3284 pub network_thread_close: NetworkThreadCloseHandle,
3285 pub fetch_buffer: Arc<FetchBuffer>,
3286 pub fetch_collector: Arc<FetchCollector<K, V>>,
3287 /// The metrics registry. Owned here so Phase M7 can expose the public
3288 /// `metrics()` accessor over the same registry the fetch path records into.
3289 pub metrics: Arc<Metrics>,
3290 /// Consumer-level poll/commit timing metrics
3291 /// (`KafkaConsumerMetrics`), recording into the same `metrics` registry.
3292 pub kafka_consumer_metrics: Arc<KafkaConsumerMetrics>,
3293 /// Async-consumer background-task / event-queue metrics
3294 /// (`AsyncConsumerMetrics`), recording into the same `metrics` registry.
3295 pub async_consumer_metrics: Arc<AsyncConsumerMetrics>,
3296 /// The fetch metrics manager (Java's test constructor takes it since
3297 /// KAFKA-19542).
3298 pub fetch_metrics_manager: Arc<FetchMetricsManager>,
3299 /// The rebalance-callback metrics manager (Java's test constructor takes it
3300 /// since KAFKA-19542).
3301 pub rebalance_callback_metrics_manager: Arc<RebalanceCallbackMetricsManager>,
3302 /// Shared mirror of the background-event queue depth.
3303 pub background_event_queue_size: Arc<AtomicI64>,
3304 pub rebalance_listener_invoker: ConsumerRebalanceListenerInvoker,
3305 pub offset_commit_callback_invoker: Arc<OffsetCommitCallbackInvoker<K, V>>,
3306 pub interceptors: Arc<Mutex<ConsumerInterceptors<K, V>>>,
3307 pub isolation_level: IsolationLevel,
3308 pub time: Arc<dyn Time>,
3309 /// Shared `Arc<Mutex<Option<Arc<ConsumerGroupMetadataImpl>>>>` slot. Java has a
3310 /// **single** `AtomicReference<Optional<ConsumerGroupMetadata>>` field
3311 /// (`AsyncKafkaConsumer#groupMetadata`); the same slot is referenced by
3312 /// the `MemberStateListener` registered on the membership manager AND
3313 /// read by the public `groupMetadata()` accessor. The Rust
3314 /// translation enforces that single-source-of-truth contract by
3315 /// requiring the production ctor to build the slot once and pass it
3316 /// through here — the same Arc is then registered on
3317 /// `ConsumerMembershipManager` via [`Self::state_notifier`] AND
3318 /// stored on the consumer struct's `group_metadata` field.
3319 pub group_metadata: Arc<Mutex<Option<Arc<ConsumerGroupMetadataImpl>>>>,
3320 /// Shared `Arc<Mutex<HashSet<TopicPartition>>>` slot mirroring
3321 /// Java's `groupAssignmentSnapshot` field
3322 /// (`AsyncKafkaConsumer#groupAssignmentSnapshot`). Same single-source-of-truth
3323 /// contract as [`Self::group_metadata`] — the production ctor builds
3324 /// once and threads the Arc through both
3325 /// `ConsumerStateNotifier::on_group_assignment_updated` (writer) and
3326 /// `AsyncKafkaConsumer::run_rebalance_callbacks_on_close` (reader).
3327 pub group_assignment_snapshot: Arc<Mutex<HashSet<TopicPartition>>>,
3328 /// The single `MemberStateListener` instance (Java
3329 /// `AsyncKafkaConsumer#memberStateListener`, an anonymous-inner-class
3330 /// `memberStateListener`) that writes to
3331 /// [`Self::group_metadata`] and [`Self::group_assignment_snapshot`].
3332 /// The production ctor clones this Arc and registers it on
3333 /// `ConsumerMembershipManager` BEFORE the bg-task spawn; the
3334 /// consumer struct stores it for the `state_notifier()` accessor used
3335 /// in close-time `reset_group_metadata()` and by tests. Tests can
3336 /// register `consumer.state_notifier()` on a custom membership
3337 /// manager when they bypass the production ctor.
3338 pub state_notifier: Arc<ConsumerStateNotifier>,
3339 /// The shared [`PositionsValidator`] (Java `AsyncKafkaConsumer#positionsValidator`).
3340 /// The SAME `Arc` must also have been passed to the
3341 /// `OffsetsRequestManager` in `request_managers`, or the app-side skip
3342 /// decision reads state the background side never writes.
3343 pub positions_validator: Arc<PositionsValidator>,
3344}
3345
3346/// Closes the consumer's deserializers when construction fails after they were
3347/// wrapped, as the `catch (Throwable t)` of Java's `AsyncKafkaConsumer` constructor does
3348/// : it calls `close(Duration.ZERO,
3349/// LEAVE_GROUP, true)`, whose `closeQuietly(deserializers, "consumer
3350/// deserializers", firstException)`
3351/// (`AsyncKafkaConsumer#close(Duration,GroupMembershipOperation,boolean)`) closes them and only logs a failure,
3352/// since `swallowException` is `true`.
3353///
3354/// Rust-only (DoD #7): an RAII stand-in for that `catch`. `new_inner` returns
3355/// through many `?`s after the wrap; holding the deserializers in this guard
3356/// covers every one of them, and any step added later. The guard is declared
3357/// before every other component built after the wrap, so on an error those are
3358/// dropped first and the guard's `Arc` is the only one left when it closes the
3359/// deserializers. Nothing else of Java's `close(Duration.ZERO, ..)` applies:
3360/// `new_inner` disarms the guard when it hands the deserializers to the
3361/// `FetchCollector`, after its last fallible step, and only then spawns the
3362/// background task's `std::thread`. So no task or socket exists on any error
3363/// path, and the rest is released on drop.
3364///
3365/// `new` is synchronous and [`Deserializers::close`] is async, so the close
3366/// runs through [`Utils::close_quietly_from_sync`], which the producer's
3367/// constructor shares for its serializers.
3368struct CloseDeserializersOnError<K: 'static, V: 'static> {
3369 deserializers: Option<Arc<Deserializers<K, V>>>,
3370}
3371
3372impl<K, V> CloseDeserializersOnError<K, V>
3373where
3374 K: Send + Sync + 'static,
3375 V: Send + Sync + 'static,
3376{
3377 fn new(deserializers: Arc<Deserializers<K, V>>) -> Self {
3378 Self { deserializers: Some(deserializers) }
3379 }
3380
3381 /// Construction succeeded: hands the deserializers on, so nothing closes
3382 /// them here.
3383 fn disarm(mut self) -> Arc<Deserializers<K, V>> {
3384 self.deserializers.take().expect("the guard is disarmed once")
3385 }
3386}
3387
3388impl<K: 'static, V: 'static> Drop for CloseDeserializersOnError<K, V> {
3389 fn drop(&mut self) {
3390 let Some(deserializers) = self.deserializers.take() else {
3391 return;
3392 };
3393 let Some(mut deserializers) = Arc::into_inner(deserializers) else {
3394 // Unreachable: every clone belongs to a component built after the
3395 // guard, all of which are dropped before it.
3396 log::error!("Failed to close consumer deserializers: they are still shared");
3397 return;
3398 };
3399 Utils::close_quietly_from_sync(
3400 Box::pin(async move {
3401 if let Err(error) = deserializers.close().await {
3402 // Java's `closeQuietly`
3403 // (`org.apache.kafka.common.utils.Utils#closeQuietly(AutoCloseable,String,AtomicReference)`).
3404 log::error!("Failed to close consumer deserializers: {error}");
3405 }
3406 }),
3407 "consumer deserializers",
3408 );
3409 }
3410}
3411
3412impl<K, V> AsyncKafkaConsumer<K, V>
3413where
3414 K: Send + Sync + 'static,
3415 V: Send + Sync + 'static,
3416{
3417 /// Constructs an `AsyncKafkaConsumer` from pre-built components.
3418 ///
3419 /// Test-visible seam. The production factory in
3420 /// `consumer/kafka_consumer.rs::KafkaConsumer::new` flows through
3421 /// `AsyncKafkaConsumer::new` (Phase 12 commit 4/N), which builds the
3422 /// dependency closure and delegates to this method. Tests build
3423 /// their own components (typically with a `MockClient`-backed
3424 /// `ConsumerNetworkThread`) and call this directly.
3425 ///
3426 /// Mirrors Java's test-visible constructor at
3427 /// `AsyncKafkaConsumer(LogContext, String, Deserializers, ...)` (the 20-arg form), with the
3428 /// metrics / telemetry parameters dropped per Phase 11 PLAN.md
3429 /// deferrals.
3430 /// Production constructor — translates Java's primary
3431 /// `AsyncKafkaConsumer(ConsumerConfig, Deserializer<K>, Deserializer<V>,
3432 /// Optional<StreamsRebalanceData>)`
3433 /// in three buildable slices per Phase-12 PLAN.md:
3434 ///
3435 /// - **Commit (1/N) — this method:** Builds channels, subscriptions,
3436 /// metadata, `NetworkClient`, `NetworkClientDelegate`,
3437 /// `BackgroundEventHandler`, `FetchBuffer`, `FetchConfig`,
3438 /// `Deserializers`, `ConsumerInterceptors`, and the
3439 /// `OffsetCommitCallbackInvoker`. Returns
3440 /// `Err(Error::unsupported_version(...))` at the end because
3441 /// the `RequestManagers` (commit (2/N)) and bg-task spawn (commit
3442 /// (3/N)) are not yet wired — see PLAN.md commit-table rows 1-3.
3443 /// - **Commit (2/N):** Adds `RequestManagers` wiring (coordinator,
3444 /// commit, heartbeat, membership, offsets, topic-metadata, fetch)
3445 /// with the group-protocol gate of Java's constructor, plus the
3446 /// `ConsumerStateNotifier::register_state_listener` registration on
3447 /// `ConsumerMembershipManager` (PLAN.md §"State-notifier
3448 /// registration").
3449 /// - **Commit (3/N):** Adds bg-task spawn + `NetworkThreadCloseHandle`
3450 /// assembly + `Self::with_components(...)` call. After commit
3451 /// (3/N) this constructor compiles end-to-end.
3452 ///
3453 /// PLAINTEXT only in Phase 12 (PLAN.md §"Out of scope" — SSL/SASL
3454 /// `ChannelBuilder` escape hatches deferred). Bootstrap addresses
3455 /// resolved via [`crate::ClientUtils::parse_and_validate_addresses`]
3456 /// — same call the producer uses (mirrors Java
3457 /// `ClientUtils.parseAndValidateAddresses`).
3458 ///
3459 /// # Java field-init order
3460 ///
3461 /// Mirrors the table in PLAN.md (Java constructor statement → Rust action).
3462 /// Each block below quotes the Java statement that originated it.
3463 ///
3464 /// # Errors
3465 ///
3466 /// Java wraps the whole constructor body in
3467 /// `catch (Throwable t) { ... throw new KafkaException("Failed to construct
3468 /// kafka consumer", t); }`, so EVERY
3469 /// construction failure reaches the caller as a `KafkaException` carrying
3470 /// that message with the underlying failure as its cause. This wrapper
3471 /// reproduces that; [`Self::new_inner`] holds the body.
3472 ///
3473 /// The other half of Java's catch body — `close(Duration.ZERO, ...)` to
3474 /// release partially-built resources (KAFKA-2121) — closes the
3475 /// deserializers once they are wrapped (see `CloseDeserializersOnError`).
3476 /// `new_inner` spawns the background task's `std::thread` only after
3477 /// `deserializers.disarm()`, which follows its last fallible step, so no
3478 /// task or socket needs shutting down on any error path.
3479 ///
3480 /// # Java's no-deserializer constructors are deliberately not translated
3481 ///
3482 /// Java's public entry point is `KafkaConsumer`, which has four
3483 /// constructors; the two without deserializers — `KafkaConsumer(Map)`
3484 /// and `KafkaConsumer(Properties)` —
3485 /// delegate to `this(configs, null, null)` and rely on
3486 /// `config.getConfiguredInstance(KEY_DESERIALIZER_CLASS_CONFIG,
3487 /// Deserializer.class)` to instantiate the class named by
3488 /// `key.deserializer` reflectively. Rust has no reflection and a
3489 /// deserializer is typed in this consumer's own `K` / `V`, so the
3490 /// deserializers are always supplied here as instances, and
3491 /// [`ConsumerConfig`](crate::consumer::ConsumerConfig) carries no
3492 /// deserializer state — no field, no config-key constant, and no setter.
3493 /// A `key.deserializer` / `value.deserializer` entry in the property map is
3494 /// therefore an unrecognised key, warned about and ignored, exactly as
3495 /// `key.serializer` is on the producer side.
3496 ///
3497 /// This mirrors the producer exactly, including the CLAUDE.md §2
3498 /// consequence for the plain name `new` — see
3499 /// [`KafkaProducer::new`](crate::producer::KafkaProducer::new) for the full
3500 /// reasoning, which is not repeated here.
3501 pub fn new(
3502 config: ConsumerConfig,
3503 key_deserializer: Box<dyn crate::common::serialization::DynDeserializer<K>>,
3504 value_deserializer: Box<dyn crate::common::serialization::DynDeserializer<V>>,
3505 ) -> Result<Self, Error> {
3506 Self::new_inner(config, key_deserializer, value_deserializer).map_err(|err| {
3507 // Java's wrap is UNCONDITIONAL — unlike
3508 // `ConsumerUtils.maybeWrapAsKafkaException`, it wraps a
3509 // `KafkaException` too, so there is no `is_kafka_error()` guard
3510 // here. "Failed to construct kafka consumer" is the string users
3511 // match on.
3512 Error::KafkaError(crate::common::KafkaError::with_message_source(
3513 crate::common::protocol::Errors::UnknownServerError,
3514 "Failed to construct kafka consumer",
3515 err,
3516 ))
3517 })
3518 }
3519
3520 /// The body of Java's constructor `try` block
3521 /// of `AsyncKafkaConsumer`. See [`Self::new`] for the
3522 /// `catch (Throwable t)` wrap applied to every error it returns.
3523 fn new_inner(
3524 mut config: ConsumerConfig,
3525 key_deserializer: Box<dyn crate::common::serialization::DynDeserializer<K>>,
3526 value_deserializer: Box<dyn crate::common::serialization::DynDeserializer<V>>,
3527 ) -> Result<Self, Error> {
3528 use crate::ApiVersions;
3529 use crate::ClientUtils;
3530 use crate::DefaultHostResolver;
3531 use crate::MetadataRecoveryStrategy;
3532 use crate::NetworkClient;
3533 use crate::common::internals::ClusterResourceListeners;
3534 use crate::common::network::ChannelBuilders;
3535 use crate::common::network::Selector;
3536 use crate::consumer::internals::AutoOffsetResetStrategy;
3537 use crate::consumer::internals::CommitRequestManager;
3538 use crate::consumer::internals::ConsumerHeartbeatRequestManager;
3539 use crate::consumer::internals::ConsumerMembershipManager;
3540 use crate::consumer::internals::ConsumerMetadata;
3541 use crate::consumer::internals::ConsumerUtils;
3542 use crate::consumer::internals::CoordinatorRequestManager;
3543 use crate::consumer::internals::FetchConfig;
3544 use crate::consumer::internals::FetchRequestManager;
3545 use crate::consumer::internals::NetworkClientDelegate;
3546 use crate::consumer::internals::OffsetsRequestManager;
3547 use crate::consumer::internals::RequestManagers;
3548 use crate::consumer::internals::TopicMetadataRequestManager;
3549
3550 // Java's `ConsumerConfig` is built with `doLog = true`, which logs
3551 // every parsed value (`AbstractConfig.logAll`).
3552 config.log_values();
3553 log::debug!("Initializing the Kafka consumer");
3554
3555 // Java's `KafkaConsumer` passes `Time.SYSTEM`, and the constructor
3556 // hands that one instance to every component it builds.
3557 let time: Arc<dyn Time> = Arc::new(crate::common::utils::SystemTime);
3558
3559 // `ConsumerConfig::new`, the only public way to build a config, already
3560 // derives `client.id`. A crate-internal config built from
3561 // `ConsumerConfig::defaults()` with its fields written directly (the
3562 // unit tests do this) skips `new` and may still carry an empty
3563 // `client.id`. Java's config is immutable and always finalised by
3564 // `postProcessParsedConfig`; construction is the Rust equivalent
3565 // point, so apply the same `maybeOverrideClientId` rule. It is
3566 // idempotent on a config `new` built.
3567 config.maybe_override_client_id()?;
3568
3569 // Java `new GroupRebalanceConfig(config, ProtocolType.CONSUMER)`
3570 // (in `org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer`'s constructor) validates a set
3571 // `group.instance.id`
3572 // (`org.apache.kafka.clients.GroupRebalanceConfig#GroupRebalanceConfig(AbstractConfig,ProtocolType)`)
3573 // regardless of `client.id`; `ConsumerConfig` only does so when it derives the id.
3574 if let Some(group_instance_id) = config.group_instance_id() {
3575 crate::common::requests::JoinGroupRequest::validate_group_instance_id(group_instance_id)?;
3576 }
3577
3578 // Java: `clientId = config.getString(CLIENT_ID_CONFIG)`.
3579 let client_id: Arc<str> = Arc::from(config.client_id());
3580 // Java: `autoCommitEnabled = config.getBoolean(...)`.
3581 let auto_commit_enabled = config.enable_auto_commit();
3582 // Java: `defaultApiTimeoutMs = Duration.ofMillis(...)`.
3583 // Read but stored on the consumer struct via
3584 // `with_components`.
3585 let _default_api_timeout_ms = config.default_api_timeout_ms;
3586
3587 // Java: `backgroundEventQueue` /
3588 // `applicationEventQueue` allocations. Both are unbounded —
3589 // mirrors Java's `LinkedBlockingQueue<>`.
3590 let (bg_event_tx, _bg_event_rx) = mpsc::unbounded_channel::<BackgroundEventEnvelope>();
3591 let (_app_event_tx, _app_event_rx) =
3592 mpsc::unbounded_channel::<crate::consumer::internals::events::ApplicationEventEnvelope>();
3593 // Java: `subscriptions = createSubscriptionState(config,
3594 // logContext)`. The `auto.offset.reset` strategy is parsed once at
3595 // ctor time.
3596 let auto_offset_reset = AutoOffsetResetStrategy::from_string(config.auto_offset_reset())?;
3597 let subscriptions: Arc<Mutex<SubscriptionState>> =
3598 Arc::new(Mutex::new(SubscriptionState::new(auto_offset_reset)));
3599
3600 // Java: `interceptorList`, `interceptors = new
3601 // ConsumerInterceptors<>(...)`. The Java reflection-based loader is
3602 // not translated (per PLAN.md "Out of scope"); the Rust ctor
3603 // builds an empty interceptor chain. Users supply interceptors via
3604 // a future config-extension API.
3605 let _interceptors: Arc<Mutex<ConsumerInterceptors<K, V>>> =
3606 Arc::new(Mutex::new(ConsumerInterceptors::<K, V>::new(Vec::new())));
3607
3608 // Java: `deserializers = new Deserializers<>(...)`.
3609 // From here on, a failed construction closes them (Java's catch,
3610 // in `org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer`'s constructor); see
3611 // `CloseDeserializersOnError`. Declared before every later component, so it is dropped after them.
3612 let deserializers =
3613 CloseDeserializersOnError::new(Arc::new(Deserializers::new(key_deserializer, value_deserializer)));
3614
3615 // Java: `clusterResourceListeners`. Phase-11
3616 // deferral keeps notifier wiring as a no-op (`ClusterResourceListeners::new()`).
3617 let cluster_resource_listeners = ClusterResourceListeners::new();
3618
3619 // Java: `metadata = metadataFactory.build(...)`.
3620 let metadata = Arc::new(ConsumerMetadata::with_config(
3621 &config,
3622 Arc::clone(&subscriptions),
3623 cluster_resource_listeners,
3624 ));
3625
3626 // Java 4.4 — `ClientUtils.maybeBootstrapMetadataSynchronously(config,
3627 // bootstrap.servers, metadata)` (KIP-909 / KAFKA-20939): with
3628 // `bootstrap.resolve.timeout.ms=0` (the default) the addresses are
3629 // resolved and the metadata bootstrapped here, so an unresolvable host
3630 // fails construction; with a positive value the NetworkClient resolves
3631 // them asynchronously.
3632 ClientUtils::maybe_bootstrap_metadata_synchronously(
3633 config.bootstrap_resolve_timeout_ms(),
3634 config.client_dns_lookup(),
3635 config.bootstrap_servers(),
3636 &metadata,
3637 )?;
3638
3639 // Java: `fetchConfig = new FetchConfig(config)`.
3640 let fetch_config = FetchConfig::with_consumer_config(&config)?;
3641 // Java: `isolationLevel = fetchConfig.isolationLevel`.
3642 let _isolation_level = fetch_config.isolation_level;
3643
3644 // Java: `apiVersions = new ApiVersions()`.
3645 let api_versions = Arc::new(ApiVersions::new());
3646
3647 // Java: `backgroundEventHandler = new
3648 // BackgroundEventHandler(...)`. The `AsyncConsumerMetrics` and the
3649 // shared background-event queue-depth counter are wired below (after
3650 // the `metrics` registry exists) before the handler is shared.
3651 let mut background_event_handler = BackgroundEventHandler::new(bg_event_tx);
3652
3653 // Java: `fetchBuffer = new FetchBuffer(logContext)`.
3654 let fetch_buffer = Arc::new(FetchBuffer::new());
3655
3656 // Java: `metrics = createMetrics(config, time, reporters)`
3657 // then `fetchMetricsManager = createFetchMetricsManager(metrics)`.
3658 // The consumer owns the `Arc<Metrics>` (kept for Phase M7's public
3659 // `metrics()` accessor); the `Arc<FetchMetricsManager>` is shared into
3660 // the fetch path (FetchRequestManager / FetchCollector). The full
3661 // Metrics-wiring (`consumer.metrics()`, reporter list) is finalized in
3662 // M7 over THIS same registry — no re-plumb.
3663 let (metrics, fetch_metrics_manager) = Self::create_fetch_metrics_manager(&config, Arc::clone(&time));
3664
3665 // M4: the consumer-level metrics register against the SAME
3666 // `Arc<Metrics>` registry as every other manager. The heartbeat and
3667 // offset-commit metrics managers are built below, only when their
3668 // request manager is: Java builds each inside that request manager's
3669 // constructor (`org.apache.kafka.clients.consumer.internals.CommitRequestManager`,
3670 // `ConsumerHeartbeatRequestManager`), so a consumer without a
3671 // `group.id` never registers them, and nothing removes them on close.
3672 let kafka_consumer_metrics = Arc::new(KafkaConsumerMetrics::new(Arc::clone(&metrics)));
3673
3674 // M6: the async-consumer background-task / event-queue metrics
3675 // (`AsyncConsumerMetrics`, `AsyncKafkaConsumer.java`). Registered
3676 // against the SAME `Arc<Metrics>` under `CONSUMER_METRIC_GROUP`
3677 // (`consumer-metrics`). Wired into the bg task, the application/
3678 // background event handlers, and the network client delegate (the
3679 // record sites Java passes `asyncConsumerMetrics` to). The two
3680 // `Arc<AtomicI64>` queue-depth counters mirror Java's O(1)
3681 // `queue.size()` for the application/background event queues (the
3682 // tokio mpsc sender exposes no `len()`).
3683 let async_consumer_metrics = Arc::new(AsyncConsumerMetrics::new(
3684 Arc::clone(&metrics),
3685 crate::consumer::internals::ConsumerUtils::CONSUMER_METRIC_GROUP,
3686 ));
3687 let application_event_queue_size = Arc::new(AtomicI64::new(0));
3688 let background_event_queue_size = Arc::new(AtomicI64::new(0));
3689
3690 // Wire the background-event handler's metrics + queue-depth counter
3691 // before it is shared into the delegate (single owner here).
3692 background_event_handler
3693 .set_async_consumer_metrics(Arc::clone(&async_consumer_metrics), Arc::clone(&background_event_queue_size));
3694 let background_event_handler = Arc::new(background_event_handler);
3695
3696 // Java: `networkClientDelegateSupplier =
3697 // NetworkClientDelegate.supplier(...)`. Mirrors the producer's
3698 // `with_config` Selector / NetworkClient wiring at
3699 // `src/producer/kafka_producer.rs:278-292`.
3700 //
3701 // Mirrors Java's `AsyncKafkaConsumer` `LogContext` prefix
3702 // `[Consumer clientId=..., groupId=...] `.
3703 let log_context = match config.group_id() {
3704 Some(group_id) => {
3705 LogContext::new(format!("[Consumer clientId={}, groupId={}] ", config.client_id(), group_id))
3706 },
3707 None => LogContext::new(format!("[Consumer clientId={}] ", config.client_id())),
3708 };
3709 // Java: `ChannelBuilder channelBuilder =
3710 // ClientUtils.createChannelBuilder(config, time, logContext)`. Selects
3711 // the channel builder from `security.protocol` + `ssl.*` / `sasl.*`
3712 // (PLAINTEXT / SSL / SASL_PLAINTEXT / SASL_SSL). SASL mechanism PLAIN
3713 // only: `ConsumerConfig` already rejected any other one when it was
3714 // built. Errors surface as a `Error` from the ctor (no panic).
3715 let channel_builder = ChannelBuilders::client_channel_builder(
3716 config.security_protocol,
3717 Some(&config.ssl_config),
3718 Some(&config.sasl_config),
3719 None, // listener_name
3720 config.client_id(),
3721 log_context.clone(),
3722 )
3723 // Java's reachable failure here is `SslFactory.configure` throwing
3724 // `ConfigException` (`org.apache.kafka.common.security.ssl.SslFactory#configure`) — the missing-argument
3725 // `IllegalArgumentException`s in `ChannelBuilders.create` cannot be reached
3726 // from a `ConsumerConfig`, which always supplies both sub-configs. So the
3727 // class is `ConfigException`, inside the `KafkaException` hierarchy;
3728 // `illegal_argument` put it outside, where `is_kafka_error()` is `false`.
3729 // Same as `KafkaProducer::new` and `KafkaAdminClient`.
3730 .map_err(|e| Error::config_message(format!("Failed to create channel builder: {e}")))?;
3731 let mut selector = Selector::with_defaults_and_log_context(
3732 config.connections_max_idle_ms,
3733 channel_builder,
3734 log_context.clone(),
3735 );
3736 selector.set_time(Arc::clone(&time));
3737 let shared_metadata = metadata.metadata_arc();
3738 let mut network_client = NetworkClient::with_metadata_rebootstrap_trigger_ms(
3739 selector,
3740 shared_metadata,
3741 config.client_id(),
3742 ConsumerUtils::CONSUMER_MAX_INFLIGHT_REQUESTS_PER_CONNECTION as usize,
3743 config.reconnect_backoff_ms,
3744 config.reconnect_backoff_max_ms,
3745 config.send_buffer_bytes,
3746 config.receive_buffer_bytes,
3747 config.request_timeout_ms,
3748 config.socket_connection_setup_timeout_ms,
3749 config.socket_connection_setup_timeout_max_ms,
3750 true, // discover_broker_versions — mirrors Java
3751 Arc::clone(&api_versions),
3752 DefaultHostResolver::new(),
3753 // `org.apache.kafka.clients.ClientUtils#createNetworkClient` reads
3754 // both from the config; the default strategy is `rebootstrap`.
3755 // `ConsumerConfig::new` already rejected any name other than
3756 // `none` / `rebootstrap`, so `for_name` cannot fail here.
3757 config.metadata_recovery_rebootstrap_trigger_ms,
3758 MetadataRecoveryStrategy::for_name(config.metadata_recovery_strategy())
3759 .map_err(Error::local_illegal_argument)?,
3760 log_context.clone(),
3761 );
3762 network_client.set_time(Arc::clone(&time));
3763 // `org.apache.kafka.clients.ClientUtils#createNetworkClient` passes `metadata.cluster.check.enable`
3764 // to the `NetworkClient` constructor (KIP-1242).
3765 network_client.set_metadata_cluster_check_enable(config.metadata_cluster_check_enable);
3766 // `ClientUtils.createNetworkClient(config, bootstrap.servers, ...)` passes
3767 // `bootstrapConfiguration(config, bootstrapServers)` to the
3768 // `NetworkClient` constructor (KIP-909,
3769 // `org.apache.kafka.clients.consumer.internals.NetworkClientDelegate#supplier`).
3770 network_client.set_bootstrap_configuration(ClientUtils::bootstrap_configuration(
3771 config.bootstrap_resolve_timeout_ms(),
3772 config.client_dns_lookup(),
3773 config.retry_backoff_ms(),
3774 config.bootstrap_servers(),
3775 )?);
3776 let mut network_client_delegate_inner = NetworkClientDelegate::new(
3777 &config,
3778 network_client,
3779 Arc::clone(&metadata).metadata_arc(),
3780 Arc::clone(&background_event_handler),
3781 false, // notify_metadata_errors_via_error_queue — Java
3782 // passes `false` for the consumer ctor (Java
3783 // `AsyncKafkaConsumer` constructor).
3784 );
3785 // M6: Java passes `asyncConsumerMetrics` to the delegate ctor; wire it
3786 // here before the delegate is shared with the bg task.
3787 network_client_delegate_inner.set_async_consumer_metrics(Arc::clone(&async_consumer_metrics));
3788
3789 // The single "nudge the background task" primitive: the selector's own
3790 // wakeup handle, i.e. Java's `Selector.wakeup()`. Everything that needs
3791 // the bg task to stop waiting pokes THIS — `ApplicationEventHandler::add`
3792 // (Java's `wakeupNetworkThread()`), the §31 rebalance-ack path,
3793 // `FetchRequestManager`'s completion signal, `wakeup()` and
3794 // `signal_close()`. It is a lock-free `Arc<Notify>`, so firing it never
3795 // contends with the delegate mutex the bg task holds while polling.
3796 //
3797 // Taken from the delegate rather than created here on purpose: a
3798 // separate `Notify` would only be forwarded to this one, and having two
3799 // interchangeable-looking nudge channels is what let several call sites
3800 // poke the wrong one.
3801 let event_notify = network_client_delegate_inner.wakeup_handle();
3802 let _network_client_delegate = Arc::new(tokio::sync::Mutex::new(network_client_delegate_inner));
3803
3804 // Java: `offsetCommitCallbackInvoker = new
3805 // OffsetCommitCallbackInvoker(interceptors)`. The interceptor
3806 // chain is constructed once and cloned/wrapped here; the
3807 // commit-callback invoker owns its own ConsumerInterceptors
3808 // instance for `on_commit` dispatch (per Phase-9 design).
3809 //
3810 // DOCUMENTED LIMITATION (not a TODO): config-based interceptor
3811 // loading is untranslated in this milestone. Java passes
3812 // the SAME `interceptors` reference to both the consumer and the
3813 // invoker; that `interceptors` list is populated by the reflective
3814 // `interceptor.classes` loader, which has no Rust counterpart here.
3815 // The Rust `new` path therefore always builds an EMPTY
3816 // `ConsumerInterceptors` for both the consumer and this invoker, so
3817 // the two are behaviorally identical today (both hold empty Vecs).
3818 // The seam that CAN carry a non-empty interceptor chain is
3819 // `with_components` (`pub(crate)`); when reflective interceptor
3820 // loading is added in a future milestone, share the consumer's
3821 // interceptor Arc here so the invoker dispatches through the same
3822 // loaded list rather than this empty one.
3823 let _offset_commit_callback_invoker: Arc<OffsetCommitCallbackInvoker<K, V>> =
3824 Arc::new(OffsetCommitCallbackInvoker::new(ConsumerInterceptors::<K, V>::new(Vec::new())));
3825
3826 // Java: `groupMetadata.set(initializeGroupMetadata(...))`
3827 // — only when `group.id` is present. The cache itself lives on
3828 // the consumer struct (built inside `with_components`).
3829 //
3830 // `initializeGroupMetadata(String, Optional<String>)`
3831 // (`org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer#initializeGroupMetadata`) rejects a
3832 // present-but-empty `group.id` before building anything:
3833 //
3834 // ```java
3835 // if (groupId != null) {
3836 // if (groupId.isEmpty()) {
3837 // throw new InvalidGroupIdException("The configured " + ConsumerConfig.GROUP_ID_CONFIG
3838 // + " should not be an empty string or whitespace.");
3839 // } else {
3840 // return Optional.of(initializeConsumerGroupMetadata(groupId, groupInstanceId));
3841 // }
3842 // }
3843 // ```
3844 //
3845 // Without it, `Some("")` is "in a group" for the coordinator / commit /
3846 // heartbeat / membership wiring below but "not in a group" for
3847 // `return_error_if_group_id_not_defined`, so one consumer holds two
3848 // contradictory answers and puts an empty group id on the wire.
3849 // (Java's check is `isEmpty()` only, despite the message naming
3850 // whitespace; the message is reproduced verbatim regardless.)
3851 let group_id = match config.group_id() {
3852 Some("") => {
3853 return Err(Error::invalid_group_id(format!(
3854 "The configured {} should not be an empty string or whitespace.",
3855 ConsumerConfig::GROUP_ID_CONFIG
3856 )));
3857 },
3858 Some(group_id) => Some(group_id.to_string()),
3859 None => None,
3860 };
3861
3862 // ═══════════════════════════════════════════════════════════════
3863 // Phase 12 commit (2/N): RequestManagers wiring.
3864 // ═══════════════════════════════════════════════════════════════
3865 //
3866 // Java: `requestManagersSupplier =
3867 // RequestManagers.supplier(...)`. The Rust translation builds each
3868 // manager directly. Group-protocol gate: coordinator / commit /
3869 // heartbeat / membership are only built when `group.id` is
3870 // present (Java's `Optional.ofNullable(groupId).map(...)` pattern
3871 // inside `RequestManagers.supplier`).
3872 //
3873 // Slot sharing across managers (Java uses heap references; Rust
3874 // wraps the shared slot in `Arc<...>` — see `RequestManagers`
3875 // field docs and Phase 12 commit message for the refactor that
3876 // landed alongside this commit):
3877 // * `coordinator`: `Arc<CoordinatorRequestManager>` —
3878 // shared with the heartbeat manager (heartbeat reads the
3879 // discovered coordinator node every poll). Uses interior
3880 // mutability (`Arc<CoordinatorRequestManagerInner>`).
3881 // * `commit`: `Arc<CommitRequestManager>` — shared with the
3882 // membership manager (membership calls
3883 // `maybeAutoCommitSyncBeforeRebalance` from `reconcile`).
3884 // * `consumer_membership`: `Arc<ConsumerMembershipManager>` —
3885 // shared with the heartbeat manager (heartbeat reads
3886 // state/epoch/assignment every poll).
3887 // The bg-task `run_once` polls these three Arc-shared slots
3888 // explicitly (`coordinator` via `lock()`, `commit` via
3889 // `ApplicationEventProcessor` event arms, `membership` via
3890 // `reconcile()`); see `consumer_network_thread.rs`.
3891
3892 let current_time_ms = time.milliseconds();
3893
3894 let coordinator: Option<Arc<CoordinatorRequestManager>> = group_id.as_ref().map(|gid| {
3895 Arc::new(CoordinatorRequestManager::new(
3896 config.retry_backoff_ms(),
3897 config.retry_backoff_max_ms(),
3898 gid.clone(),
3899 ))
3900 });
3901
3902 let commit: Option<Arc<CommitRequestManager>> = group_id.as_ref().map(|gid| {
3903 Arc::new(CommitRequestManager::new(
3904 &config,
3905 Arc::clone(&metadata),
3906 Arc::clone(&subscriptions),
3907 gid.clone(),
3908 config.group_instance_id().map(|s| s.to_string()),
3909 Arc::clone(&time),
3910 current_time_ms,
3911 ))
3912 });
3913
3914 // Wire the CoordinatorRequestManager handle into the
3915 // CommitRequestManager. Java passes the coordinator directly to
3916 // the `CommitRequestManager` constructor
3917 // (`AsyncKafkaConsumer.java` calling
3918 // `CommitRequestManager.<init>(coordinatorRequestManager, ...)`).
3919 // In Rust both managers are `Arc`-shared and reference each
3920 // other through interior mutability; wiring happens after both
3921 // are built. Without this, the commit manager's response
3922 // handlers and retry drivers cannot call
3923 // `mark_coordinator_unknown` on `NotCoordinator` /
3924 // `CoordinatorNotAvailable` errors — which is what drives the
3925 // bg-task's next `poll(now)` to re-issue `FindCoordinator`.
3926 // Wake the bg task when the OffsetFetch retry driver (a spawned task)
3927 // re-enqueues a retry, so it is sent after its backoff rather than
3928 // after the network poll's own timeout. Reuses the same
3929 // `event_notify` the application-event enqueue path pokes.
3930 if let Some(commit_arc) = commit.as_ref() {
3931 commit_arc.set_completion_notify(Arc::clone(&event_notify));
3932 }
3933 // Likewise when the FindCoordinator forwarder applies a response: the
3934 // coordinator manager reports `i64::MAX` while its request is in flight
3935 // (KAFKA-20253), so without this wake the newly discovered coordinator
3936 // would sit unused until the network poll timed out.
3937 if let Some(coord_arc) = coordinator.as_ref() {
3938 coord_arc.set_completion_notify(Arc::clone(&event_notify));
3939 }
3940 if let (Some(coord_arc), Some(commit_arc)) = (coordinator.as_ref(), commit.as_ref()) {
3941 commit_arc.set_coordinator(Arc::clone(coord_arc));
3942 }
3943
3944 // Wire the `OffsetCommitCallbackInvoker` into the commit manager as a
3945 // type-erased `AutoCommitInterceptorHook` so the auto-commit success
3946 // path can enqueue the interceptor `on_commit` invocation. Java holds
3947 // the invoker directly on `CommitRequestManager`; in Rust the invoker
3948 // is generic over `<K, V>` and the commit manager is not, so it is
3949 // wired post-construction via the erased trait
3950 // (`org.apache.kafka.clients.consumer.internals.CommitRequestManager#autoCommitCallback`).
3951 if let Some(commit_arc) = commit.as_ref() {
3952 commit_arc.set_auto_commit_interceptor_hook(Arc::clone(&_offset_commit_callback_invoker)
3953 as Arc<dyn crate::consumer::internals::AutoCommitInterceptorHook>);
3954 }
3955
3956 // M4: build the OffsetCommitMetricsManager for the commit manager
3957 // (Java: in the `CommitRequestManager` constructor,
3958 // `org.apache.kafka.clients.consumer.internals.CommitRequestManager`), so the commit-response handler
3959 // records per-commit request latency (`CommitRequestManager.OffsetCommitRequestState#onResponse`).
3960 if let Some(commit_arc) = commit.as_ref() {
3961 commit_arc.set_offset_commit_metrics_manager(Arc::new(
3962 crate::consumer::internals::metrics::OffsetCommitMetricsManager::new(&metrics),
3963 ));
3964 }
3965
3966 // Java: `if (groupMetadata.get().isPresent() &&
3967 // groupProtocol == CONSUMER) config.ignore(GROUP_REMOTE_ASSIGNOR_CONFIG)`.
3968 // Rust does not track "ignored" config keys (no `ConfigDef`
3969 // equivalent); this is a comment-only translation. The classic
3970 // protocol path is deferred per `consumer-threading.md` §20, so
3971 // we only need to silence the warning for KIP-848 (`Consumer`)
3972 // consumers.
3973 // (No-op in Rust.)
3974
3975 // KIP-848 ConsumerMembershipManager — only built for KIP-848
3976 // (`group.protocol = consumer`) when `group.id` is present.
3977 let membership_opt: Option<Arc<ConsumerMembershipManager>> = match (group_id.as_ref(), commit.as_ref()) {
3978 (Some(gid), Some(commit_arc)) => Some(Arc::new(ConsumerMembershipManager::new(
3979 gid.clone(),
3980 config.group_instance_id().map(|s| s.to_string()),
3981 // Java's `GroupRebalanceConfig.rackId` (`org.apache.kafka.clients.GroupRebalanceConfig#rackId`):
3982 // `client.rack`, absent when empty.
3983 Some(config.client_rack()).filter(|rack| !rack.is_empty()).map(str::to_string),
3984 config.max_poll_interval_ms(),
3985 config.group_remote_assignor().map(|s| s.to_string()),
3986 Arc::clone(&subscriptions),
3987 // share() the commit handle: the membership manager + the
3988 // RequestManagers.commit slot point to the same
3989 // `Arc<CommitRequestManagerInner>` (Java holds one
3990 // reference each).
3991 Some(Arc::clone(commit_arc)),
3992 Arc::clone(&metadata),
3993 Arc::clone(&background_event_handler),
3994 auto_commit_enabled,
3995 // M5: rebalance latency/rate/failure metrics, registered against
3996 // the consumer's shared Arc<Metrics> (M3 field). Java builds the
3997 // ConsumerRebalanceMetricsManager inside the membership-manager
3998 // constructor; we build it here and pass it in.
3999 Some(Arc::new(
4000 crate::consumer::internals::metrics::ConsumerRebalanceMetricsManager::new(
4001 &metrics,
4002 Arc::clone(&subscriptions),
4003 ),
4004 )),
4005 Arc::clone(&time),
4006 ))),
4007 _ => None,
4008 };
4009
4010 // ConsumerHeartbeatRequestManager — only built when membership
4011 // is present (heartbeat needs the membership state machine to
4012 // build heartbeat-request bodies).
4013 let consumer_heartbeat: Option<ConsumerHeartbeatRequestManager> =
4014 match (coordinator.as_ref(), membership_opt.as_ref()) {
4015 (Some(coord_arc), Some(membership)) => {
4016 let mut hb = ConsumerHeartbeatRequestManager::new(
4017 current_time_ms,
4018 &config,
4019 Arc::clone(coord_arc),
4020 Arc::clone(&subscriptions),
4021 Arc::clone(membership),
4022 Arc::clone(&background_event_handler),
4023 );
4024 // M4: build the HeartbeatMetricsManager (Java: in the
4025 // `ConsumerHeartbeatRequestManager` constructor,
4026 // `org.apache.kafka.clients.consumer.internals.ConsumerHeartbeatRequestManager`) so the
4027 // send/response paths record `last-heartbeat-seconds-ago` /
4028 // `heartbeat-latency`
4029 // (`AbstractHeartbeatRequestManager#makeHeartbeatRequest`,
4030 // `AbstractHeartbeatRequestManager#logResponse`).
4031 hb.set_metrics_manager(Arc::new(
4032 crate::consumer::internals::metrics::HeartbeatMetricsManager::new(&metrics),
4033 ));
4034 // Wake the bg task when the heartbeat response forwarder (a
4035 // spawned task) has queued a completion, so a new
4036 // assignment, fence or fatal error is applied at once
4037 // rather than after the poll timeout (up to the heartbeat
4038 // interval while the request is in flight).
4039 hb.set_completion_notify(Arc::clone(&event_notify));
4040 Some(hb)
4041 },
4042 _ => None,
4043 };
4044
4045 // Java: `this.positionsValidator = new PositionsValidator(logContext, time,
4046 // subscriptions,
4047 // metadata);` (`org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer`'s constructor). Built here,
4048 // on the application side, because the app task consults it on the `poll()` critical path; the same `Arc`
4049 // is handed to the `OffsetsRequestManager`
4050 // below, which passes it on to its `OffsetFetcherUtils` (Java's `RequestManagers.supplier` →
4051 // `OffsetsRequestManager` constructor). One instance, three owners.
4052 let positions_validator = Arc::new(PositionsValidator::new(Arc::clone(&subscriptions), Arc::clone(&metadata)));
4053
4054 // OffsetsRequestManager — always built (Java's
4055 // `RequestManagers.supplier` builds this unconditionally).
4056 let offsets = Some(OffsetsRequestManager::new(
4057 Arc::clone(&subscriptions),
4058 Arc::clone(&metadata),
4059 fetch_config.isolation_level,
4060 Arc::clone(&time),
4061 config.retry_backoff_ms(),
4062 config.request_timeout_ms() as i64,
4063 config.default_api_timeout_ms as i64,
4064 Arc::clone(&api_versions),
4065 commit.as_ref().map(Arc::clone),
4066 Arc::clone(&positions_validator),
4067 ));
4068 // Wake the bg task when a LIST_OFFSETS / OFFSETS_FOR_LEADER_EPOCH
4069 // forwarder queues its response, so `end_offsets` & co. complete as
4070 // soon as the response is read (as Java's do inside `client.poll`)
4071 // rather than after the network poll's own timeout.
4072 if let Some(offsets_mgr) = offsets.as_ref() {
4073 offsets_mgr.set_completion_notify(Arc::clone(&event_notify));
4074 }
4075
4076 // TopicMetadataRequestManager — always built.
4077 let topic_metadata = Some(TopicMetadataRequestManager::new(&config));
4078
4079 // FetchRequestManager — always built. Java hands it the
4080 // `NetworkClientDelegate` for `isUnavailable` / `maybeThrowAuthFailure`;
4081 // here the bg task lends it the delegate on every `poll` /
4082 // `poll_on_close` instead (see `NodeStatus`).
4083 let fetch = {
4084 use crate::common::utils::internals::BufferSupplier;
4085
4086 let mut frm = FetchRequestManager::new(
4087 Arc::clone(&metadata),
4088 Arc::clone(&subscriptions),
4089 fetch_config.clone(),
4090 Arc::clone(&fetch_buffer),
4091 Arc::new(BufferSupplier::create()),
4092 Arc::clone(&api_versions),
4093 Arc::clone(&fetch_metrics_manager),
4094 config.retry_backoff_ms(),
4095 );
4096 // Wake the bg task when a fetch response is ready so it is drained
4097 // into the FetchBuffer promptly, instead of waiting for the
4098 // network poll's maximumTimeToWait. Reuses the same `event_notify`
4099 // the application-event enqueue path pokes.
4100 frm.set_completion_notify(Arc::clone(&event_notify));
4101 Some(frm)
4102 };
4103
4104 // Wrap the assembled `RequestManagers` in
4105 // `Arc<std::sync::Mutex<...>>` (Phase 10 pattern #3).
4106 let request_managers = Arc::new(std::sync::Mutex::new(RequestManagers::new(
4107 coordinator,
4108 topic_metadata,
4109 commit,
4110 consumer_heartbeat,
4111 membership_opt.clone(),
4112 offsets,
4113 fetch,
4114 )));
4115
4116 // ═══════════════════════════════════════════════════════════════
4117 // State-notifier construction + registration on membership.
4118 // ═══════════════════════════════════════════════════════════════
4119 //
4120 // Java keeps a SINGLE `AtomicReference<Optional<ConsumerGroupMetadata>> groupMetadata`
4121 // field and a SINGLE `MemberStateListener` instance
4122 // (`org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer#groupMetadata`,
4123 // `AsyncKafkaConsumer#memberStateListener` and the constructor). Both the constructor's initial
4124 // `groupMetadata.set(initializeGroupMetadata(...))` write AND the listener's `updateGroupMetadata(...)` writes
4125 // target the same slot.
4126 //
4127 // The Rust translation enforces this by building the
4128 // `group_metadata` / `group_assignment_snapshot` Arcs and the
4129 // `ConsumerStateNotifier` ONCE here, then threading the same
4130 // instances through both the membership-manager registration
4131 // (so heartbeat-response observations drive `update_group_metadata`
4132 // on the shared slot) AND the `AsyncKafkaConsumerComponents`
4133 // hand-off (so `Consumer::group_metadata()` reads the same slot).
4134 //
4135 // Issue 2 from the Phase-12 Critic review: previously the ctor
4136 // built TWO notifiers (one here, one inside `with_components`),
4137 // so `group_metadata` updates went to a slot the app side never
4138 // read. Single-notifier wiring now closes that gap.
4139 // Seed the group-metadata slot exactly as Java does at construction:
4140 // `groupMetadata.set(initializeGroupMetadata(config, groupRebalanceConfig))`
4141 // (Java's constructor, resolving through
4142 // `AsyncKafkaConsumer#initializeGroupMetadata(ConsumerConfig,GroupRebalanceConfig)` to the
4143 // `(groupId, groupInstanceId)` overload). A consumer WITH a `group.id` therefore starts with
4144 // `UNKNOWN_GENERATION_ID` / `UNKNOWN_MEMBER_ID` AND the configured `group.instance.id` already in place — so a
4145 // static member's `group.instance.id` reaches the first `TxnOffsetCommitRequest`
4146 // before any heartbeat response arrives (issues.md Issue C1). A
4147 // groupless consumer gets `None`. An empty `group.id` was already
4148 // rejected upstream (the ctor's `invalid_group_id` arm), matching Java
4149 // `initializeGroupMetadata(String,Optional)`.
4150 let group_metadata: Arc<Mutex<Option<Arc<ConsumerGroupMetadataImpl>>>> = Arc::new(Mutex::new(
4151 initialize_group_metadata(group_id.as_deref(), config.group_instance_id()),
4152 ));
4153 let group_assignment_snapshot: Arc<Mutex<HashSet<TopicPartition>>> = Arc::new(Mutex::new(HashSet::new()));
4154 let state_notifier = Arc::new(ConsumerStateNotifier::new(
4155 Arc::clone(&group_metadata),
4156 Arc::clone(&group_assignment_snapshot),
4157 Arc::new(AtomicBool::new(false)),
4158 ));
4159
4160 if let Some(membership) = membership_opt.as_ref() {
4161 // Java's `org.apache.kafka.clients.consumer.internals.RequestManagers#supplier` (KIP-848 / `consumer`
4162 // group protocol arm) registers TWO listeners on the membership
4163 // manager in this exact order:
4164 //
4165 // membershipManager.registerStateListener(commitRequestManager);
4166 // membershipManager.registerStateListener(applicationThreadMemberStateListener);
4167 //
4168 // The Rust translation mirrors that fan-out:
4169 //
4170 // 1. `commit` (when `group.id` is present) — its
4171 // `MemberStateListener::on_member_epoch_updated` writes the
4172 // broker-assigned UUID into `CommitRequestManager`'s
4173 // internal `MemberInfo`, which is then read at
4174 // `OffsetCommitRequest` build time. Without this
4175 // registration the OffsetCommit goes out with the default
4176 // empty member id and the broker rejects it with
4177 // `UNKNOWN_MEMBER_ID`. See Phase 12.5 COMMENTS.DONE Issue 7.
4178 //
4179 // 2. `state_notifier` (always) — writes the new
4180 // `ConsumerGroupMetadata` into the shared `Arc<Mutex<...>>`
4181 // slot that `Consumer::group_metadata()` reads. This is
4182 // Java's `applicationThreadMemberStateListener` equivalent
4183 // and Phase 12 Issue 2's single-notifier wiring.
4184 //
4185 // Note: in the `(coordinator, commit, membership)` build chain
4186 // above, `commit` is `Some` whenever `membership_opt` is `Some`
4187 // (both are gated on `group_id`), so the `commit.as_ref()` check
4188 // here is defense-in-depth.
4189 // `commit` was moved into `RequestManagers::new` above; read
4190 // it back through the `commit_handle()` accessor (returns a
4191 // cloned `Option<Arc<...>>`).
4192 let commit_listener_handle: Option<Arc<CommitRequestManager>> = {
4193 let rm_guard = request_managers.lock().expect("rm not poisoned");
4194 rm_guard.commit_handle()
4195 };
4196 if let Some(commit_arc) = commit_listener_handle {
4197 membership
4198 .abstract_mm
4199 .register_state_listener(commit_arc as Arc<dyn MemberStateListener>);
4200 }
4201 membership
4202 .abstract_mm
4203 .register_state_listener(Arc::clone(&state_notifier) as Arc<dyn MemberStateListener>);
4204 }
4205
4206 // ═══════════════════════════════════════════════════════════════
4207 // Phase 12 commit (3/N): bg-task spawn + components hand-off.
4208 // ═══════════════════════════════════════════════════════════════
4209 //
4210 // Java: the remaining `ApplicationEventProcessor`
4211 // + `ApplicationEventHandler` + `CompletableEventReaper` +
4212 // `ConsumerRebalanceListenerInvoker` + `ConsumerNetworkThread`
4213 // build + spawn. Each Java step maps line-for-line to the Rust
4214 // block below.
4215 use crate::consumer::internals::ConsumerNetworkThread;
4216 use crate::consumer::internals::events::ApplicationEventProcessor;
4217
4218 // Java: `applicationEventProcessor`.
4219 let application_event_reaper: Arc<std::sync::Mutex<CompletableEventReaper>> =
4220 Arc::new(std::sync::Mutex::new(CompletableEventReaper::new()));
4221 let app_event_processor = ApplicationEventProcessor::new(
4222 Arc::clone(&request_managers),
4223 Arc::clone(&metadata),
4224 Arc::clone(&subscriptions),
4225 Arc::clone(&application_event_reaper),
4226 Arc::clone(&time),
4227 );
4228
4229 // Java: `applicationEventHandler`. M6: wire the
4230 // `AsyncConsumerMetrics` + shared application-event queue-depth
4231 // counter before sharing the handler (Java passes
4232 // `asyncConsumerMetrics` to the ctor).
4233 let mut application_event_handler = ApplicationEventHandler::new(_app_event_tx, Arc::clone(&event_notify));
4234 application_event_handler
4235 .set_async_consumer_metrics(Arc::clone(&async_consumer_metrics), Arc::clone(&application_event_queue_size));
4236 let application_event_handler = Arc::new(application_event_handler);
4237
4238 // Java: `rebalanceListenerInvoker`. Java passes a
4239 // `RebalanceCallbackMetricsManager` + `Time` into the constructor; we
4240 // wire them post-construction so the no-arg `new` stays usable in
4241 // tests. The metrics manager registers against the consumer's shared
4242 // `Arc<Metrics>` (M3 field); the clock is the consumer's `time`.
4243 // Java keeps the manager in a field since KAFKA-19542 so `close()` can
4244 // close it (`org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer#rebalanceCallbackMetricsManager`).
4245 let rebalance_callback_metrics_manager = Arc::new(RebalanceCallbackMetricsManager::new(&metrics));
4246 let mut rebalance_listener_invoker = ConsumerRebalanceListenerInvoker::new(Arc::clone(&subscriptions));
4247 rebalance_listener_invoker.set_metrics(Arc::clone(&rebalance_callback_metrics_manager), Arc::clone(&time));
4248
4249 // Java: `backgroundEventReaper`. We reuse the same
4250 // `CompletableEventReaper` as the application reaper since the
4251 // Rust translation has one reaper per consumer (Phase 10 design
4252 // pattern #2). Java has two reapers but uses them
4253 // interchangeably for the consumer's purposes.
4254
4255 // Wakeup primitive shared with the bg task.
4256 let wakeup_trigger = WakeupTrigger::new();
4257
4258 // Java: `fetchCollector`.
4259 let fetch_collector = Arc::new(FetchCollector::new(
4260 Arc::clone(&metadata),
4261 Arc::clone(&subscriptions),
4262 fetch_config,
4263 // Every fallible step is behind us.
4264 deserializers.disarm(),
4265 Arc::clone(&fetch_metrics_manager),
4266 Arc::clone(&time),
4267 ));
4268
4269 // Java: `config.logUnused()` → `log::debug!(...)`.
4270 log::debug!("Kafka consumer initialized");
4271
4272 // ── Bg-task spawn ──
4273 //
4274 // Construct `ConsumerNetworkThread` and spawn its run loop.
4275 // `signal_close_fn` / `wakeup_fn` are erased through
4276 // [`NetworkThreadCloseHandle`] so the outer `AsyncKafkaConsumer`
4277 // struct stays non-generic over `K`.
4278 //
4279 // `max_time_to_wait_ms` is built here and passed into both
4280 // `ConsumerNetworkThread::new` (which writes the post-poll
4281 // computed bound on every `run_once` iteration) AND the consumer
4282 // struct (which reads via `maximum_time_to_wait_ms()`). The bg
4283 // task seeds it to `MAX_POLL_TIMEOUT_MS` inside its ctor — mirrors
4284 // Java's `ApplicationEventHandler.maximumTimeToWait()` slot shared
4285 // with the bg thread.
4286 let max_time_to_wait_ms: Arc<AtomicI64> = Arc::new(AtomicI64::new(
4287 ConsumerNetworkThread::<NetworkClient<Selector, DefaultHostResolver>>::MAX_POLL_TIMEOUT_MS,
4288 ));
4289
4290 let mut network_thread = ConsumerNetworkThread::new(
4291 Arc::clone(&time),
4292 _app_event_rx,
4293 Arc::clone(&application_event_reaper),
4294 app_event_processor,
4295 Arc::clone(&_network_client_delegate),
4296 Arc::clone(&request_managers),
4297 membership_opt.clone(),
4298 wakeup_trigger.clone(),
4299 Arc::clone(&max_time_to_wait_ms),
4300 );
4301 // M6: Java passes `asyncConsumerMetrics` to the `ConsumerNetworkThread`
4302 // ctor; wire it (plus the application-event queue-depth counter the bg
4303 // task resets to 0 on drain) before spawning the bg task.
4304 network_thread
4305 .set_async_consumer_metrics(Arc::clone(&async_consumer_metrics), Arc::clone(&application_event_queue_size));
4306
4307 // Capture the running-flag + wakeup handles before moving
4308 // `network_thread` into `spawn_dedicated`. The erased closures
4309 // call these to signal close / wake the bg task without
4310 // holding a reference to the concrete `K` type.
4311 let (signal_close_fn, wakeup_fn) = build_close_handle_fns(
4312 network_thread.running_handle(),
4313 network_thread.close_timeout_handle(),
4314 Arc::clone(&event_notify),
4315 );
4316
4317 // The `max_time_to_wait_ms` slot is seeded with
4318 // `MAX_POLL_TIMEOUT_MS` at ctor time (line above) and the bg task
4319 // writes to the SAME `Arc<AtomicI64>` cell on every `run_once`
4320 // iteration via `cached_max_time_to_wait_ms.store(...)`
4321 // (consumer_network_thread.rs). The app-side
4322 // `AsyncKafkaConsumer::maximum_time_to_wait_ms()` accessor reads
4323 // the bg-task's current value through this shared Arc — Java's
4324 // `cachedMaximumTimeToWait` is a single `long` field
4325 // (`org.apache.kafka.clients.consumer.internals.ConsumerNetworkThread#cachedMaximumTimeToWait`); the
4326 // Arc<AtomicI64> is the Rust equivalent that bridges the two task boundaries.
4327
4328 // Phase 21: run the bg loop on its OWN dedicated `std::thread`
4329 // hosting a `current_thread` tokio runtime, instead of
4330 // `tokio::spawn`ing it onto the caller's multi-thread
4331 // work-stealing runtime. Profiling showed ~54% of consumer CPU
4332 // was tokio multi-thread scheduler park/unpark churn from running
4333 // this single high-frequency IO task on a ~12-worker pool; a
4334 // dedicated single-thread runtime (librdkafka's model) removes it.
4335 //
4336 // This is purely an internal execution-strategy change — the
4337 // shutdown sequencing (`signal_close_fn` + `wakeup_fn` flip the
4338 // running flag and wake the selector; the bg loop exits on
4339 // `is_running()==false` then `cleanup()`), the shared channels,
4340 // and the `max_time_to_wait_ms` Arc are all identical to the
4341 // `tokio::spawn` form.
4342 //
4343 // `enable_all()` is REQUIRED: the IO driver backs the
4344 // Selector/mio loop and the time driver backs the heartbeat /
4345 // poll-timeout timers. The §10 network-poll-to-completion
4346 // (cancel-safety), the §11 `Selector`/`Notify` wakeup, and the
4347 // §31 listener/commit callbacks (which run on the APP task — the
4348 // bg only enqueues the callback event, stores the ack receiver and
4349 // `try_recv`s it on later iterations) are all unaffected by
4350 // moving the bg loop onto a `current_thread` runtime.
4351 //
4352 // The loop is `ConsumerNetworkThread::run`, which carries Java's
4353 // per-iteration `catch (Throwable)`: a panicking iteration is logged
4354 // and the loop continues, and `cleanup()` still runs on exit.
4355 let (done_rx, thread_handle) = network_thread.spawn_dedicated();
4356
4357 let network_thread_close =
4358 NetworkThreadCloseHandle::with_dedicated(signal_close_fn, wakeup_fn, done_rx, thread_handle);
4359
4360 // ── Assemble `AsyncKafkaConsumerComponents` and hand off ──
4361 //
4362 // The Phase-11 test seam stays — the production path builds the
4363 // components struct and calls `Self::with_components(...)`.
4364 // Phase-12 Issue 2 (Critic review) consolidated to a single
4365 // `ConsumerStateNotifier`: the Arc registered on the membership
4366 // manager earlier in this ctor and the Arc stored on the consumer
4367 // struct are the SAME instance, mirroring Java's single
4368 // `memberStateListener`.
4369
4370 // Java's `org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer` constructor: `config.logUnused()`,
4371 // after every component has read its keys.
4372 config.mark_read_keys_used();
4373 config.log_unused();
4374 let components = AsyncKafkaConsumerComponents {
4375 config,
4376 client_id,
4377 group_id,
4378 positions_validator,
4379 subscriptions,
4380 metadata,
4381 request_managers,
4382 background_event_rx: _bg_event_rx,
4383 application_event_handler,
4384 completable_event_reaper: application_event_reaper,
4385 max_time_to_wait_ms,
4386 wakeup_trigger,
4387 network_thread_close,
4388 fetch_buffer,
4389 fetch_collector,
4390 metrics,
4391 kafka_consumer_metrics,
4392 async_consumer_metrics,
4393 fetch_metrics_manager,
4394 rebalance_callback_metrics_manager,
4395 background_event_queue_size,
4396 rebalance_listener_invoker,
4397 offset_commit_callback_invoker: _offset_commit_callback_invoker,
4398 interceptors: _interceptors,
4399 isolation_level: _isolation_level,
4400 time,
4401 group_metadata,
4402 group_assignment_snapshot,
4403 state_notifier,
4404 };
4405
4406 Ok(Self::with_components(components))
4407 }
4408
4409 /// Builds the consumer's `Metrics` registry and `FetchMetricsManager`.
4410 ///
4411 /// Translates Java's `ConsumerUtils.createMetrics(config, time, reporters)`
4412 /// followed by `createFetchMetricsManager(metrics)`. The `MetricConfig`
4413 /// carries the `metrics.num.samples`, `metrics.sample.window.ms`, and
4414 /// `metrics.recording.level` settings and the single `client-id` tag; the
4415 /// registry uses the `"consumer"` metric group prefix. The (no-op) reporter
4416 /// list and the JMX context are deferred to Phase M7; for M3 the registry is
4417 /// reporter-less but fully functional. Returns the owned `Arc<Metrics>`
4418 /// (kept on the consumer for M7's public accessor) and the
4419 /// `Arc<FetchMetricsManager>` shared into the fetch path.
4420 fn create_fetch_metrics_manager(
4421 config: &ConsumerConfig,
4422 time: Arc<dyn Time>,
4423 ) -> (Arc<Metrics>, Arc<FetchMetricsManager>) {
4424 const CONSUMER_METRIC_GROUP_PREFIX: &str = "consumer";
4425 const CONSUMER_CLIENT_ID_METRIC_TAG: &str = "client-id";
4426
4427 let mut tags = std::collections::BTreeMap::new();
4428 tags.insert(CONSUMER_CLIENT_ID_METRIC_TAG.to_string(), config.client_id().to_string());
4429
4430 let recording_level = RecordingLevel::for_name(&config.metrics_recording_level).unwrap_or(RecordingLevel::Info);
4431 let metric_config = MetricConfig::new()
4432 .set_samples(config.metrics_num_samples)
4433 .set_time_window_ms(config.metrics_sample_window_ms)
4434 .set_record_level(recording_level)
4435 .set_tags(tags);
4436
4437 let metrics = Arc::new(Metrics::with_default_config_time(Arc::new(metric_config), time));
4438
4439 // `client-id` is a default config tag, so it is added automatically to
4440 // every metric name; the registry's template tag set therefore lists
4441 // only `client-id` (matching Java's singleton tag set).
4442 let mut registry_tags = indexmap::IndexSet::new();
4443 registry_tags.insert(CONSUMER_CLIENT_ID_METRIC_TAG.to_string());
4444 let registry = FetchMetricsRegistry::new(registry_tags, CONSUMER_METRIC_GROUP_PREFIX);
4445
4446 let manager = Arc::new(FetchMetricsManager::new(Arc::clone(&metrics), registry));
4447 (metrics, manager)
4448 }
4449
4450 pub(crate) fn with_components(components: AsyncKafkaConsumerComponents<K, V>) -> Self {
4451 let auto_commit_enabled = components.config.enable_auto_commit();
4452 let default_api_timeout_ms = components.config.default_api_timeout_ms as i64;
4453 let retry_backoff_ms = components.config.retry_backoff_ms();
4454 let interceptors_for_commit = Arc::clone(&components.interceptors);
4455 let commit_state = Arc::new(AsyncCommitState::new(
4456 Arc::clone(&components.offset_commit_callback_invoker) as Arc<dyn OffsetCommitCallbackQueue>,
4457 Arc::new(move |offsets: &HashMap<TopicPartition, OffsetAndMetadata>| {
4458 interceptors_for_commit
4459 .lock()
4460 .unwrap_or_else(std::sync::PoisonError::into_inner)
4461 .on_commit(offsets);
4462 }),
4463 ));
4464
4465 // The `state_notifier`, `group_metadata`, and
4466 // `group_assignment_snapshot` slots are constructed by the caller
4467 // (production ctor: `Self::new`; tests: their fixture builder)
4468 // and threaded through here. The production ctor registers the
4469 // SAME `state_notifier` Arc on the membership manager BEFORE the
4470 // bg-task spawn — mirroring Java's single `MemberStateListener`
4471 // instance (`org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer#memberStateListener`). Tests
4472 // construct their own Arcs and either register the notifier
4473 // themselves or skip registration if they don't exercise the
4474 // membership path.
4475 Self {
4476 subscriptions: components.subscriptions,
4477 metadata: components.metadata,
4478 request_managers: components.request_managers,
4479 background_event_rx: components.background_event_rx,
4480 application_event_handler: components.application_event_handler,
4481 completable_event_reaper: components.completable_event_reaper,
4482 max_time_to_wait_ms: components.max_time_to_wait_ms,
4483 wakeup_trigger: components.wakeup_trigger,
4484 network_thread_close: components.network_thread_close,
4485 fetch_buffer: components.fetch_buffer,
4486 fetch_collector: components.fetch_collector,
4487 metrics: components.metrics,
4488 kafka_consumer_metrics: components.kafka_consumer_metrics,
4489 async_consumer_metrics: components.async_consumer_metrics,
4490 fetch_metrics_manager: components.fetch_metrics_manager,
4491 rebalance_callback_metrics_manager: components.rebalance_callback_metrics_manager,
4492 background_event_queue_size: components.background_event_queue_size,
4493 client_id: components.client_id,
4494 group_id: components.group_id,
4495 group_metadata: components.group_metadata,
4496 group_assignment_snapshot: components.group_assignment_snapshot,
4497 has_pending_reconciliation: components.state_notifier.has_pending_reconciliation_handle(),
4498 state_notifier: components.state_notifier,
4499 positions_validator: components.positions_validator,
4500 rebalance_listener_invoker: components.rebalance_listener_invoker,
4501 offset_commit_callback_invoker: Arc::clone(&components.offset_commit_callback_invoker),
4502 interceptors: components.interceptors,
4503 auto_commit_enabled,
4504 default_api_timeout_ms,
4505 retry_backoff_ms,
4506 isolation_level: components.isolation_level,
4507 closed: Arc::new(AtomicBool::new(false)),
4508 rebalance_listener: Mutex::new(None),
4509 inflight_poll: None,
4510 config: components.config,
4511 time: components.time,
4512 commit_state,
4513 }
4514 }
4515
4516 // ── Sync state-read methods ────────────────────────────────────────
4517 //
4518 // Per `consumer-threading.md` §16, each method acquires the
4519 // SubscriptionState lock briefly, reads, drops the guard — NEVER
4520 // holds the guard across `.await`. Since these methods are `fn`
4521 // (not `async`), there is no `.await` boundary at all.
4522 //
4523 // # Closed-consumer behavior
4524 //
4525 // Java's accessors call `acquireAndEnsureOpen()` which throws
4526 // `IllegalStateException("This consumer has already been closed.")`
4527 // when the consumer is closed. Calling one after `close()` is a
4528 // programming error, so each Rust accessor below panics with Java's
4529 // message instead (`panic_if_closed`), as a Java argument-precondition
4530 // throw stays a Rust panic. The signatures keep their plain return types;
4531 // `Consumer`'s rustdoc documents the panic under `# Panics`.
4532 // `groupMetadata()` panics the same way when `group.id` is unset
4533 // (Java's `throwIfGroupIdNotDefined()`). Every `async fn` (poll / commit /
4534 // position / committed / current_lag / unsubscribe / close / etc.) returns
4535 // the closed-consumer error through `ensure_open()`.
4536 //
4537 // # Bootstrap-failure behavior (deliberate Java divergence, KIP-909)
4538 //
4539 // Java 4.4's `acquireAndEnsureOpen()` also throws the permanent
4540 // `BootstrapResolutionException`
4541 // (`org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer#acquireAndEnsureOpen`), so `assignment()`,
4542 // `subscription()`, `paused()` and `groupMetadata()` throw it once asynchronous bootstrap resolution has failed.
4543 // The Rust accessors have no error channel, and a bootstrap failure is not a
4544 // programming error that would justify a panic, so they keep returning
4545 // their current value. Every `Result`-returning API, `current_lag`
4546 // included, returns `Error::BootstrapResolution` through `ensure_open()`.
4547 //
4548 // # Mutable Set semantics (deliberate Java divergence)
4549 //
4550 // Java wraps the returned `Set` with `Collections.unmodifiableSet(...)`.
4551 // The Rust accessors return an owned `HashSet` — the caller may
4552 // freely mutate it without affecting the consumer's internal state.
4553 // This is idiomatic Rust and is observable only through user code
4554 // that depended on `UnsupportedOperationException` (none of the
4555 // translated tests do).
4556
4557 /// Java: `Set<TopicPartition> assignment()`.
4558 ///
4559 /// **Returns an owned mutable `HashSet`** (Java returns
4560 /// `Collections.unmodifiableSet(...)`). **Panics when the consumer is
4561 /// closed** (Java throws `IllegalStateException`). See the module-level
4562 /// "Sync state-read methods" comment for rationale.
4563 ///
4564 /// **Returns the current value after a permanent bootstrap failure**
4565 /// (Java throws `BootstrapResolutionException`, KIP-909); see the
4566 /// module-level "Sync state-read methods" comment.
4567 pub fn assignment(&self) -> std::collections::HashSet<TopicPartition> {
4568 self.panic_if_closed();
4569 let subs = self.subscriptions.lock().unwrap();
4570 subs.assigned_partitions()
4571 }
4572
4573 /// Java: `Set<String> subscription()`.
4574 ///
4575 /// **Returns an owned mutable `HashSet`** (Java returns
4576 /// `Collections.unmodifiableSet(...)`). **Panics when the consumer is
4577 /// closed** (Java throws `IllegalStateException`).
4578 ///
4579 /// **Returns the current value after a permanent bootstrap failure**
4580 /// (Java throws `BootstrapResolutionException`, KIP-909); see the
4581 /// module-level "Sync state-read methods" comment.
4582 pub fn subscription(&self) -> std::collections::HashSet<String> {
4583 self.panic_if_closed();
4584 let subs = self.subscriptions.lock().unwrap();
4585 subs.subscription()
4586 }
4587
4588 /// Java: `Set<TopicPartition> paused()`.
4589 ///
4590 /// **Returns an owned mutable `HashSet`** (Java returns
4591 /// `Collections.unmodifiableSet(...)`). **Panics when the consumer is
4592 /// closed** (Java throws `IllegalStateException`).
4593 ///
4594 /// **Returns the current value after a permanent bootstrap failure**
4595 /// (Java throws `BootstrapResolutionException`, KIP-909); see the
4596 /// module-level "Sync state-read methods" comment.
4597 pub fn paused(&self) -> std::collections::HashSet<TopicPartition> {
4598 self.panic_if_closed();
4599 let subs = self.subscriptions.lock().unwrap();
4600 subs.paused_partitions()
4601 }
4602
4603 /// Java: `String clientId()`. Returned as a borrowed `&str` per
4604 /// CLAUDE.md §14 (most general borrowed form for getters).
4605 ///
4606 /// Java's method is package-private and only its tests call it, so it is
4607 /// test-only here and not on the `Consumer` trait.
4608 ///
4609 /// **Returns the configured value silently when the consumer is
4610 /// closed** (Java throws `IllegalStateException`). The `client_id`
4611 /// is immutable for the lifetime of the consumer, so returning it
4612 /// post-close is harmless.
4613 #[cfg(test)]
4614 pub(crate) fn client_id(&self) -> &str {
4615 &self.client_id
4616 }
4617
4618 /// Test seam: how many fetch requests the production fetch manager has
4619 /// built for `node_id` (each one a FETCH handed to the network client).
4620 #[cfg(test)]
4621 pub(crate) fn fetch_requests_built_for_test(&self, node_id: i32) -> u64 {
4622 let guard = self.request_managers.lock().expect("request managers mutex poisoned");
4623 guard
4624 .fetch
4625 .as_ref()
4626 .map_or(0, |fetch| fetch.fetch_requests_built_for_test(node_id))
4627 }
4628
4629 /// Test seam: weak references to the metadata and subscription state, so a
4630 /// test can check that a closed and dropped consumer frees them.
4631 #[cfg(test)]
4632 pub(crate) fn metadata_and_subscriptions_for_test(
4633 &self,
4634 ) -> (std::sync::Weak<ConsumerMetadata>, std::sync::Weak<Mutex<SubscriptionState>>) {
4635 (Arc::downgrade(&self.metadata), Arc::downgrade(&self.subscriptions))
4636 }
4637
4638 /// Java: `Map<MetricName, ? extends Metric> metrics()`
4639 /// (`AsyncKafkaConsumer#metrics`:
4640 /// `return Collections.unmodifiableMap(metrics.metrics());`).
4641 ///
4642 /// A live, read-only view of the consumer's owned `Arc<Metrics>` registry —
4643 /// the SAME registry into which every metrics manager (fetch,
4644 /// kafka-consumer, heartbeat, offset-commit, rebalance +
4645 /// rebalance-callback, async) registers (M3–M6). The view therefore holds
4646 /// the full Java metric set, and sees metrics registered later, as Java's
4647 /// `Collections.unmodifiableMap` does. Taking it copies nothing.
4648 pub fn metrics(&self) -> MetricsView {
4649 self.metrics.metrics()
4650 }
4651
4652 /// Java: `ConsumerGroupMetadata groupMetadata()`
4653 /// (`AsyncKafkaConsumer#groupMetadata()`).
4654 ///
4655 /// **Panics when the consumer is closed** (Java's `acquireAndEnsureOpen()`
4656 /// throws `IllegalStateException`) and **when `group.id` is unset** (Java's
4657 /// `throwIfGroupIdNotDefined()` throws `InvalidGroupIdException`), with
4658 /// Java's messages: both are programming errors. See the module-level
4659 /// "Sync state-read methods" comment.
4660 ///
4661 /// The returned value shares the cached `Arc`, as Java returns the cached
4662 /// object. The slot is seeded at construction (`initialize_group_metadata`,
4663 /// mirroring Java's constructor), carrying `UNKNOWN_GENERATION_ID` (-1) and
4664 /// `UNKNOWN_MEMBER_ID` ("") until
4665 /// [`ConsumerStateNotifier::on_member_epoch_updated`] — the
4666 /// [`MemberStateListener`] registered on the `ConsumerMembershipManager` —
4667 /// overwrites them with the first heartbeat response's epoch / member id.
4668 ///
4669 /// **Returns the current value after a permanent bootstrap failure**
4670 /// (Java throws `BootstrapResolutionException`, KIP-909); see the
4671 /// module-level "Sync state-read methods" comment.
4672 pub fn group_metadata(&self) -> Arc<dyn ConsumerGroupMetadata> {
4673 self.panic_if_closed();
4674 panic_if_group_id_not_defined(self.group_id.as_deref());
4675 group_metadata_from(&self.group_metadata)
4676 }
4677
4678 /// Returns the [`ConsumerStateNotifier`] this consumer expects to
4679 /// be registered as a [`MemberStateListener`] on the
4680 /// `ConsumerMembershipManager`. The production factory (Phase 12)
4681 /// performs this registration immediately after constructing the
4682 /// membership manager; tests can do the same on their stand-in
4683 /// manager.
4684 ///
4685 /// Mirrors Java's anonymous-inner-class `memberStateListener` field
4686 /// `AsyncKafkaConsumer#memberStateListener`: that listener is passed
4687 /// into the membership-manager builder so the bg task can invoke it
4688 /// during reconciliation.
4689 pub(crate) fn state_notifier(&self) -> Arc<ConsumerStateNotifier> {
4690 Arc::clone(&self.state_notifier)
4691 }
4692
4693 /// Java: `void wakeup()`. Sync — callable from any task, including
4694 /// signal handlers.
4695 pub fn wakeup(&self) {
4696 // Two halves, two primitives, deliberately:
4697 // 1. cancel the token — this is the user-visible half, what makes the
4698 // next blocking-style API return `Error::Wakeup` (§11);
4699 // 2. nudge the transport notify so an in-flight `KafkaClient::poll`
4700 // returns promptly instead of running out its poll wait. Idempotent
4701 // and not user-visible, so it is safe even when the token was
4702 // already cancelled.
4703 self.wakeup_trigger.wakeup();
4704 self.network_thread_close.wakeup();
4705 }
4706
4707 /// Returns a `Clone + Send + Sync` [`ConsumerHandle`] exposing
4708 /// [`Self::wakeup`] and the reentrant-safe consumer ops, callable from
4709 /// another task / thread. See [`ConsumerHandle`] for the rationale
4710 /// (Java's `Consumer` is freely shareable across threads; this is the
4711 /// safe Rust equivalent for both the cross-task `wakeup()` pattern and
4712 /// in-callback rebalance-listener reentrancy).
4713 pub fn handle(&self) -> ConsumerHandle {
4714 ConsumerHandle::for_async(AsyncConsumerHandleState {
4715 wakeup_trigger: self.wakeup_trigger.clone(),
4716 bg_wakeup: self.network_thread_close.wakeup_fn_clone(),
4717 application_event_handler: Arc::clone(&self.application_event_handler),
4718 subscriptions: Arc::clone(&self.subscriptions),
4719 metadata: Arc::clone(&self.metadata),
4720 group_metadata: Arc::clone(&self.group_metadata),
4721 time: Arc::clone(&self.time),
4722 default_api_timeout_ms: self.default_api_timeout_ms,
4723 isolation_level: self.isolation_level,
4724 metrics: Arc::clone(&self.metrics),
4725 commit_state: Arc::clone(&self.commit_state),
4726 group_id: self.group_id.clone(),
4727 kafka_consumer_metrics: Arc::clone(&self.kafka_consumer_metrics),
4728 callbacks_in_progress: Arc::clone(self.offset_commit_callback_invoker.callbacks_in_progress()),
4729 closed: Arc::clone(&self.closed),
4730 })
4731 }
4732
4733 /// Enters a callback scope by hand, for tests whose body stands in for
4734 /// a rebalance-listener or commit-callback body and drives the
4735 /// consumer's [`ConsumerHandle`] directly. It is the marker the
4736 /// production listener and commit-callback invocations hold, so the
4737 /// fixture cannot admit what production would reject.
4738 #[cfg(test)]
4739 pub(crate) fn enter_callback_scope(&self) -> CallbackInProgress {
4740 self.offset_commit_callback_invoker.enter_callback()
4741 }
4742
4743 /// Returns the cached `max_time_to_wait` value, set by the bg task
4744 /// after each `run_once` iteration. Used by the poll loop (commit
4745 /// (4/N)).
4746 pub(crate) fn maximum_time_to_wait_ms(&self) -> i64 {
4747 self.max_time_to_wait_ms.load(Ordering::Acquire)
4748 }
4749
4750 /// `true` iff [`Self::close`] has already returned.
4751 pub(crate) fn is_closed(&self) -> bool {
4752 self.closed.load(Ordering::Acquire)
4753 }
4754
4755 /// The closed half of Java's `acquireAndEnsureOpen()` for the sync
4756 /// accessors: panics with Java's `IllegalStateException` message, since
4757 /// they have no error channel and calling them after `close()` is a
4758 /// programming error (see the "Sync state-read methods" comment).
4759 fn panic_if_closed(&self) {
4760 if self.is_closed() {
4761 panic!("{CONSUMER_CLOSED_MSG}");
4762 }
4763 }
4764
4765 // ── Subscribe / unsubscribe / assign ───────────────────────────────
4766 //
4767 // Translates Java's `subscribe(Collection<String>)`,
4768 // `subscribe(Collection<String>, ConsumerRebalanceListener)`,
4769 // `subscribe(SubscriptionPattern)`, `subscribe(SubscriptionPattern,
4770 // ConsumerRebalanceListener)`, `subscribe(Pattern)`,
4771 // `subscribe(Pattern, ConsumerRebalanceListener)`, `unsubscribe()`,
4772 // and `assign(Collection<TopicPartition>)`.
4773 //
4774 // Each method:
4775 // 1. Verifies `closed` (Java's `acquireAndEnsureOpen`).
4776 // 2. Validates arguments (returning `Error::local_illegal_argument`
4777 // where Java throws `IllegalArgumentException`). Rust's type
4778 // system makes the `null`-target tests un-translatable;
4779 // `"".trim().is_empty()` covers the empty/blank case.
4780 // 3. Briefly acquires the `SubscriptionState` lock to read
4781 // `assigned_partitions`, drops the guard, then enqueues an
4782 // `ApplicationEvent` carrying the change. Mirrors
4783 // `consumer-threading.md` §16 lock discipline — the actual
4784 // mutation of `SubscriptionState` happens on the bg task via
4785 // the matching `ApplicationEventProcessor` arm.
4786 //
4787 // `fetchBuffer.retainAll(...)` calls present in the Java source are
4788 // omitted here — the consumer's `FetchBuffer` is wired in commit
4789 // (4/N) along with the poll loop. The bg-side
4790 // `ApplicationEventProcessor` already retains-all when processing
4791 // the matching subscribe/assign event, so the only consequence of
4792 // the gap is that records already buffered for removed partitions
4793 // are returned on the next `poll()` — Phase 11 commit (4/N) closes
4794 // this seam.
4795
4796 /// Translates Java's `private void throwIfGroupIdNotDefined()`
4797 /// (`AsyncKafkaConsumer#throwIfGroupIdNotDefined`). Java throws
4798 /// `InvalidGroupIdException` (`ApiException` subclass with
4799 /// `Errors.InvalidGroupId`); the Rust analog is
4800 /// `Error::invalid_group_id(...)` which surfaces a `KafkaError`
4801 /// variant carrying `Errors::InvalidGroupId` so user code can
4802 /// dispatch on the error code.
4803 fn return_error_if_group_id_not_defined(&self) -> Result<(), Error> {
4804 return_error_if_group_id_not_defined(self.group_id.as_deref())
4805 }
4806
4807 /// Translates Java's `acquireAndEnsureOpen()` — the runtime
4808 /// reentrancy guard is dropped per Phase 11 PLAN.md deferral #4
4809 /// (Rust's `&mut self` enforces single-caller exclusivity at
4810 /// compile time), so this is the `closed` check followed by
4811 /// `metadata.maybeThrowBootstrapFatalException()` (KIP-909): once
4812 /// asynchronous bootstrap resolution has failed, every `Result`-returning
4813 /// API that acquires the consumer returns the
4814 /// [`BootstrapResolutionError`](crate::common::errors::BootstrapResolutionError).
4815 /// The sync accessors (`assignment`, `subscription`, `paused`,
4816 /// `group_metadata`) have no error channel: they panic on a closed
4817 /// consumer and keep returning their value after a bootstrap failure —
4818 /// see the "Sync state-read methods" comment.
4819 fn ensure_open(&self) -> Result<(), Error> {
4820 if self.is_closed() {
4821 return Err(Error::local_illegal_state(CONSUMER_CLOSED_MSG));
4822 }
4823 self.metadata.maybe_return_bootstrap_fatal_error()
4824 }
4825
4826 /// Java: `void subscribe(Collection<String>)`.
4827 ///
4828 /// Subscribes to the given topics. An empty list acts as
4829 /// `unsubscribe()`. Errors:
4830 /// - [`Error::local_illegal_argument`] if any topic is empty / whitespace.
4831 /// - [`Error::invalid_group_id`] if `group.id` is unset
4832 /// (Java's `InvalidGroupIdException`).
4833 pub async fn subscribe_with_topics(&mut self, topics: Vec<String>) -> Result<(), Error> {
4834 self.subscribe_internal_topics(topics, None).await
4835 }
4836
4837 /// Java: `void subscribe(Collection<String>, ConsumerRebalanceListener)`.
4838 ///
4839 /// Same as [`Self::subscribe_with_topics`] but registers a rebalance listener.
4840 /// Java throws `IllegalArgumentException` for a null listener;
4841 /// Rust makes the `Option`-of-`Arc` representation explicit, so the
4842 /// listener form takes a concrete `Arc` and the listener is
4843 /// always non-null at the type level.
4844 pub async fn subscribe_with_topics_listener(
4845 &mut self,
4846 topics: Vec<String>,
4847 listener: Arc<dyn ConsumerRebalanceListener>,
4848 ) -> Result<(), Error> {
4849 self.subscribe_internal_topics(topics, Some(listener)).await
4850 }
4851
4852 /// Java: `void subscribe(SubscriptionPattern)` — server-side regex
4853 /// subscribe (KIP-848 RE2J).
4854 pub async fn subscribe_with_pattern(&mut self, pattern: SubscriptionPattern) -> Result<(), Error> {
4855 self.subscribe_to_regex(pattern, None).await
4856 }
4857
4858 /// Java: `void subscribe(SubscriptionPattern, ConsumerRebalanceListener)`.
4859 pub async fn subscribe_with_pattern_listener(
4860 &mut self,
4861 pattern: SubscriptionPattern,
4862 listener: Arc<dyn ConsumerRebalanceListener>,
4863 ) -> Result<(), Error> {
4864 self.subscribe_to_regex(pattern, Some(listener)).await
4865 }
4866
4867 // Java's `void subscribe(Pattern)` / `void subscribe(Pattern,
4868 // ConsumerRebalanceListener)` — client-side regex subscribe — are
4869 // deliberately NOT implemented in Rust. Only the `SubscriptionPattern`
4870 // form above is translated; it hands the pattern to the group coordinator
4871 // for server-side RE2/J evaluation (KIP-848) instead of matching it
4872 // against the consumer's own metadata.
4873
4874 /// Translates Java's `subscribeInternal(Collection<String>, Optional<ConsumerRebalanceListener>)`.
4875 async fn subscribe_internal_topics(
4876 &mut self,
4877 topics: Vec<String>,
4878 listener: Option<Arc<dyn ConsumerRebalanceListener>>,
4879 ) -> Result<(), Error> {
4880 self.ensure_open()?;
4881 self.return_error_if_group_id_not_defined()?;
4882
4883 if topics.is_empty() {
4884 // Java: `topics.isEmpty()` is treated as the same as
4885 // `unsubscribe()`. Match the recursion.
4886 return self.unsubscribe().await;
4887 }
4888
4889 for topic in &topics {
4890 if topic.trim().is_empty() {
4891 return Err(Error::local_illegal_argument(
4892 "Topic collection to subscribe to cannot contain null or empty topic",
4893 ));
4894 }
4895 }
4896
4897 log::info!("Subscribed to topic(s): {}", topics.join(", "));
4898
4899 let topics_set: std::collections::HashSet<String> = topics.into_iter().collect();
4900 let deadline_ms = self.default_api_timeout_deadline_ms();
4901 let (handle, receiver, _erased) = CompletableEvent::make_completable_event::<()>(deadline_ms);
4902 // Java passes the listener INSIDE the event so the bg task owns
4903 // installation — the app side never registers the listener until
4904 // the event has been accepted. Mirror this by sending the
4905 // listener through the event AND only mirroring it into the
4906 // app-side `rebalance_listener` slot AFTER the submit resolves
4907 // `Ok(())` (so a failed submission does not leave the app-side
4908 // slot pointing at a listener that never landed in
4909 // `SubscriptionState`).
4910 //
4911 // The mirror is written UNCONDITIONALLY, including with `None`:
4912 // Java has a single slot (`SubscriptionState.rebalanceListener`)
4913 // and `subscribe(topics)` without a listener calls
4914 // `registerRebalanceListener(Optional.empty())`
4915 // (`org.apache.kafka.clients.consumer.internals.SubscriptionState#subscribe(Set,Optional)`), so a listener-less
4916 // subscribe must CLEAR the previous registration. Skipping the write for
4917 // `None` would leave the app-side mirror pointing at the replaced
4918 // listener, which `leave_group_on_close` would then wrongly invoke
4919 // (and which would keep the listener alive past its release
4920 // point).
4921 let listener_for_app_side = listener.as_ref().map(Arc::clone);
4922 // Java's `subscribe(...)` does NOT call `setActiveTask` — match
4923 // by passing `enable_wakeup=false`.
4924 self.submit_and_await::<()>(
4925 ApplicationEvent::TopicSubscriptionChange { handle, topics: topics_set, listener },
4926 receiver,
4927 deadline_ms,
4928 "Timeout expired while waiting for the subscribe event to complete",
4929 false,
4930 )
4931 .await?;
4932 *self.rebalance_listener.lock().unwrap() = listener_for_app_side;
4933 Ok(())
4934 }
4935
4936 /// Translates Java's `subscribeToRegex(SubscriptionPattern, Optional<ConsumerRebalanceListener>)`.
4937 async fn subscribe_to_regex(
4938 &mut self,
4939 pattern: SubscriptionPattern,
4940 listener: Option<Arc<dyn ConsumerRebalanceListener>>,
4941 ) -> Result<(), Error> {
4942 self.ensure_open()?;
4943 self.return_error_if_group_id_not_defined()?;
4944 if pattern.pattern().is_empty() {
4945 return Err(Error::local_illegal_argument("Topic pattern to subscribe to cannot be empty"));
4946 }
4947
4948 log::info!("Subscribing to regular expression {}", pattern.pattern());
4949
4950 let deadline_ms = self.default_api_timeout_deadline_ms();
4951 let (handle, receiver, _erased) = CompletableEvent::make_completable_event::<()>(deadline_ms);
4952 // See `subscribe_internal_topics` for why we store the listener
4953 // only after the submit resolves Ok.
4954 let listener_for_app_side = listener.as_ref().map(Arc::clone);
4955 self.submit_and_await::<()>(
4956 ApplicationEvent::TopicRe2JPatternSubscriptionChange { handle, pattern, listener },
4957 receiver,
4958 deadline_ms,
4959 "Timeout expired while waiting for the subscribe-regex event to complete",
4960 false,
4961 )
4962 .await?;
4963 *self.rebalance_listener.lock().unwrap() = listener_for_app_side;
4964 Ok(())
4965 }
4966
4967 /// Java: `void unsubscribe()`.
4968 ///
4969 /// Unsubscribes from all topics / patterns and clears the assignment.
4970 /// Java enqueues an `UnsubscribeEvent` and then loops
4971 /// `processBackgroundEvents(future, timer, ignoreErrorPredicate)` so
4972 /// any rebalance-listener callbacks raised during the teardown can be
4973 /// driven from the caller's task while the unsubscribe future is
4974 /// outstanding (Java `AsyncKafkaConsumer#unsubscribe`). The Rust
4975 /// translation routes the handle's receiver through
4976 /// [`Self::process_background_events_until`] so the same interleaved
4977 /// drain happens — without it, the bg task would await the
4978 /// listener-callback ack indefinitely while the app side blocks on
4979 /// `add_and_get`.
4980 pub async fn unsubscribe(&mut self) -> Result<(), Error> {
4981 // Java's `acquireAndEnsureOpen()` sits OUTSIDE the `try`
4982 // (`org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer#unsubscribe`), so a closed-consumer failure
4983 // is not covered by the `catch (Exception e) { log.error("Unsubscribe
4984 // failed", e); throw e; }` below — match that by returning before the
4985 // guarded section.
4986 self.ensure_open()?;
4987
4988 // Everything Java runs inside its `try` lives in the inner helper, so
4989 // that a failure escaping it takes Java's outer-catch path: log
4990 // "Unsubscribe failed" and propagate WITHOUT resetting the group
4991 // metadata (Java's `resetGroupMetadata()` is the last statement of
4992 // the `try`, and is skipped when the `try` throws).
4993 let result = self.unsubscribe_inner().await;
4994
4995 // NOTE: the app-side `rebalance_listener` mirror is deliberately NOT
4996 // cleared here. Java's `SubscriptionState.unsubscribe()`
4997 // (`org.apache.kafka.clients.consumer.internals.SubscriptionState#unsubscribe`) clears the subscription, the
4998 // assignment, the pattern and the subscription type but leaves
4999 // `rebalanceListener` alone — `registerRebalanceListener` is only ever
5000 // called from the three `subscribe(...)` overloads — and
5001 // `AsyncKafkaConsumer#unsubscribe` does not touch it
5002 // either. Since Java has ONE slot and the mirror is what
5003 // `process_background_events` reads to invoke the callback, clearing it
5004 // here would silently skip a `PartitionsRemoved` event that the bg task
5005 // enqueued while the registration was still live but the app drains
5006 // after `unsubscribe()` returns. The retained listener is released by
5007 // the next `subscribe(...)` (which writes the mirror unconditionally,
5008 // `None` included) or when the consumer is dropped.
5009
5010 match result {
5011 Ok(()) => {
5012 // Java: `resetGroupMetadata()` in `unsubscribe` — clear the cached
5013 // generation_id / member_id, preserving the old group_id +
5014 // group_instance_id (the slot stays Some(...) so subsequent
5015 // group_metadata() observations match Java's
5016 // "post-unsubscribe" contract; see Issue 21).
5017 self.state_notifier.reset_group_metadata();
5018 Ok(())
5019 },
5020 Err(Error::Timeout(msg)) => {
5021 // Java's inner `catch (TimeoutException e)` logs an error and
5022 // falls through to `resetGroupMetadata()`, so the unsubscribe
5023 // still reports success.
5024 log::error!("Failed while waiting for the unsubscribe event to complete: {msg}");
5025 self.state_notifier.reset_group_metadata();
5026 Ok(())
5027 },
5028 Err(err) => {
5029 // Java's outer `catch (Exception e)`: log and rethrow, with no
5030 // `resetGroupMetadata()` — the caller can still inspect
5031 // `group_metadata()`'s member_id / generation_id.
5032 log::error!("Unsubscribe failed: {err}");
5033 Err(err)
5034 },
5035 }
5036 }
5037
5038 /// The body of Java's `unsubscribe()` `try` block
5039 /// (`AsyncKafkaConsumer#unsubscribe`, excluding the trailing
5040 /// `resetGroupMetadata()`).
5041 ///
5042 /// Split out so that the caller can reproduce Java's outer
5043 /// `catch (Exception e)` — which logs and rethrows without resetting the
5044 /// group metadata — while the inner `catch (TimeoutException e)` result is
5045 /// still distinguishable.
5046 async fn unsubscribe_inner(&mut self) -> Result<(), Error> {
5047 self.fetch_buffer.retain_all(&std::collections::HashSet::new());
5048
5049 let assigned_for_log = {
5050 let subs = self.subscriptions.lock().unwrap();
5051 subs.assigned_partitions()
5052 };
5053 log::info!("Unsubscribing all topics or patterns and assigned partitions {assigned_for_log:?}");
5054
5055 let now_ms = self.time.milliseconds();
5056 let deadline_ms = self.default_api_timeout_deadline_ms();
5057 let (handle, receiver, _erased) = CompletableEvent::make_completable_event::<()>(deadline_ms);
5058 // Enqueue the event without blocking on it — the iterative drain
5059 // below polls the receiver.
5060 self.application_event_handler
5061 .add(ApplicationEvent::Unsubscribe { handle }, now_ms)?;
5062
5063 // Java's `ignoreErrorEventException` predicate: swallow
5064 // `GroupAuthorizationException` / `TopicAuthorizationException`
5065 // surfaced as fatal background errors during unsubscribe so the
5066 // unsubscribe still completes. Rust surfaces these as
5067 // [`Error::TopicAuthorization`] / [`Error::GroupAuthorization`].
5068 let ignore_predicate = |err: &Error| matches!(err, Error::TopicAuthorization(_) | Error::GroupAuthorization(_));
5069
5070 // Java's `unsubscribe()` does NOT call
5071 // `wakeupTrigger.setActiveTask(...)` (see
5072 // `AsyncKafkaConsumer#unsubscribe`), and its wait is not wakeable.
5073 self.process_background_events_until::<()>(
5074 receiver,
5075 deadline_ms,
5076 ignore_predicate,
5077 "Failed while waiting for the unsubscribe event to complete",
5078 // AK 4.3.1 (KAFKA-20428): unsubscribe passes
5079 // `skipAssignmentEvents = true` so a `PartitionsAssigned`
5080 // event queued by an in-flight reconciliation is not applied
5081 // (the consumer is already leaving).
5082 /* skip_assignment_events = */
5083 true,
5084 )
5085 .await
5086 }
5087
5088 /// Java: `void assign(Collection<TopicPartition>)`.
5089 ///
5090 /// Manually assigns the given partitions. An empty collection acts
5091 /// as `unsubscribe()`. Errors:
5092 /// - [`Error::local_illegal_argument`] if any topic is empty / whitespace.
5093 pub async fn assign(&mut self, partitions: Vec<TopicPartition>) -> Result<(), Error> {
5094 self.ensure_open()?;
5095
5096 if partitions.is_empty() {
5097 return self.unsubscribe().await;
5098 }
5099
5100 for tp in &partitions {
5101 if tp.topic().trim().is_empty() {
5102 return Err(Error::local_illegal_argument(
5103 "Topic partitions to assign to cannot have null or empty topic",
5104 ));
5105 }
5106 }
5107
5108 let partitions_set: std::collections::HashSet<TopicPartition> = partitions.into_iter().collect();
5109 // Java's `assign`: `fetchBuffer.retainAll(currentTopicPartitions)`
5110 // — drop buffered fetches for partitions that are no longer
5111 // assigned so the next poll() doesn't surface stale records.
5112 self.fetch_buffer.retain_all(&partitions_set);
5113 let now_ms = self.time.milliseconds();
5114 let deadline_ms = self.default_api_timeout_deadline_ms();
5115 let (handle, receiver, _erased) = CompletableEvent::make_completable_event::<()>(deadline_ms);
5116 self.submit_and_await::<()>(
5117 ApplicationEvent::AssignmentChange { handle, current_time_ms: now_ms, partitions: partitions_set },
5118 receiver,
5119 deadline_ms,
5120 "Timeout expired while waiting for the assignment-change event to complete",
5121 // Java's `assign(...)` does NOT call `setActiveTask` — match.
5122 false,
5123 )
5124 .await
5125 }
5126
5127 /// Java: `defaultApiTimeoutDeadlineMs()`.
5128 fn default_api_timeout_deadline_ms(&self) -> i64 {
5129 CompletableEvent::calculate_deadline_ms(self.time.milliseconds(), self.default_api_timeout_ms)
5130 }
5131
5132 // ── §31 process_background_events ─────────────────────────────────
5133 //
5134 // Per `consumer-threading.md` §31, only `poll()` (each
5135 // `check_inflight_poll` pass) and `unsubscribe()` call this, as Java
5136 // calls `processBackgroundEvents()` only from those. It drains the
5137 // bg-event channel via `try_recv` in a `while let` loop and dispatches
5138 // each event on the caller's task — listener callbacks invoke the
5139 // user-supplied `ConsumerRebalanceListener` inline.
5140
5141 /// Drains the background-events channel and dispatches each event
5142 /// on the caller's task. Mirrors Java's `boolean processBackgroundEvents()`.
5143 ///
5144 /// Returns:
5145 /// - `Ok(had_events)` on success (no error events drained). The
5146 /// boolean reflects Java's return — `true` if any events were
5147 /// processed in this call, `false` otherwise. Used by
5148 /// [`Self::process_background_events_until`] to decide whether
5149 /// to keep spinning the drain loop or fall through to the
5150 /// bounded `pollInterval` wait (Java
5151 /// `AsyncKafkaConsumer#processBackgroundEvents(Future,Timer,Predicate,boolean)`).
5152 /// - `Err(Error)` on the first error event drained. Subsequent
5153 /// events are still processed (mirroring Java's
5154 /// `firstError.compareAndSet`); the additional errors are logged
5155 /// at `warn` level.
5156 ///
5157 /// Always invokes the background-event reaper at the end of the
5158 /// drain (Java: `backgroundEventReaper.reap(time.milliseconds())`),
5159 /// ensuring expired `CompletableEvent`s do not accumulate.
5160 ///
5161 /// # Lock discipline (§16)
5162 ///
5163 /// This method MUST NOT hold the `SubscriptionState` mutex guard
5164 /// across the listener invocation. The implementation does not
5165 /// acquire the guard at all — the listener invoker reads paused
5166 /// partitions inside its own brief lock window.
5167 pub(crate) async fn process_background_events(&mut self) -> Result<bool, Error> {
5168 self.process_background_events_inner(false).await
5169 }
5170
5171 /// Inner implementation of [`Self::process_background_events`]
5172 /// (Java's `processBackgroundEvents(boolean skipAssignmentEvents)`).
5173 ///
5174 /// `skip_assignment_events` (AK 4.3.1, KAFKA-20428) — Java's
5175 /// `skipAssignmentEvents`. When `true` (`unsubscribe()`), a
5176 /// `PartitionsAssigned` event is NOT processed (no assignment applied, no
5177 /// callback run); its ack is completed with an error to unblock the bg
5178 /// reconciliation, and it is NOT recorded into `first_error`. These
5179 /// assignment-update events are only relevant from `poll()`; during
5180 /// `unsubscribe()` the consumer is already leaving, so applying a new
5181 /// assignment would be wrong. `PartitionsRemoved` (revoke / lost) events
5182 /// are always processed regardless of this flag (they are relevant during
5183 /// unsubscribe so the user can flush offsets).
5184 async fn process_background_events_inner(&mut self, skip_assignment_events: bool) -> Result<bool, Error> {
5185 let mut first_error: Option<Error> = None;
5186 let mut had_events = false;
5187 // Java records `recordBackgroundEventQueueProcessingTime(now - startMs)`
5188 // for the whole drained batch (after `drainEvents`). The Rust drain is
5189 // incremental (`try_recv` loop), so capture `start_ms` before the loop
5190 // and record once after. Clone the metrics `Arc` up front so `&mut self`
5191 // stays usable in the loop body.
5192 let async_consumer_metrics = Arc::clone(&self.async_consumer_metrics);
5193 let start_ms = self.time.milliseconds();
5194
5195 // Java `BackgroundEventHandler.drainEvents` records
5196 // `recordBackgroundEventQueueSize(0)` UNCONDITIONALLY on every drain —
5197 // there is no `isEmpty()` early-return (unlike `processApplicationEvents`).
5198 // `process_background_events` runs from `poll()` and `unsubscribe()`
5199 // (Java's only drain sites), so a polling consumer refreshes this gauge
5200 // even while idle. That unconditional refresh is preserved: the record
5201 // below runs on every drain, empty or not.
5202 //
5203 // What is NOT preserved is the literal `0`, and deliberately so. Java's
5204 // `drainTo` calls `fullyLock()`, so nothing can arrive mid-drain and `0`
5205 // is exact. The `try_recv` loop below does not block senders, so storing
5206 // `0` up front would overwrite the `fetch_add(1)` of any `add` racing the
5207 // loop whose event is still queued. Instead each received envelope
5208 // decrements by one, leaving the counter conserved (`+1` per successful
5209 // send in `BackgroundEventHandler::add`, `-1` per dequeue) and therefore
5210 // equal to the true depth at every observation point.
5211
5212 loop {
5213 let envelope = match self.background_event_rx.try_recv() {
5214 Ok(env) => env,
5215 Err(mpsc::error::TryRecvError::Empty) => break,
5216 Err(mpsc::error::TryRecvError::Disconnected) => {
5217 // The bg task has shut down. Nothing more to drain;
5218 // surface only if no other error has been recorded.
5219 if first_error.is_none() && !self.is_closed() {
5220 first_error =
5221 Some(Error::local_illegal_state("Consumer background task is no longer running."));
5222 }
5223 break;
5224 },
5225 };
5226 had_events = true;
5227 // Conserve the depth counter: one dequeue, one decrement.
5228 self.background_event_queue_size.fetch_sub(1, Ordering::SeqCst);
5229 // Java: record the time this event spent in the queue.
5230 async_consumer_metrics.record_background_event_queue_time(self.time.milliseconds() - envelope.enqueued_ms);
5231
5232 match envelope.event {
5233 BackgroundEvent::Error { error } => {
5234 Self::record_first_error(&mut first_error, error);
5235 },
5236 BackgroundEvent::PartitionsRemoved { method_name, partitions, ack, ack_through_application_events } => {
5237 // AK 4.3.1 (KAFKA-20106): `process(PartitionsRemovedEvent)`
5238 // → `invokeRebalanceCallbackAndNotifyBackgroundThread`.
5239 // `method_name` is `ON_PARTITIONS_REVOKED` or
5240 // `ON_PARTITIONS_LOST` (assign is a separate event now).
5241 //
5242 // Read the currently-registered listener and drop the
5243 // guard before invoking (§16 / §31). The
5244 // `rebalance_listener` lock is separate from
5245 // `SubscriptionState`, so no recursive lock concern.
5246 let listener = self.rebalance_listener.lock().unwrap().clone();
5247
5248 // Holds the ack across the listener `.await`, so a leave's
5249 // ack still goes back behind the listener's ops if this
5250 // future is dropped mid-listener.
5251 let ack_guard = PartitionsRemovedAckGuard {
5252 application_event_handler: Arc::clone(&self.application_event_handler),
5253 time: Arc::clone(&self.time),
5254 method_name,
5255 ack: Some(ack),
5256 ack_through_application_events,
5257 };
5258
5259 // The listener may call back into the consumer through
5260 // its `ConsumerHandle` (Java: the same thread re-enters
5261 // `acquire()`); the marker admits it for the duration.
5262 let _in_callback = self.offset_commit_callback_invoker.enter_callback();
5263 let result = match listener {
5264 Some(listener) => {
5265 // Invoke on the caller's task — never `tokio::spawn`.
5266 use crate::consumer::internals::ConsumerRebalanceListenerMethodName as M;
5267 match method_name {
5268 M::OnPartitionsRevoked => {
5269 self.rebalance_listener_invoker
5270 .invoke_partitions_revoked(&listener, &partitions)
5271 .await
5272 },
5273 M::OnPartitionsLost => {
5274 self.rebalance_listener_invoker
5275 .invoke_partitions_lost(&listener, &partitions)
5276 .await
5277 },
5278 // A `PartitionsRemoved` event never carries
5279 // `ON_PARTITIONS_ASSIGNED` (that path is the
5280 // `PartitionsAssigned` event); treat defensively
5281 // as revoked to stay Java-faithful.
5282 M::OnPartitionsAssigned => {
5283 self.rebalance_listener_invoker
5284 .invoke_partitions_revoked(&listener, &partitions)
5285 .await
5286 },
5287 }
5288 },
5289 None => Ok(()),
5290 };
5291
5292 let result = result.map_err(|err| {
5293 crate::consumer::internals::ConsumerUtils::maybe_wrap_as_kafka_error_with_msg(
5294 err,
5295 "User rebalance callback throws an error",
5296 )
5297 });
5298
5299 // Return the result and poke the bg-task wakeup
5300 // `Notify` (Phase 41b) so the bg loop wakes promptly and
5301 // `try_recv`s this ack on its next `reconcile` entry —
5302 // rather than waiting out the selector poll timeout. This
5303 // reuses the existing application-event wakeup primitive
5304 // (Java's `wakeupNetworkThread()` → `Selector.wakeup()`
5305 // analog); it does NOT shrink `poll_wait_time_ms` (no
5306 // busy-spin — Perf Contract item 2). A leave's callback
5307 // returns through the application-event queue instead,
5308 // whose `add` wakes the loop the same way.
5309 //
5310 // The wake MUST be the application-event notify, NOT
5311 // `network_thread_close.wakeup()` / `wakeup_trigger.wakeup()`.
5312 // Those cancel the wakeup token, which is Java's
5313 // `KafkaConsumer.wakeup()` — it arms a user-visible
5314 // `Error::Wakeup` that the *next* public API call
5315 // raises (§11). Since every rebalance fires a listener
5316 // callback, using it here made a spurious `Wakeup` the
5317 // normal outcome of any rebalance, breaking `poll()` for
5318 // every consumer with a listener registered. Both signals
5319 // reach the selector through `run_once`'s `select!`, so
5320 // this wakes the loop just as promptly with no
5321 // user-visible side effect.
5322 ack_guard.complete(result.clone());
5323
5324 if let Err(err) = result {
5325 Self::record_first_error(&mut first_error, err);
5326 }
5327 },
5328 BackgroundEvent::PartitionsAssigned { assigned_partitions, added_partitions, ack }
5329 if skip_assignment_events =>
5330 {
5331 // AK 4.3.1 (KAFKA-20428): during unsubscribe, skip
5332 // processing assignment-update events — they are only
5333 // relevant from `poll()`. Java completes the event's future
5334 // EXCEPTIONALLY to unblock the reconciliation in the
5335 // background, logs, and continues WITHOUT recording the
5336 // error into `firstError`.
5337 let _ = assigned_partitions;
5338 let _ = added_partitions;
5339 // Java: `new KafkaException("Assignment event skipped ...")` — a
5340 // bare KafkaException carries no error code; use the neutral
5341 // `UnknownServerError` code while preserving the message text.
5342 let _ = ack.send(Err(Error::with_message(
5343 crate::common::protocol::Errors::UnknownServerError,
5344 "Assignment event skipped because consumer is unsubscribing",
5345 )));
5346 log::debug!("Skipped processing PartitionsAssigned during unsubscribe");
5347 self.application_event_handler.wake_background_task();
5348 },
5349 BackgroundEvent::PartitionsAssigned { assigned_partitions, added_partitions, ack } => {
5350 // AK 4.3.1 (KAFKA-20106): `process(PartitionsAssignedEvent)`.
5351 // 1. Apply the new assignment on the bg thread, triggered
5352 // and awaited here so `consumer.assignment()` only
5353 // changes within `poll()` (`applyNewAssignment` →
5354 // `ApplyAssignmentEvent`).
5355 // 2. If a listener is registered, run `on_partitions_assigned`.
5356 // 3. Reply on the ack (the `PartitionsAssignedEvent.future()`)
5357 // so the bg reconciliation resumes.
5358 let assigned_set: HashSet<TopicPartition> = assigned_partitions.iter().cloned().collect();
5359
5360 // Step 1 — applyNewAssignment: enqueue + await the
5361 // ApplyAssignmentEvent. Java `addAndGet(applyEvent)`.
5362 let apply_result = {
5363 let now_ms = self.time.milliseconds();
5364 let apply_deadline = i64::MAX; // Java: ApplyAssignmentEvent deadlineMs = Long.MAX_VALUE
5365 let (apply_handle, apply_rx, _erased) =
5366 CompletableEvent::make_completable_event::<()>(apply_deadline);
5367 self.application_event_handler
5368 .add_and_get::<()>(
5369 ApplicationEvent::ApplyAssignment {
5370 handle: apply_handle,
5371 assigned_partitions: assigned_set,
5372 added_partitions: added_partitions.clone(),
5373 },
5374 apply_rx,
5375 now_ms,
5376 )
5377 .await
5378 };
5379
5380 if let Err(err) = apply_result {
5381 // Java: wrap as "Failed to apply the new assignment",
5382 // complete the event future exceptionally, and throw
5383 // (recorded into firstError here).
5384 let wrapped = crate::consumer::internals::ConsumerUtils::maybe_wrap_as_kafka_error_with_msg(
5385 err,
5386 "Failed to apply the new assignment",
5387 );
5388 let _ = ack.send(Err(wrapped.clone()));
5389 self.application_event_handler.wake_background_task();
5390 Self::record_first_error(&mut first_error, wrapped);
5391 continue;
5392 }
5393
5394 // Steps 2 + 3 — run `on_partitions_assigned` if a listener
5395 // exists, else complete the future with success.
5396 let listener = self.rebalance_listener.lock().unwrap().clone();
5397 // Admits the listener's `ConsumerHandle` calls (see the
5398 // `PartitionsRemoved` arm above).
5399 let _in_callback = self.offset_commit_callback_invoker.enter_callback();
5400 let result = match listener {
5401 Some(listener) => {
5402 self.rebalance_listener_invoker
5403 .invoke_partitions_assigned(&listener, &added_partitions)
5404 .await
5405 },
5406 None => Ok(()),
5407 };
5408 let result = result.map_err(|err| {
5409 crate::consumer::internals::ConsumerUtils::maybe_wrap_as_kafka_error_with_msg(
5410 err,
5411 "User rebalance callback throws an error",
5412 )
5413 });
5414 let send_result = result.clone();
5415 let _ = ack.send(send_result);
5416 self.application_event_handler.wake_background_task();
5417
5418 if let Err(err) = result {
5419 Self::record_first_error(&mut first_error, err);
5420 }
5421 },
5422 }
5423 }
5424
5425 // The depth refresh Java does before its drain (see the note above it).
5426 // Recorded here instead, and unconditionally, so it reflects what the
5427 // conserved counter actually holds after the loop — including anything
5428 // enqueued while the loop was running.
5429 // Floored at 0 for the same reason as the application-event gauge.
5430 async_consumer_metrics
5431 .record_background_event_queue_size(self.background_event_queue_size.load(Ordering::SeqCst).max(0) as i32);
5432
5433 // Java: record the total processing time for the drained
5434 // batch (only when at least one event was processed, matching Java's
5435 // `if (!events.isEmpty())` guard).
5436 if had_events {
5437 async_consumer_metrics.record_background_event_queue_processing_time(self.time.milliseconds() - start_ms);
5438 }
5439
5440 // Java: reap expired completable events regardless of
5441 // drain outcome. Done after the drain so events added by this
5442 // tick get a chance to land before being reaped.
5443 {
5444 let now_ms = self.time.milliseconds();
5445 let mut reaper = self.completable_event_reaper.lock().unwrap();
5446 reaper.reap(now_ms);
5447 }
5448
5449 match first_error {
5450 Some(err) => Err(err),
5451 None => Ok(had_events),
5452 }
5453 }
5454
5455 /// Iterative variant of [`Self::process_background_events`] used by
5456 /// `unsubscribe()`, the one blocking-style API besides `poll()` that
5457 /// interleaves bg-event draining with waiting on its own event.
5458 ///
5459 /// Mirrors Java's
5460 /// `<T> T processBackgroundEvents(Future<T> future, Timer timer,
5461 /// Predicate<Exception> ignoreErrorEventException,
5462 /// boolean skipAssignmentEvents)`.
5463 /// Each iteration:
5464 ///
5465 /// 1. Drains the bg-event channel (invokes any pending listener
5466 /// callbacks on the caller's task); an error the predicate does not
5467 /// ignore is returned.
5468 /// 2. Returns the result if the event is done.
5469 /// 3. Otherwise, when the drain ran no events, waits up to 100 ms for
5470 /// the result.
5471 /// 4. Loops while the absolute `deadline_ms` is not exceeded.
5472 ///
5473 /// The wait is not wakeable: Java's `unsubscribe()` does not call
5474 /// `wakeupTrigger.setActiveTask(...)`.
5475 ///
5476 /// Returns `Err(Error::timeout(...))` when the deadline expires without
5477 /// a completion.
5478 async fn process_background_events_until<T: Send + 'static>(
5479 &mut self,
5480 mut receiver: tokio::sync::oneshot::Receiver<Result<T, Error>>,
5481 deadline_ms: i64,
5482 ignore_error_predicate: impl Fn(&Error) -> bool,
5483 timeout_msg: impl AsRef<str>,
5484 skip_assignment_events: bool,
5485 ) -> Result<T, Error> {
5486 loop {
5487 let had_events = match self.process_background_events_inner(skip_assignment_events).await {
5488 Ok(had) => had,
5489 Err(err) => {
5490 if ignore_error_predicate(&err) {
5491 // Java swallows the matched exception inside the
5492 // try/catch around `processBackgroundEvents()`.
5493 false
5494 } else {
5495 return Err(err);
5496 }
5497 },
5498 };
5499
5500 // Java: `if (future.isDone()) return getResult(future)`.
5501 if let Some(result) = try_take_completion(&mut receiver) {
5502 return result;
5503 }
5504
5505 // Java: if no events were processed this tick, do a
5506 // bounded wait (100ms) for the future.
5507 if !had_events {
5508 let remaining = self.remaining_ms(deadline_ms);
5509 if remaining <= 0 {
5510 return Err(Error::timeout(timeout_msg.as_ref().to_string()));
5511 }
5512 let wait = std::cmp::min(remaining, 100) as u64;
5513 match tokio::time::timeout(Duration::from_millis(wait), &mut receiver).await {
5514 Ok(Ok(result)) => return result,
5515 Ok(Err(_recv_err)) => {
5516 return Err(Error::local_illegal_state(
5517 "Background task dropped the completion sender without completing it",
5518 ));
5519 },
5520 // Java's `swallow TimeoutException` — keep looping.
5521 Err(_elapsed) => {},
5522 }
5523 }
5524
5525 // Java: `while (timer.notExpired())`.
5526 if self.remaining_ms(deadline_ms) <= 0 {
5527 return Err(Error::timeout(timeout_msg.as_ref().to_string()));
5528 }
5529 }
5530 }
5531
5532 /// Submit `event` and await its typed completion: Java's
5533 /// `applicationEventHandler.addAndGet(event)`, preceded by
5534 /// `wakeupTrigger.setActiveTask(event.future())` when `enable_wakeup`.
5535 ///
5536 /// This is the wait of every blocking-style consumer API except `poll()`
5537 /// and `unsubscribe()`. It does NOT drain the background-event queue:
5538 /// Java runs `processBackgroundEvents()` only from those two, so a queued
5539 /// rebalance callback or `ErrorEvent` stays queued for the next `poll()`
5540 /// (`consumer-threading.md` §31). That is safe because the bg loop never
5541 /// waits for a listener ack while it services other events: reconcile and
5542 /// release acks are `try_recv`ed across iterations, and the only awaited
5543 /// ack belongs to the `unsubscribe()` leave, whose caller drains.
5544 ///
5545 /// `enable_wakeup` mirrors Java's per-API `setActiveTask` decision. Java
5546 /// calls `wakeupTrigger.setActiveTask(event.future())` before
5547 /// `applicationEventHandler.addAndGet(event)` (`committed`, `partitionsFor`,
5548 /// `listTopics`, `updateFetchPositions`: `AsyncKafkaConsumer#committed(Set,Duration)`,
5549 /// `#partitionsFor(String,Duration)`, `#listTopics(Duration)`, `#updateFetchPositions`), so a wakeup pending when
5550 /// the call starts always fails it, the future being not yet done: the pending wakeup is
5551 /// raised (and the token rotated) before the event is submitted. Java's
5552 /// `addAndGet` still enqueues that event, but its future is failed by then
5553 /// and nothing reads what the background thread does with it, so the
5554 /// event is not submitted here.
5555 async fn submit_and_await<T: Send + 'static>(
5556 &self,
5557 event: ApplicationEvent,
5558 receiver: tokio::sync::oneshot::Receiver<Result<T, Error>>,
5559 deadline_ms: i64,
5560 timeout_msg: impl AsRef<str>,
5561 enable_wakeup: bool,
5562 ) -> Result<T, Error> {
5563 if enable_wakeup && let Err(err) = self.wakeup_trigger.maybe_trigger_wakeup() {
5564 self.wakeup_trigger.rotate();
5565 return Err(err);
5566 }
5567 let now_ms = self.time.milliseconds();
5568 self.application_event_handler.add(event, now_ms)?;
5569 await_event_completion(
5570 &self.wakeup_trigger,
5571 self.time.as_ref(),
5572 receiver,
5573 deadline_ms,
5574 timeout_msg,
5575 enable_wakeup,
5576 /* result_first_on_entry = */ false,
5577 )
5578 .await
5579 }
5580
5581 /// Awaits an already-submitted event's result without draining the
5582 /// background-event queue: Java's `ConsumerUtils.getResult(future, timer)`
5583 /// (see [`await_event_completion`]). A result already there on entry wins
5584 /// over a pending wakeup, which stays armed for the next call (Java's
5585 /// `setActiveTask` on a done future).
5586 async fn await_completion<T: Send + 'static>(
5587 &self,
5588 receiver: tokio::sync::oneshot::Receiver<Result<T, Error>>,
5589 deadline_ms: i64,
5590 timeout_msg: impl AsRef<str>,
5591 enable_wakeup: bool,
5592 ) -> Result<T, Error> {
5593 await_event_completion(
5594 &self.wakeup_trigger,
5595 self.time.as_ref(),
5596 receiver,
5597 deadline_ms,
5598 timeout_msg,
5599 enable_wakeup,
5600 /* result_first_on_entry = */ true,
5601 )
5602 .await
5603 }
5604
5605 /// Close-path variant of [`Self::submit_and_await`], used by
5606 /// `leave_group_on_close`.
5607 ///
5608 /// Enqueues the event and waits for its result without processing
5609 /// background events, as Java's `addAndGet` does, with these differences:
5610 ///
5611 /// - Wakeups are not observed — close has already disabled them
5612 /// (`wakeup_trigger.disable()`).
5613 /// - The wait is NOT bounded by the event's `deadline_ms`. Java's
5614 /// `addAndGet` blocks on an unbounded `ConsumerUtils.getResult(future)`
5615 /// (`ApplicationEventHandler.addAndGet`): the event's future is
5616 /// completed by the network thread — by its processing, by the reaper
5617 /// with a `TimeoutException` once `deadline_ms` has passed
5618 /// (`ConsumerNetworkThread.runOnce` processes the queue before it
5619 /// reaps), or on shutdown. Returning at `deadline_ms` here instead
5620 /// lets `close(Duration::ZERO)` stop the bg task before it has
5621 /// processed the event, so the member never leaves the group.
5622 /// - Deliberate deviation: Java has no bound at all, so a network
5623 /// thread stuck inside one iteration hangs `close()`. Here the wait
5624 /// gives up with an `Error::Timeout` at
5625 /// `max(deadline_ms, submission time) + max(request.timeout.ms, 100 ms)`.
5626 /// It is measured from the submission at the latest, and its grace is
5627 /// never zero, so it cannot already be expired when the event is
5628 /// enqueued (a long revoked/lost listener, or `request.timeout.ms=0`).
5629 /// With a healthy bg loop the reaper completes the event first, so
5630 /// the backstop changes nothing there.
5631 ///
5632 /// Returns `Ok(None)` when the bg task dropped the completion sender
5633 /// without completing it (it has shut down): there is nothing left to
5634 /// wait for.
5635 async fn submit_and_await_for_close<T: Send + 'static>(
5636 &self,
5637 event: ApplicationEvent,
5638 mut receiver: tokio::sync::oneshot::Receiver<Result<T, Error>>,
5639 deadline_ms: i64,
5640 timeout_msg: impl AsRef<str>,
5641 ) -> Result<Option<T>, Error> {
5642 use tokio::sync::oneshot::error::TryRecvError;
5643
5644 // Smallest grace the backstop gives the bg task after submission, used
5645 // when `request.timeout.ms` is below it (it may be 0): one full
5646 // 100 ms wait slice of the loop below. `add` wakes the bg task, so a
5647 // healthy loop processes the event well within it.
5648 const MIN_BACKSTOP_GRACE_MS: i64 = 100;
5649
5650 let now_ms = self.time.milliseconds();
5651 self.application_event_handler.add(event, now_ms)?;
5652 // The event keeps `deadline_ms` (the reaper expires it there, as
5653 // Java's does); only the backstop is measured from the submission, so
5654 // it is never already expired when the steps before this one (a
5655 // revoked/lost listener, the auto-commit) used up the close timeout.
5656 let grace_ms = std::cmp::max(self.config.request_timeout_ms() as i64, MIN_BACKSTOP_GRACE_MS);
5657 let backstop_deadline_ms = std::cmp::max(deadline_ms, now_ms).saturating_add(grace_ms);
5658 loop {
5659 match receiver.try_recv() {
5660 Ok(result) => return result.map(Some),
5661 Err(TryRecvError::Closed) => return Ok(None),
5662 Err(TryRecvError::Empty) => {},
5663 }
5664 let remaining = self.remaining_ms(backstop_deadline_ms);
5665 if remaining <= 0 {
5666 return Err(Error::timeout(timeout_msg.as_ref().to_string()));
5667 }
5668 // Wait in slices of at most 100 ms, so the backstop is measured on
5669 // the consumer's `Time`.
5670 let wait = Duration::from_millis(std::cmp::min(remaining, 100) as u64);
5671 match tokio::time::timeout(wait, &mut receiver).await {
5672 Ok(Ok(result)) => return result.map(Some),
5673 Ok(Err(_recv_err)) => return Ok(None),
5674 Err(_elapsed) => {},
5675 }
5676 }
5677 }
5678
5679 /// Returns the milliseconds remaining until the supplied deadline,
5680 /// saturating at zero.
5681 fn remaining_ms(&self, deadline_ms: i64) -> i64 {
5682 let now = self.time.milliseconds();
5683 deadline_ms.saturating_sub(now).max(0)
5684 }
5685
5686 /// Java: `KafkaException e = ConsumerUtils.maybeWrapAsKafkaException(t);`
5687 /// followed by `firstError.compareAndSet(null, e)`
5688 /// (`AsyncKafkaConsumer#processBackgroundEvents(boolean)`) — the error is wrapped FIRST, so
5689 /// both the recorded error and the warn-logged one are `KafkaException`s;
5690 /// then first-error-wins, and subsequent errors are logged at `warn`.
5691 ///
5692 /// The wrap is conditional (see
5693 /// [`maybe_wrap_as_kafka_error`](crate::consumer::internals::ConsumerUtils::maybe_wrap_as_kafka_error)):
5694 /// an error already in the `KafkaException` hierarchy passes through
5695 /// unchanged, so this only affects the generic runtime-error variants for
5696 /// which `is_kafka_error()` would otherwise answer `false`.
5697 fn record_first_error(slot: &mut Option<Error>, err: Error) {
5698 let err = crate::consumer::internals::ConsumerUtils::maybe_wrap_as_kafka_error(err);
5699 if slot.is_none() {
5700 *slot = Some(err);
5701 } else {
5702 log::warn!("An error occurred when processing the background event: {err}");
5703 }
5704 }
5705
5706 // ── Poll ───────────────────────────────────────────────────────────
5707 //
5708 // Translates Java's `AsyncKafkaConsumer.poll(Duration timeout)` body
5709 // The Java implementation drives a
5710 // `do { } while (timer.notExpired())` loop with three stages:
5711 //
5712 // 1. `wakeupTrigger.maybeTriggerWakeup()` at the TOP of the loop —
5713 // observe a wakeup posted between polls.
5714 // 2. `checkInflightPoll(timer, firstPass)` — start a new
5715 // `AsyncPollEvent` or evaluate whether the existing one has
5716 // finished. Also runs the per-iteration
5717 // `offsetCommitCallbackInvoker.executeCallbacks()` +
5718 // `processBackgroundEvents()` pair (§31 invocation thread).
5719 // 3. `pollForFetches(timer)` — drain the fetch buffer via
5720 // `FetchCollector.collectFetch`.
5721 //
5722 // Records are returned as soon as the collector yields a non-empty
5723 // fetch; otherwise the loop continues until `timer` expires, at which
5724 // point we return `ConsumerRecords::empty()`.
5725
5726 /// Java: `ConsumerRecords<K, V> poll(Duration timeout)`.
5727 ///
5728 /// The translated body mirrors Java line-for-line; deviations are
5729 /// limited to:
5730 ///
5731 /// - The `try/finally` in Java is realized with an inner helper
5732 /// (`poll_inner`) so `kafkaConsumerMetrics.recordPollEnd` runs on
5733 /// every exit path (the `finally`), matching
5734 /// `AsyncKafkaConsumer#poll`.
5735 /// - `interceptors.onConsume(...)` mutates the records in place via
5736 /// `Mutex<ConsumerInterceptors>`.
5737 ///
5738 /// # Cancellation
5739 ///
5740 /// Dropping the future (a `tokio::time::timeout` or `select!` around it)
5741 /// while a deserializer is pending returns nothing and moves no consumed
5742 /// offset: every position the collect had moved goes back to its offset
5743 /// before the call, with the partition's current leader (a leader change
5744 /// meanwhile is validated as usual), and the records it had decoded are
5745 /// fetched again by a later `poll()` (see `FetchCollector::collect_fetch`).
5746 pub async fn poll(&mut self, timeout: Duration) -> Result<ConsumerRecords<K, V>, Error> {
5747 self.ensure_open()?;
5748
5749 // Java: `kafkaConsumerMetrics.recordPollStart(timer.currentTimeMs())`
5750 // in `org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer#poll` — recorded right after
5751 // `acquireAndEnsureOpen`, before the subscription check. The timer is
5752 // created at `poll()` entry, so `currentTimeMs()` is the entry time.
5753 let start_ms = self.time.milliseconds();
5754 self.kafka_consumer_metrics.record_poll_start(start_ms);
5755
5756 // Java's `try { … } finally { recordPollEnd(...) }`: run the body and
5757 // record poll-end on every exit path (including errors).
5758 let result = self.poll_inner(timeout, start_ms).await;
5759
5760 // Java: `kafkaConsumerMetrics.recordPollEnd(timer.currentTimeMs())`
5761 // in `AsyncKafkaConsumer#poll`.
5762 self.kafka_consumer_metrics.record_poll_end(self.time.milliseconds());
5763 result
5764 }
5765
5766 /// The `try`-body of [`Self::poll`] (`AsyncKafkaConsumer#poll`).
5767 /// Separated so [`Self::poll`] can record `recordPollEnd` in a
5768 /// `finally`-equivalent regardless of how this returns.
5769 async fn poll_inner(&mut self, timeout: Duration, start_ms: i64) -> Result<ConsumerRecords<K, V>, Error> {
5770 // Java: `subscriptions.hasNoSubscriptionOrUserAssignment()`.
5771 {
5772 let subs = self.subscriptions.lock().unwrap();
5773 if subs.has_no_subscription_or_user_assignment() {
5774 return Err(Error::local_illegal_state(
5775 "Consumer is not subscribed to any topics or assigned any partitions",
5776 ));
5777 }
5778 }
5779
5780 let poll_deadline_ms =
5781 CompletableEvent::calculate_deadline_ms(start_ms, ConsumerUtils::duration_to_millis(timeout));
5782 let mut first_pass = true;
5783
5784 loop {
5785 // Stage 1: observe pending wakeup before doing any work.
5786 if let Err(err) = self.wakeup_trigger.maybe_trigger_wakeup() {
5787 // Java throws WakeupException unconditionally; the Rust
5788 // analog rotates the token AFTER raising so a subsequent
5789 // poll() observes a fresh token (consumer-threading.md §11).
5790 self.wakeup_trigger.rotate();
5791 return Err(err);
5792 }
5793
5794 // Stage 2: drive the `AsyncPollEvent` lifecycle. This also
5795 // executes pending OffsetCommitCallback invocations and drains
5796 // background events for §31 listener callbacks.
5797 self.check_inflight_poll(poll_deadline_ms, first_pass).await?;
5798 first_pass = false;
5799
5800 // Stage 3: collect fetched records (blocks on the FetchBuffer
5801 // wakeup when none are buffered yet — see `poll_for_fetches`).
5802 let mut records = self.poll_for_fetches(poll_deadline_ms).await?;
5803 // Java: `if (!fetch.isEmpty())` where `Fetch.isEmpty()` is
5804 // `numRecords == 0 && !positionAdvanced` — NOT the public
5805 // `ConsumerRecords.isEmpty()` (records-only). Returning here when
5806 // only the position advanced (e.g. an all-aborted batch under
5807 // READ_COMMITTED) avoids blocking until the poll timeout
5808 // (`org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer#poll`).
5809 if !records.is_fetch_empty() {
5810 // Java: `sendPrefetches(timer)` — eagerly enqueue the next
5811 // batch of fetches so the user's processing overlaps with
5812 // the next request. In Rust this maps to a non-blocking
5813 // `CreateFetchRequests` event.
5814 self.send_prefetches();
5815 // Java: `interceptors.onConsume(...)`.
5816 {
5817 let chain = self.interceptors.lock().unwrap();
5818 chain.on_consume(&mut records);
5819 }
5820 return Ok(records);
5821 }
5822
5823 // Java: `while (timer.notExpired())`.
5824 if self.time.milliseconds() >= poll_deadline_ms {
5825 break;
5826 }
5827
5828 // Rust-only (CLAUDE.md §11.6; Java's thread spins without starving
5829 // anything): an iteration can come round without ever returning
5830 // `Pending`. `poll_for_fetches` returns at once when
5831 // `maximumTimeToWait` is 0 (Java's value while, for example, a
5832 // JOINING member's first heartbeat is in flight with a zero
5833 // interval, or an auto-commit timer has expired), and
5834 // `await_wakeup` / the reconciliation-check wait complete without
5835 // suspending when the buffer was already woken. Looping straight
5836 // back would monopolise the caller's tokio worker for the whole
5837 // window: no other task on it runs and its I/O driver is not
5838 // polled. Yield once per empty iteration, as `Selector::poll`
5839 // does on its non-blocking path, so co-scheduled tasks progress.
5840 // The background task runs on its own thread, so this only
5841 // changes how the waiting app task shares its worker.
5842 tokio::task::yield_now().await;
5843 }
5844
5845 Ok(ConsumerRecords::empty())
5846 }
5847
5848 /// Java: `private void checkInflightPoll(Timer timer, boolean firstPass)`
5849 /// (`AsyncKafkaConsumer#checkInflightPoll`).
5850 ///
5851 /// Manages the lifecycle of the [`ApplicationEvent::AsyncPoll`] event. If
5852 /// no event is currently processing, a new one is started asynchronously.
5853 /// Each invocation checks whether the *inflight* event has completed; if
5854 /// so, a new event is submitted in its place so a fetch request stays in
5855 /// flight. If the completed event left records buffered no new event is
5856 /// submitted here (it would gate those records behind a fresh
5857 /// validate-positions stage). Instead the buffered records are returned and
5858 /// the next fetch is pipelined by [`Self::poll`] via `send_prefetches`.
5859 ///
5860 /// The pending `OffsetCommitCallback` queue is drained and
5861 /// `process_background_events` is invoked, so a failed callback / fatal
5862 /// background error short-circuits with the inflight event cleared
5863 /// (matching Java's `try { … } catch (Throwable t) { … }`).
5864 async fn check_inflight_poll(&mut self, poll_deadline_ms: i64, first_pass: bool) -> Result<(), Error> {
5865 // Clear the current inflight poll if we can, so a new one (and a new
5866 // fetch) is submitted below. On the first pass this may clear a
5867 // leftover from the previous poll(). On later passes it clears
5868 // inflights that have completed (KAFKA-20780). A completed poll that
5869 // filled the buffer is kept, so its records are returned first (see
5870 // `maybe_clear_previous_inflight_poll`).
5871 if self
5872 .inflight_poll
5873 .as_ref()
5874 .is_some_and(|inflight| first_pass || inflight.state.is_complete())
5875 {
5876 self.maybe_clear_previous_inflight_poll()?;
5877 }
5878
5879 let mut newly_submitted_event = false;
5880 if self.inflight_poll.is_none() {
5881 let state = Arc::new(AsyncPollState::new());
5882 let now_ms = self.time.milliseconds();
5883 let event = ApplicationEvent::AsyncPoll {
5884 deadline_ms: poll_deadline_ms,
5885 poll_time_ms: now_ms,
5886 state: Arc::clone(&state),
5887 };
5888 log::trace!("Inflight event AsyncPoll(deadline={poll_deadline_ms}, time={now_ms}) submitted");
5889 // `add` is non-blocking — bg task drives the state machine.
5890 self.application_event_handler.add(event, now_ms)?;
5891 self.inflight_poll = Some(InflightPoll { deadline_ms: poll_deadline_ms, state });
5892 newly_submitted_event = true;
5893 }
5894
5895 // Java: `offsetCommitCallbackInvoker.executeCallbacks();` +
5896 // `processBackgroundEvents();`.
5897 //
5898 // Both are user-supplied code paths — if either throws, we clear
5899 // the inflight poll and propagate the error.
5900 let invocation_result = self.run_check_inflight_drain().await;
5901 if let Err(err) = invocation_result {
5902 log::trace!("Inflight event AsyncPoll failed due to {err}, clearing");
5903 self.inflight_poll = None;
5904 // Java: `throw ConsumerUtils.maybeWrapAsKafkaException(t)`
5905 // in `org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer#checkInflightPoll` — the conditional
5906 // wrap, so a generic runtime error from user-supplied callback code still
5907 // reaches the application as a `KafkaException`.
5908 return Err(crate::consumer::internals::ConsumerUtils::maybe_wrap_as_kafka_error(err));
5909 }
5910
5911 if self.inflight_poll.is_some() {
5912 self.maybe_clear_current_inflight_poll(newly_submitted_event)?;
5913 }
5914
5915 Ok(())
5916 }
5917
5918 /// Drain pending OffsetCommitCallback invocations + bg-events.
5919 ///
5920 /// Pulled out of [`Self::check_inflight_poll`] so the `?`-based error
5921 /// propagation can run inside a single try-block-equivalent body —
5922 /// Java's `try { ... } catch (Throwable t)` semantics translate as
5923 /// "run this helper, observe the result".
5924 async fn run_check_inflight_drain(&mut self) -> Result<(), Error> {
5925 // Invoke any callbacks queued by previous async commits. An error a
5926 // callback returns propagates, as the exception Java's
5927 // `executeCallbacks()` lets escape
5928 // (`org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer#checkInflightPoll`).
5929 self.offset_commit_callback_invoker.invoke_pending_callbacks().await?;
5930 // Drain pending background events (rebalance-listener callbacks,
5931 // fatal errors). §31: must run on the caller's task.
5932 self.process_background_events().await?;
5933 Ok(())
5934 }
5935
5936 /// Java: `private void maybeClearPreviousInflightPoll()`
5937 /// (`AsyncKafkaConsumer#maybeClearPreviousInflightPoll`).
5938 fn maybe_clear_previous_inflight_poll(&mut self) -> Result<(), Error> {
5939 let inflight = match self.inflight_poll.as_ref() {
5940 Some(i) => i,
5941 None => return Ok(()),
5942 };
5943 if inflight.state.is_complete() {
5944 let err_opt = inflight.state.error();
5945 if let Some(error) = err_opt {
5946 log::trace!("Previous inflight event AsyncPoll completed with an error ({error}), clearing");
5947 self.inflight_poll = None;
5948 return Err(error);
5949 }
5950 // Successful case: check if the bg task populated the buffer.
5951 if self.fetch_buffer.is_empty() {
5952 log::trace!("Previous inflight event AsyncPoll completed without filling the buffer, clearing");
5953 self.inflight_poll = None;
5954 } else {
5955 // Buffer is full — keep the event so the caller can drain
5956 // the buffer before a fresh event is enqueued (Java's
5957 // "0 timeout starvation" guard).
5958 log::trace!("Previous inflight event AsyncPoll completed and filled the buffer, not clearing");
5959 }
5960 return Ok(());
5961 }
5962
5963 // Java: `else if (inflightPoll.isExpired(time) && inflightPoll.isValidatePositionsComplete())`.
5964 let now_ms = self.time.milliseconds();
5965 if inflight.is_expired(now_ms) && inflight.state.is_validate_positions_complete() {
5966 log::trace!("Previous inflight event AsyncPoll expired without completing, clearing");
5967 self.inflight_poll = None;
5968 }
5969 Ok(())
5970 }
5971
5972 /// Java: `private void maybeClearCurrentInflightPoll(boolean newlySubmittedEvent)`
5973 /// (`AsyncKafkaConsumer#maybeClearCurrentInflightPoll`).
5974 fn maybe_clear_current_inflight_poll(&mut self, newly_submitted_event: bool) -> Result<(), Error> {
5975 let inflight = match self.inflight_poll.as_ref() {
5976 Some(i) => i,
5977 None => return Ok(()),
5978 };
5979 if inflight.state.is_complete() {
5980 let err_opt = inflight.state.error();
5981 self.inflight_poll = None;
5982 if let Some(error) = err_opt {
5983 log::trace!("Inflight event AsyncPoll completed with an error ({error}), clearing");
5984 return Err(error);
5985 }
5986 log::trace!("Inflight event AsyncPoll completed without error, clearing");
5987 return Ok(());
5988 }
5989
5990 if !newly_submitted_event {
5991 let now_ms = self.time.milliseconds();
5992 if inflight.is_expired(now_ms) && inflight.state.is_validate_positions_complete() {
5993 log::trace!("Inflight event AsyncPoll expired without completing, clearing");
5994 self.inflight_poll = None;
5995 }
5996 }
5997 Ok(())
5998 }
5999
6000 /// Java: `private Fetch<K, V> pollForFetches(Timer timer)`
6001 /// (`AsyncKafkaConsumer#pollForFetches`).
6002 ///
6003 /// Collects buffered records; if none are available yet, **blocks** on
6004 /// the [`FetchBuffer`] wakeup until the background task adds data, the
6005 /// timeout elapses, or `wakeup()` fires — then collects again. This is
6006 /// the faithful translation of Java's `pollForFetches`, which calls
6007 /// `fetchBuffer.awaitWakeup(pollTimer)` between the two `collectFetch()`
6008 /// calls. The earlier Rust port dropped that block and the caller
6009 /// re-checked in a tight loop, which (a) busy-spun a core whenever the
6010 /// buffer was momentarily empty and (b) left no fetch reliably in flight
6011 /// while waiting — the steady-state latency tail
6012 /// (`design/current/consumer-latency-findings.md`). Blocking here parks
6013 /// an idle consumer and lets a long-polling fetch's response wake the
6014 /// wait the instant it lands (`FetchBuffer::add` → `await_wakeup`).
6015 ///
6016 /// Errors (e.g. `OffsetOutOfRange`, `TopicAuthorizationFailed`)
6017 /// propagate to the caller — Java raises them out of `poll(Duration)`.
6018 async fn poll_for_fetches(&self, poll_deadline_ms: i64) -> Result<ConsumerRecords<K, V>, Error> {
6019 // Java's first `collectFetch()` — return immediately if data is ready.
6020 // Java (`pollForFetches`) uses `Fetch.isEmpty()`
6021 // (`numRecords == 0 && !positionAdvanced`), so a position-advanced /
6022 // zero-record fetch returns here rather than blocking on the buffer.
6023 //
6024 // The reconciliation-check wait, the `can_skip_update_fetch_positions`
6025 // fast path and the validate-positions gate all live in
6026 // [`Self::collect_fetch`], exactly as Java puts them in
6027 // `collectFetch()`.
6028 let fetch = self.collect_fetch().await?;
6029 if !fetch.is_fetch_empty() {
6030 return Ok(fetch);
6031 }
6032
6033 // Java: `pollTimeout` is computed AFTER the first `collectFetch()`
6034 // returns empty — no need to compute it when data is already
6035 // available. See [`Self::poll_for_fetches_timeout_ms`].
6036 let remaining = self.remaining_ms(poll_deadline_ms);
6037 let poll_timeout_ms = self.poll_for_fetches_timeout_ms(remaining);
6038
6039 // Java: `wakeupTrigger.setFetchAction(fetchBuffer)`, which throws
6040 // `WakeupException` for a wakeup already pending — also when
6041 // `pollTimeout` is 0, since Java has no early return here. So a
6042 // `wakeup()` sent while this poll ran its commit callbacks or
6043 // rebalance listeners (`checkInflightPoll`) is raised by this poll,
6044 // not left armed for the next call when the deadline has passed.
6045 // As at the loop top, the token is rotated after raising (§11).
6046 if let Err(err) = self.wakeup_trigger.maybe_trigger_wakeup() {
6047 self.wakeup_trigger.rotate();
6048 return Err(err);
6049 }
6050
6051 // Java's `fetchBuffer.awaitWakeup(pollTimer)` returns at once for a
6052 // timer of 0, so with no time left there is nothing to wait for and
6053 // no fetch to send ahead of a wait: go straight to the second
6054 // `collectFetch()`, as Java does.
6055 if poll_timeout_ms > 0 {
6056 // Ensure a fetch is in flight before we block. `await_wakeup` only
6057 // wakes when the bg task adds fetched data, so blocking with no fetch
6058 // outstanding strands the wait until `maximum_time_to_wait` even
6059 // though records may be available at the broker. This happens when
6060 // the consumer has just caught up and the prefetch chain
6061 // (`poll`-returns-records → `send_prefetches`) did not fire — the
6062 // residual latency tail in `design/current/consumer-latency-findings.md`.
6063 // `createFetchRequests` is a no-op for any node that already has a
6064 // fetch in flight (the in-flight skip in `prepare_fetch_requests`),
6065 // so this issues a fetch only when none is outstanding. The issued
6066 // fetch long-polls at the broker (`fetch.max.wait.ms`) and its
6067 // response wakes us the instant data lands.
6068 self.send_prefetches();
6069
6070 // Java: `fetchBuffer.awaitWakeup(pollTimer)`, which a `wakeup()`
6071 // with the fetch action set interrupts (`WakeupTrigger.wakeup`
6072 // calls `fetchBuffer.wakeup()` and leaves a `WakeupFuture`
6073 // pending). The Rust wakeup model (§11) realizes that by racing
6074 // the cancellation token instead of a back-channel: when
6075 // `wakeup()` cancels the token this arm wins, and the wakeup stays
6076 // pending for the poll() loop top, which surfaces `Error::Wakeup`
6077 // and rotates the token.
6078 let token = self.wakeup_trigger.current_token();
6079 tokio::select! {
6080 biased;
6081 _ = token.cancelled() => {}
6082 _ = self.fetch_buffer.await_wakeup(Duration::from_millis(poll_timeout_ms as u64)) => {}
6083 }
6084 }
6085
6086 // Java's second `collectFetch()` — may still be empty on a timeout or
6087 // a wakeup; the caller's loop re-checks the deadline / surfaces the
6088 // wakeup. All three of its guards run again, as in Java.
6089 self.collect_fetch().await
6090 }
6091
6092 /// How long [`Self::poll_for_fetches`] may block on the fetch buffer,
6093 /// given `remaining_ms` of the caller's poll timer. Java computes this
6094 /// inline in `pollForFetches` (`AsyncKafkaConsumer#pollForFetches`, 4.4);
6095 /// it is a separate method here so the value the application task waits
6096 /// for can be asserted directly (the KAFKA-20854 / KAFKA-20970 /
6097 /// KAFKA-21010 busy-loop tests).
6098 ///
6099 /// `pollTimeout = min(maximumTimeToWait, timer.remainingMs())`. Capping at
6100 /// `maximumTimeToWait` (the background task's cached minimum over the
6101 /// request managers) bounds how long this blocks, so the poll loop re-runs
6102 /// `check_inflight_poll` — draining §31 background events / rebalance
6103 /// callbacks — at least that often. KAFKA-20854 dropped Java's
6104 /// `isCommittedOffsetsManagementEnabled()` condition on the `min`; this
6105 /// port always applied it.
6106 ///
6107 /// Then the wait is bounded by `retry.backoff.ms` when background progress
6108 /// may make fetching possible soon, judged from the current application-task
6109 /// state rather than possibly stale background-task state:
6110 /// - no assigned partitions: group membership not established yet,
6111 /// assignments revoked but not reassigned, bootstrap DNS resolution
6112 /// (KIP-909) still in progress, or manual assignment not done yet;
6113 /// - some partition without a valid position: the background task may be
6114 /// fetching committed offsets, looking up offsets by timestamp, or
6115 /// backing off after a failure;
6116 /// - (KAFKA-20854) some fetchable partition with no buffered data: it may
6117 /// have been skipped for reconnect backoff, an in-flight request or a
6118 /// missing leader, and the background task no longer wakes the buffer
6119 /// for those states, so bound the wait to retry once the condition
6120 /// clears.
6121 ///
6122 /// Java copies the assignment for the position check; the first two
6123 /// predicates here use the allocation-free `num_assigned_partitions` /
6124 /// `has_all_fetch_positions` (Phase 27 Fix #3). Each `SubscriptionState`
6125 /// read takes and drops the lock, as each Java `synchronized` call does,
6126 /// and the buffer's lock is never taken while it is held. No `.await` is
6127 /// held across either guard (§16).
6128 fn poll_for_fetches_timeout_ms(&self, remaining_ms: i64) -> i64 {
6129 let poll_timeout_ms = self.maximum_time_to_wait_ms().min(remaining_ms);
6130 if poll_timeout_ms <= self.retry_backoff_ms {
6131 return poll_timeout_ms;
6132 }
6133 let needs_backoff = {
6134 let subs = self.subscriptions.lock().unwrap();
6135 subs.num_assigned_partitions() == 0 || !subs.has_all_fetch_positions()
6136 };
6137 if needs_backoff {
6138 return self.retry_backoff_ms;
6139 }
6140 let buffered = self.fetch_buffer.buffered_partitions();
6141 let has_unbuffered_fetchable = self
6142 .subscriptions
6143 .lock()
6144 .unwrap()
6145 .has_fetchable_partitions(|tp| !buffered.contains(tp));
6146 if has_unbuffered_fetchable {
6147 self.retry_backoff_ms
6148 } else {
6149 poll_timeout_ms
6150 }
6151 }
6152
6153 /// AK 4.3.1 (KAFKA-20106): the first stage of `collectFetch()`.
6154 ///
6155 /// Do not return buffered records if the background hasn't checked for
6156 /// pending reconciliations for the in-flight poll event. This is key
6157 /// because partitions may need revocation, so we must wait for the
6158 /// reconciliation check that triggers commits and marks partitions as
6159 /// pending revocation before we can safely collect records from the
6160 /// buffer.
6161 ///
6162 /// Returns `true` if the caller may proceed to collect a fetch, `false`
6163 /// if it should return an empty fetch (the reconciliation check has not
6164 /// completed and there was no time to wait, or the wait timed out).
6165 ///
6166 /// # Errors
6167 ///
6168 /// `Error::Wakeup` when a `wakeup()` is pending or arrives during the
6169 /// wait, as Java's `getResult` throws the `WakeupException` the trigger
6170 /// completes the future with.
6171 ///
6172 /// Java:
6173 /// ```java
6174 /// if (hasPendingReconciliation && inflightPoll != null && !inflightPoll.isReconciliationCheckComplete()) {
6175 /// long timeoutMs = inflightPoll.deadlineMs() - time.milliseconds();
6176 /// if (timeoutMs > 0) {
6177 /// try {
6178 /// wakeupTrigger.setActiveTask(inflightPoll.reconciliationCheckFuture());
6179 /// ConsumerUtils.getResult(inflightPoll.reconciliationCheckFuture(), timeoutMs);
6180 /// } catch (TimeoutException e) { return Fetch.empty(); }
6181 /// finally { wakeupTrigger.clearTask(); }
6182 /// } else { return Fetch.empty(); }
6183 /// }
6184 /// ```
6185 ///
6186 /// The `wakeupTrigger.setActiveTask` is realized here by racing the
6187 /// rotating cancellation token (§11): a pending or concurrent `wakeup()`
6188 /// cancels the token, this wait returns, and `Error::Wakeup` is raised
6189 /// from here with the token rotated, as Java raises it from
6190 /// `collectFetch()`. The lost-wakeup race is closed by creating the
6191 /// `notified()` future BEFORE re-checking the completion flag.
6192 async fn wait_reconciliation_check(&self) -> Result<bool, Error> {
6193 if !self.has_pending_reconciliation.load(Ordering::Acquire) {
6194 return Ok(true);
6195 }
6196 let Some(inflight) = self.inflight_poll.as_ref() else {
6197 return Ok(true);
6198 };
6199 if inflight.state.is_reconciliation_check_complete() {
6200 return Ok(true);
6201 }
6202 let timeout_ms = inflight.deadline_ms.saturating_sub(self.time.milliseconds());
6203 if timeout_ms <= 0 {
6204 // No time to wait and reconciliation check not complete.
6205 return Ok(false);
6206 }
6207 // Create the notified() future BEFORE the completion re-check to avoid
6208 // a lost wakeup (create-future-then-check ordering).
6209 let notified = inflight.state.reconciliation_check_notify().notified();
6210 if inflight.state.is_reconciliation_check_complete() {
6211 return Ok(true);
6212 }
6213 let token = self.wakeup_trigger.current_token();
6214 tokio::select! {
6215 biased;
6216 _ = token.cancelled() => {
6217 // Java: `setActiveTask` completes the future with a
6218 // `WakeupException`, which `getResult` throws out of
6219 // `collectFetch()`. Raise it here and rotate the token, as
6220 // the poll loop top does (§11). A trigger disabled by close()
6221 // reports no wakeup: then this is an empty fetch.
6222 if let Err(err) = self.wakeup_trigger.maybe_trigger_wakeup() {
6223 self.wakeup_trigger.rotate();
6224 return Err(err);
6225 }
6226 Ok(false)
6227 }
6228 _ = notified => Ok(true),
6229 _ = tokio::time::sleep(Duration::from_millis(timeout_ms as u64)) => {
6230 // Java: TimeoutException -> return Fetch.empty().
6231 Ok(false)
6232 }
6233 }
6234 }
6235
6236 /// Java: `private Fetch<K, V> collectFetch()`
6237 /// (`AsyncKafkaConsumer#collectFetch`).
6238 ///
6239 /// Performs the "fetch collection" step by reading raw data out of the
6240 /// [`FetchBuffer`], converting it to a well-formed `CompletedFetch`,
6241 /// validating that it and the internal [`SubscriptionState`] are
6242 /// correct, and then converting it all into records for returning.
6243 ///
6244 /// The three guards ahead of the collection are all Java's, in Java's
6245 /// order:
6246 ///
6247 /// 1. Do not return buffered records before the background task has run
6248 /// the pending-reconciliation check ([`Self::wait_reconciliation_check`]),
6249 /// since partitions may need revocation first.
6250 /// 2. If the shared [`PositionsValidator`] says the position-validation
6251 /// step can be skipped, collect immediately — this is the fast path
6252 /// that keeps the app task off the inter-task handshake below.
6253 /// 3. Otherwise wait for the in-flight poll event to report that it has
6254 /// finished validating positions. Without this the app task and the
6255 /// background task can both update
6256 /// [`SubscriptionState::position`](SubscriptionState) for the same
6257 /// partition.
6258 ///
6259 /// # Errors
6260 ///
6261 /// Propagates a cached validation error from the shared
6262 /// [`PositionsValidator`], and any error surfaced by the collector
6263 /// itself (e.g. `OffsetOutOfRange`, `TopicAuthorizationFailed`).
6264 ///
6265 /// # Cancellation
6266 ///
6267 /// Dropped while a deserializer is pending, the collector moves back every
6268 /// position it had moved, also after a leader change, and discards what
6269 /// it had decoded, so the records are fetched again (see
6270 /// `FetchCollector::collect_fetch`).
6271 async fn collect_fetch(&self) -> Result<ConsumerRecords<K, V>, Error> {
6272 // Java's `collectFetch`: the reconciliation check.
6273 if !self.wait_reconciliation_check().await? {
6274 return Ok(ConsumerRecords::empty());
6275 }
6276
6277 // Java's `collectFetch`: `canSkipUpdateFetchPositions`.
6278 if self.positions_validator.can_skip_update_fetch_positions()? {
6279 return self.fetch_collector.collect_fetch(&self.fetch_buffer).await;
6280 }
6281
6282 // Java's `collectFetch`. With the non-blocking async poll, it's critical that
6283 // the application task wait until the background task has completed
6284 // the stage of validating positions. If the in-flight event was
6285 // cleared by `maybe_clear_*_inflight_poll`, that implies it is safe
6286 // to collect from the fetch buffer.
6287 if let Some(inflight) = self.inflight_poll.as_ref()
6288 && !inflight.state.is_validate_positions_complete()
6289 {
6290 return Ok(ConsumerRecords::empty());
6291 }
6292
6293 self.fetch_collector.collect_fetch(&self.fetch_buffer).await
6294 }
6295
6296 // ── Commit ─────────────────────────────────────────────────────────
6297 //
6298 // Translates Java's `commitSync()` / `commitAsync()` family
6299 // (`org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer`). The shared
6300 // helper `commit(commit_event)` validates group_id, drains pending
6301 // callbacks, returns the early-completed receiver for empty offsets,
6302 // adds the event, awaits `offsets_ready`, and returns the
6303 // `handle.future()` receiver. The sync / async wrappers branch on
6304 // the resulting receiver:
6305 //
6306 // - `commit_async` registers a callback (or default no-callback
6307 // interceptor invocation) via the
6308 // `OffsetCommitCallbackInvoker`, then returns immediately.
6309 // - `commit_sync` blocks on the receiver and runs the interceptor
6310 // `on_commit` chain inline on the caller's task.
6311 //
6312 // The receiver from the underlying `CommitAsync` / `CommitSync` event
6313 // resolves with the committed-offsets map (or an error). Java uses a
6314 // single `CompletableFuture<Map>` for both arms; Rust uses two: a
6315 // `PendingAsyncCommit` for app-side completion ordering (Java's
6316 // `lastPendingAsyncCommit`, in the `AsyncCommitState` shared with every
6317 // `ConsumerHandle`) and the typed receiver returned to the caller.
6318
6319 /// Translates Java's
6320 /// `private CompletableFuture<Map<TopicPartition, OffsetAndMetadata>> commit(CommitEvent)`
6321 /// (`AsyncKafkaConsumer#commit`), shared with every
6322 /// [`ConsumerHandle`] through [`AsyncCommitState::submit_commit`]; this
6323 /// side adds the offsets-ready wait.
6324 ///
6325 /// Returns the typed receiver from the commit event's handle, or `None`
6326 /// for empty explicit offsets (Java's `completedFuture(null)`). Callers
6327 /// either await it (sync path) or attach a spawned-task continuation
6328 /// (async path).
6329 async fn commit_inner(&mut self, commit_event: CommitEventKind) -> Result<Option<CommitResultReceiver>, Error> {
6330 let Some((offsets_ready_rx, receiver)) = self
6331 .commit_state
6332 .submit_commit(
6333 self.group_id.as_deref(),
6334 commit_event,
6335 &self.application_event_handler,
6336 self.time.as_ref(),
6337 )
6338 .await?
6339 else {
6340 return Ok(None);
6341 };
6342
6343 // Java: `ConsumerUtils.getResult(commitEvent.offsetsReady(), defaultApiTimeoutMs.toMillis())`.
6344 // This blocks until the bg task has resolved which offsets to
6345 // commit (so subsequent fetches don't shift the commit window).
6346 // Like Java's, the wait does not process background events (§31):
6347 // a rebalance callback or error queued meanwhile waits for the next
6348 // `poll()`.
6349 //
6350 // The wait is not wakeable, for `commit_sync` as for `commit_async`:
6351 // Java waits on `offsetsReady` without `setActiveTask`
6352 // (`org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer#commit`), so a woken `commitSync` throws
6353 // only after the background task has snapshotted the offsets
6354 // (`ApplicationEventProcessor#process(SyncCommitEvent)`), from its later
6355 // wakeup-registered waits (`AsyncKafkaConsumer#commitSync(Optional,Duration)`). Waking it here could return
6356 // `WakeupError` with the `CommitSync` event not yet processed, and a
6357 // `poll` after it could advance the positions the woken commit then
6358 // snapshots. `commitAsync` never throws `WakeupException` either
6359 // (`AsyncKafkaConsumer#commitAsync(Optional,OffsetCommitCallback)`, Issue 22).
6360 let or_deadline_ms = self.default_api_timeout_deadline_ms();
6361 self.await_completion::<()>(
6362 offsets_ready_rx,
6363 or_deadline_ms,
6364 OFFSETS_READY_TIMEOUT_MSG,
6365 /* enable_wakeup = */ false,
6366 )
6367 .await?;
6368
6369 Ok(Some(receiver))
6370 }
6371
6372 /// Translates Java's `void commitSync()` (uses default API timeout).
6373 pub async fn commit_sync(&mut self) -> Result<(), Error> {
6374 self.commit_sync_internal(None, Duration::from_millis(self.default_api_timeout_ms as u64))
6375 .await
6376 }
6377
6378 /// Translates Java's `void commitSync(Duration timeout)`.
6379 pub async fn commit_sync_with_timeout(&mut self, timeout: Duration) -> Result<(), Error> {
6380 self.commit_sync_internal(None, timeout).await
6381 }
6382
6383 /// Translates Java's
6384 /// `void commitSync(Map<TopicPartition, OffsetAndMetadata> offsets)`.
6385 pub async fn commit_sync_with_offsets(
6386 &mut self,
6387 offsets: HashMap<TopicPartition, OffsetAndMetadata>,
6388 ) -> Result<(), Error> {
6389 self.commit_sync_internal(Some(offsets), Duration::from_millis(self.default_api_timeout_ms as u64))
6390 .await
6391 }
6392
6393 /// Translates Java's
6394 /// `void commitSync(Map<TopicPartition, OffsetAndMetadata> offsets, Duration timeout)`.
6395 pub async fn commit_sync_with_offsets_timeout(
6396 &mut self,
6397 offsets: HashMap<TopicPartition, OffsetAndMetadata>,
6398 timeout: Duration,
6399 ) -> Result<(), Error> {
6400 self.commit_sync_internal(Some(offsets), timeout).await
6401 }
6402
6403 /// Translates Java's
6404 /// `private void commitSync(Optional<Map<...>>, Duration timeout)`.
6405 ///
6406 /// # Java divergence — single deadline vs Java's fresh `requestTimer`
6407 ///
6408 /// Java's `AsyncKafkaConsumer#commitSync(Optional,Duration)` computes
6409 /// TWO independent timers:
6410 /// 1. `calculateDeadlineMs(time, timeout)` is baked into the
6411 /// `SyncCommitEvent` for the bg-side commit RPC.
6412 /// 2. A FRESH `time.timer(timeout.toMillis())` is created AFTER
6413 /// `commit(...)` returns and drives both
6414 /// `awaitPendingAsyncCommits` and `ConsumerUtils.getResult`.
6415 ///
6416 /// So Java's worst-case wall-clock bound is `~2 * timeout`. The Rust
6417 /// translation uses a SINGLE `deadline_ms` computed once at the top
6418 /// and shared across all phases — a stricter `~1 * timeout` total
6419 /// bound. This is intentional: most user code expects "commitSync
6420 /// with timeout=T should not exceed ~T wall-clock", and Java's
6421 /// doubling is an artifact of the timer construction rather than a
6422 /// documented contract. Issue 18 — divergence is documented here
6423 /// and verified against Java's fresh-timer statement `time.timer(timeout.toMillis())`.
6424 async fn commit_sync_internal(
6425 &mut self,
6426 offsets: Option<HashMap<TopicPartition, OffsetAndMetadata>>,
6427 timeout: Duration,
6428 ) -> Result<(), Error> {
6429 self.ensure_open()?;
6430 // Java: `long commitStart = time.nanoseconds()` at the top of
6431 // `org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer#commitSync(Optional,Duration)`, recorded in
6432 // `finally` as `recordCommitSync(time.nanoseconds() - commitStart)`.
6433 let commit_start_ns = self.time.nanoseconds();
6434 let result = self.commit_sync_inner(offsets, timeout).await;
6435 self.kafka_consumer_metrics
6436 .record_commit_sync(self.time.nanoseconds() - commit_start_ns);
6437 result
6438 }
6439
6440 /// The `try`-body of [`Self::commit_sync_internal`]
6441 /// (`AsyncKafkaConsumer#commitSync(Optional,Duration)`). Separated so the caller can
6442 /// record `recordCommitSync` in a `finally`-equivalent.
6443 async fn commit_sync_inner(
6444 &mut self,
6445 offsets: Option<HashMap<TopicPartition, OffsetAndMetadata>>,
6446 timeout: Duration,
6447 ) -> Result<(), Error> {
6448 let now_ms = self.time.milliseconds();
6449 let deadline_ms = CompletableEvent::calculate_deadline_ms(now_ms, ConsumerUtils::duration_to_millis(timeout));
6450
6451 let receiver = self
6452 .commit_inner(CommitEventKind::Sync { offsets, deadline_ms })
6453 .await?
6454 .unwrap_or_else(completed_commit_result);
6455
6456 // Java: `awaitPendingAsyncCommitsAndExecuteCommitCallbacks(requestTimer, true)`
6457 // — drain any pending async commits BEFORE blocking on this sync
6458 // commit so the user-visible callback ordering matches Java.
6459 self.await_pending_async_commits_and_execute_commit_callbacks(deadline_ms, true)
6460 .await?;
6461
6462 // Java: `ConsumerUtils.getResult(commitFuture, requestTimer)`
6463 // with `wakeupTrigger.setActiveTask(commitFuture)` for the
6464 // duration of the await
6465 // (`AsyncKafkaConsumer#commitSync(Optional,Duration)`). Like Java's,
6466 // the wait does not process background events (§31). Issue 11 / §11: `enable_wakeup=true` makes a
6467 // concurrent `wakeup()` interrupt the wait.
6468 // Issue 19: Java's `commit(Optional<...>, Duration)` does not
6469 // attach a custom timeout message — `ConsumerUtils.getResult`
6470 // rethrows the underlying `TimeoutException` as-is
6471 // (`org.apache.kafka.clients.consumer.internals.ConsumerUtils#getResult(Future,long)`). Mirror that: pass a
6472 // brief Rust-side string that does NOT format the offsets map via
6473 // Debug.
6474 let wait_result = self
6475 .await_completion::<HashMap<TopicPartition, OffsetAndMetadata>>(
6476 receiver,
6477 deadline_ms,
6478 format!(
6479 "Timeout of {}ms expired before successfully committing offsets",
6480 ConsumerUtils::duration_to_millis(timeout)
6481 ),
6482 true,
6483 )
6484 .await;
6485 let committed: HashMap<TopicPartition, OffsetAndMetadata> = match wait_result {
6486 Ok(map) => map,
6487 Err(err) => return Err(err),
6488 };
6489
6490 // Java: `interceptors.onCommit(committedOffsets)`.
6491 self.commit_state.on_commit(&committed);
6492 Ok(())
6493 }
6494
6495 /// Translates Java's `void commitAsync()` (no callback, no offsets —
6496 /// commit `allConsumed`).
6497 pub async fn commit_async(&mut self) -> Result<(), Error> {
6498 self.commit_async_internal(None, None).await
6499 }
6500
6501 /// Translates Java's `void commitAsync(OffsetCommitCallback)`.
6502 pub async fn commit_async_with_callback(
6503 &mut self,
6504 callback: Arc<dyn crate::consumer::OffsetCommitCallback>,
6505 ) -> Result<(), Error> {
6506 self.commit_async_internal(None, Some(callback)).await
6507 }
6508
6509 /// Translates Java's
6510 /// `void commitAsync(Map<TopicPartition, OffsetAndMetadata>, OffsetCommitCallback)`.
6511 pub async fn commit_async_with_offsets_callback(
6512 &mut self,
6513 offsets: HashMap<TopicPartition, OffsetAndMetadata>,
6514 callback: Arc<dyn crate::consumer::OffsetCommitCallback>,
6515 ) -> Result<(), Error> {
6516 self.commit_async_internal(Some(offsets), Some(callback)).await
6517 }
6518
6519 /// Translates Java's
6520 /// `private void commitAsync(Optional<Map<...>>, OffsetCommitCallback)`.
6521 async fn commit_async_internal(
6522 &mut self,
6523 offsets: Option<HashMap<TopicPartition, OffsetAndMetadata>>,
6524 callback: Option<Arc<dyn crate::consumer::OffsetCommitCallback>>,
6525 ) -> Result<(), Error> {
6526 self.ensure_open()?;
6527 // Issue 22 /
6528 // `org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer#commitAsync(Optional,OffsetCommitCallback)`:
6529 // Java's `commitAsync` is documented as non-blocking and never throws `WakeupException`; `commit_inner`'s
6530 // offsets-ready wait is not wakeable, so the commit completes normally on the bg task.
6531 let receiver = self.commit_inner(CommitEventKind::Async { offsets }).await?;
6532 self.commit_state.register_async_commit(receiver, callback);
6533 Ok(())
6534 }
6535
6536 /// Translates Java's
6537 /// `private void awaitPendingAsyncCommitsAndExecuteCommitCallbacks(Timer timer, boolean enableWakeup)`
6538 /// (`AsyncKafkaConsumer#awaitPendingAsyncCommitsAndExecuteCommitCallbacks`); see
6539 /// [`AsyncCommitState::await_pending_async_commits_and_execute_commit_callbacks`],
6540 /// which every [`ConsumerHandle`] shares.
6541 async fn await_pending_async_commits_and_execute_commit_callbacks(
6542 &mut self,
6543 deadline_ms: i64,
6544 enable_wakeup: bool,
6545 ) -> Result<(), Error> {
6546 self.commit_state
6547 .await_pending_async_commits_and_execute_commit_callbacks(
6548 &self.wakeup_trigger,
6549 self.time.as_ref(),
6550 deadline_ms,
6551 enable_wakeup,
6552 )
6553 .await
6554 }
6555
6556 // ── Seek / position / committed / lag ──────────────────────────────
6557 //
6558 // Translates Java's `seek(...)` / `seekToBeginning(...)` /
6559 // `seekToEnd(...)` / `position(...)` / `committed(...)` /
6560 // `currentLag(...)` in `org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer`.
6561 //
6562 // The seek methods route through a `SeekUnvalidated` /
6563 // `ResetOffset` event. `position` and `committed` route through
6564 // `CheckAndUpdatePositions` / `FetchCommittedOffsets` events. The
6565 // current-lag arm enqueues a `CurrentLag` event.
6566 //
6567 // Each method calls `ensure_open()` first to mirror Java's
6568 // `acquireAndEnsureOpen()` closed-consumer guard.
6569
6570 /// Java: `void seek(TopicPartition, long offset)`.
6571 pub async fn seek_with_offset(&mut self, partition: TopicPartition, offset: i64) -> Result<(), Error> {
6572 if offset < 0 {
6573 return Err(Error::local_illegal_argument("seek offset must not be a negative number"));
6574 }
6575 self.ensure_open()?;
6576 log::info!("Seeking to offset {offset} for partition {partition}");
6577 let deadline_ms = self.default_api_timeout_deadline_ms();
6578 let (handle, receiver, _erased) = CompletableEvent::make_completable_event::<()>(deadline_ms);
6579 // Java's `seek(...)` does NOT call `setActiveTask` — match.
6580 self.submit_and_await::<()>(
6581 ApplicationEvent::SeekUnvalidated { handle, partition, offset, offset_epoch: None },
6582 receiver,
6583 deadline_ms,
6584 "Timeout expired while waiting for the seek event to complete",
6585 false,
6586 )
6587 .await
6588 }
6589
6590 /// Java: `void seek(TopicPartition, OffsetAndMetadata)`.
6591 pub async fn seek_with_offset_and_metadata(
6592 &mut self,
6593 partition: TopicPartition,
6594 offset_and_metadata: OffsetAndMetadata,
6595 ) -> Result<(), Error> {
6596 let offset = offset_and_metadata.offset();
6597 if offset < 0 {
6598 return Err(Error::local_illegal_argument("seek offset must not be a negative number"));
6599 }
6600 self.ensure_open()?;
6601 match offset_and_metadata.leader_epoch() {
6602 Some(epoch) => log::info!("Seeking to offset {offset} for partition {partition} with epoch {epoch}"),
6603 None => log::info!("Seeking to offset {offset} for partition {partition}"),
6604 }
6605 let deadline_ms = self.default_api_timeout_deadline_ms();
6606 let (handle, receiver, _erased) = CompletableEvent::make_completable_event::<()>(deadline_ms);
6607 self.submit_and_await::<()>(
6608 ApplicationEvent::SeekUnvalidated {
6609 handle,
6610 partition,
6611 offset,
6612 offset_epoch: offset_and_metadata.leader_epoch(),
6613 },
6614 receiver,
6615 deadline_ms,
6616 "Timeout expired while waiting for the seek event to complete",
6617 false,
6618 )
6619 .await
6620 }
6621
6622 /// Java: `void seekToBeginning(Collection<TopicPartition>)`.
6623 pub async fn seek_to_beginning(&mut self, partitions: &[TopicPartition]) -> Result<(), Error> {
6624 self.seek_with_reset_strategy(partitions, crate::consumer::internals::AutoOffsetResetStrategy::EARLIEST)
6625 .await
6626 }
6627
6628 /// Java: `void seekToEnd(Collection<TopicPartition>)`.
6629 pub async fn seek_to_end(&mut self, partitions: &[TopicPartition]) -> Result<(), Error> {
6630 self.seek_with_reset_strategy(partitions, crate::consumer::internals::AutoOffsetResetStrategy::LATEST)
6631 .await
6632 }
6633
6634 /// Translates Java's
6635 /// `private void seek(Collection<TopicPartition>, AutoOffsetResetStrategy)`.
6636 async fn seek_with_reset_strategy(
6637 &mut self,
6638 partitions: &[TopicPartition],
6639 strategy: crate::consumer::internals::AutoOffsetResetStrategy,
6640 ) -> Result<(), Error> {
6641 self.ensure_open()?;
6642 let set: std::collections::HashSet<TopicPartition> = partitions.iter().cloned().collect();
6643 let deadline_ms = self.default_api_timeout_deadline_ms();
6644 let (handle, receiver, _erased) = CompletableEvent::make_completable_event::<()>(deadline_ms);
6645 self.submit_and_await::<()>(
6646 ApplicationEvent::ResetOffset { handle, partitions: set, offset_reset_strategy: strategy },
6647 receiver,
6648 deadline_ms,
6649 "Timeout expired while waiting for the seek-with-reset-strategy event to complete",
6650 false,
6651 )
6652 .await
6653 }
6654
6655 /// Java: `long position(TopicPartition)` — uses default API timeout.
6656 pub async fn position(&mut self, partition: &TopicPartition) -> Result<i64, Error> {
6657 self.position_with_timeout(partition, Duration::from_millis(self.default_api_timeout_ms as u64))
6658 .await
6659 }
6660
6661 /// Java: `long position(TopicPartition, Duration timeout)`
6662 /// (`AsyncKafkaConsumer#position(TopicPartition,Duration)`).
6663 pub async fn position_with_timeout(&mut self, partition: &TopicPartition, timeout: Duration) -> Result<i64, Error> {
6664 self.ensure_open()?;
6665 {
6666 let subs = self.subscriptions.lock().unwrap();
6667 if !subs.is_assigned(partition) {
6668 return Err(Error::local_illegal_state(
6669 "You can only check the position for partitions assigned to this consumer.",
6670 ));
6671 }
6672 }
6673 let now_ms = self.time.milliseconds();
6674 let deadline_ms = CompletableEvent::calculate_deadline_ms(now_ms, ConsumerUtils::duration_to_millis(timeout));
6675
6676 loop {
6677 // Read the validated position under the lock; drop guard before await.
6678 let position_offset = {
6679 let subs = self.subscriptions.lock().unwrap();
6680 subs.valid_position(partition)?.map(|fp| fp.offset)
6681 };
6682 if let Some(offset) = position_offset {
6683 return Ok(offset);
6684 }
6685
6686 // Java: `updateFetchPositions(timer)` — drives the
6687 // `CheckAndUpdatePositionsEvent` round-trip with a plain
6688 // `addAndGet`, which does not process background events (§31).
6689 // Java `setActiveTask` analog: the
6690 // helper's `enable_wakeup=true` raises a wakeup pending before
6691 // the event is submitted and makes a concurrent `wakeup()`
6692 // interrupt the wait (Issue 11 / §11).
6693 let (handle, receiver, _erased) = CompletableEvent::make_completable_event::<()>(deadline_ms);
6694 let drain_result = self
6695 .submit_and_await::<()>(
6696 ApplicationEvent::CheckAndUpdatePositions { handle },
6697 receiver,
6698 deadline_ms,
6699 "Timeout expired while waiting for CheckAndUpdatePositions",
6700 true,
6701 )
6702 .await;
6703 // Java's `updateFetchPositions` catches `TimeoutException`
6704 // only and returns false; any other exception propagates
6705 // (`org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer#updateFetchPositions`). Issue 14:
6706 // replace the previous `.await.ok()` blanket swallow with explicit
6707 // error handling.
6708 match drain_result {
6709 Ok(()) => {},
6710 Err(Error::Timeout(_)) => {
6711 // Loop will re-check `remaining_ms` below and
6712 // surface the user-facing timeout error.
6713 },
6714 Err(err) => return Err(err),
6715 }
6716
6717 // Java: `wakeupTrigger.maybeTriggerWakeup()` after every
6718 // `updateFetchPositions` (`AsyncKafkaConsumer#position(TopicPartition,Duration)`): a
6719 // wakeup that came once the event's future was done, which the
6720 // wait leaves pending, fails the call here, before the
6721 // valid position is read again.
6722 if let Err(err) = self.wakeup_trigger.maybe_trigger_wakeup() {
6723 self.wakeup_trigger.rotate();
6724 return Err(err);
6725 }
6726 if self.time.milliseconds() >= deadline_ms {
6727 return Err(Error::timeout(format!(
6728 "Timeout of {}ms expired before the position for partition {} could be determined",
6729 ConsumerUtils::duration_to_millis(timeout),
6730 partition
6731 )));
6732 }
6733 }
6734 }
6735
6736 /// Java: `Map<TopicPartition, OffsetAndMetadata> committed(Set<TopicPartition>)`.
6737 pub async fn committed(
6738 &mut self,
6739 partitions: &[TopicPartition],
6740 ) -> Result<HashMap<TopicPartition, Option<OffsetAndMetadata>>, Error> {
6741 self.committed_with_timeout(partitions, Duration::from_millis(self.default_api_timeout_ms as u64))
6742 .await
6743 }
6744
6745 /// Java: `Map<TopicPartition, OffsetAndMetadata> committed(Set<TopicPartition>, Duration)`
6746 /// (`AsyncKafkaConsumer#committed(Set,Duration)`).
6747 pub async fn committed_with_timeout(
6748 &mut self,
6749 partitions: &[TopicPartition],
6750 timeout: Duration,
6751 ) -> Result<HashMap<TopicPartition, Option<OffsetAndMetadata>>, Error> {
6752 self.ensure_open()?;
6753 // Java: `long start = time.nanoseconds()` after `acquireAndEnsureOpen`
6754 // in `org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer#committed(Set,Duration)`, recorded in
6755 // `finally` as `recordCommitted(time.nanoseconds() - start)` — runs on
6756 // every exit path (empty partitions, group-id errors, timeout).
6757 let start_ns = self.time.nanoseconds();
6758 let result = self.committed_inner(partitions, timeout).await;
6759 self.kafka_consumer_metrics.record_committed(self.time.nanoseconds() - start_ns);
6760 result
6761 }
6762
6763 /// The `try`-body of [`Self::committed_with_timeout`]
6764 /// (`AsyncKafkaConsumer#committed(Set,Duration)`). Separated so the caller can
6765 /// record `recordCommitted` in a `finally`-equivalent.
6766 async fn committed_inner(
6767 &mut self,
6768 partitions: &[TopicPartition],
6769 timeout: Duration,
6770 ) -> Result<HashMap<TopicPartition, Option<OffsetAndMetadata>>, Error> {
6771 self.return_error_if_group_id_not_defined()?;
6772 if partitions.is_empty() {
6773 return Ok(HashMap::new());
6774 }
6775 let now_ms = self.time.milliseconds();
6776 let deadline_ms = CompletableEvent::calculate_deadline_ms(now_ms, ConsumerUtils::duration_to_millis(timeout));
6777 let set: std::collections::HashSet<TopicPartition> = partitions.iter().cloned().collect();
6778 let (handle, receiver, _erased) =
6779 CompletableEvent::make_completable_event::<HashMap<TopicPartition, Option<OffsetAndMetadata>>>(deadline_ms);
6780 // Java's `committed(...)` calls `setActiveTask(event.future())`
6781 // so a concurrent `wakeup()`
6782 // interrupts the wait — `enable_wakeup=true`.
6783 let result = self
6784 .submit_and_await::<HashMap<TopicPartition, Option<OffsetAndMetadata>>>(
6785 ApplicationEvent::FetchCommittedOffsets { handle, partitions: set },
6786 receiver,
6787 deadline_ms,
6788 "Timeout expired while waiting for FetchCommittedOffsets",
6789 true,
6790 )
6791 .await;
6792 match result {
6793 Ok(map) => Ok(map),
6794 Err(Error::Timeout(_)) => {
6795 // Issue 19: Java formats the partitions set via
6796 // `Set.toString()` (`[t-0, t-1]`) —
6797 // `org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer#committed(Set,Duration)`. The Rust
6798 // analog uses `TopicPartition`'s Display (`Display: "{topic}-{partition}"`) and emits the
6799 // same `[a-0, b-1]` shape rather than the noisy
6800 // `{:?}` Debug form.
6801 Err(Error::timeout(format!(
6802 "Timeout of {}ms expired before the last committed offset for partitions {} could be determined. Try tuning default.api.timeout.ms larger to relax the threshold.",
6803 ConsumerUtils::duration_to_millis(timeout),
6804 format_partitions_for_display(partitions),
6805 )))
6806 },
6807 Err(err) => Err(err),
6808 }
6809 }
6810
6811 /// Java: `OptionalLong currentLag(TopicPartition)`
6812 /// (`AsyncKafkaConsumer#currentLag`).
6813 ///
6814 /// Submits a `CurrentLag` event and waits for the background task's
6815 /// answer, as Java's `applicationEventHandler.addAndGet(new
6816 /// CurrentLagEvent(..))`. The background task reads the lag from the
6817 /// `SubscriptionState`, or issues a one-shot `ListOffsets(LATEST)` request
6818 /// when the log-end offset is unknown so the lag is known on a later call
6819 /// (`OffsetsRequestManager::current_lag`, KAFKA-20187).
6820 pub async fn current_lag(&mut self, topic_partition: &TopicPartition) -> Result<Option<i64>, Error> {
6821 self.ensure_open()?;
6822 let deadline_ms = self.default_api_timeout_deadline_ms();
6823 let (handle, receiver, _erased) = CompletableEvent::make_completable_event::<Option<i64>>(deadline_ms);
6824 // Java's `currentLag` does NOT call `setActiveTask` —
6825 // `enable_wakeup=false`.
6826 self.submit_and_await::<Option<i64>>(
6827 ApplicationEvent::CurrentLag {
6828 handle,
6829 partition: topic_partition.clone(),
6830 isolation_level: self.isolation_level,
6831 },
6832 receiver,
6833 deadline_ms,
6834 "Timeout expired while waiting for CurrentLag",
6835 false,
6836 )
6837 .await
6838 }
6839
6840 // ── Beginning / end offsets / offsetsForTimes ─────────────────────
6841
6842 /// Java: `Map<TopicPartition, Long> beginningOffsets(Collection<TopicPartition>)`.
6843 pub async fn beginning_offsets(
6844 &mut self,
6845 partitions: &[TopicPartition],
6846 ) -> Result<HashMap<TopicPartition, i64>, Error> {
6847 self.beginning_offsets_with_timeout(partitions, Duration::from_millis(self.default_api_timeout_ms as u64))
6848 .await
6849 }
6850
6851 /// Java: `Map<TopicPartition, Long> beginningOffsets(Collection<TopicPartition>, Duration)`.
6852 pub async fn beginning_offsets_with_timeout(
6853 &mut self,
6854 partitions: &[TopicPartition],
6855 timeout: Duration,
6856 ) -> Result<HashMap<TopicPartition, i64>, Error> {
6857 // Java's `ListOffsetsRequest.EARLIEST_TIMESTAMP = -2L`.
6858 self.beginning_or_end_offsets(partitions, -2, timeout).await
6859 }
6860
6861 /// Java: `Map<TopicPartition, Long> endOffsets(Collection<TopicPartition>)`.
6862 pub async fn end_offsets(&mut self, partitions: &[TopicPartition]) -> Result<HashMap<TopicPartition, i64>, Error> {
6863 self.end_offsets_with_timeout(partitions, Duration::from_millis(self.default_api_timeout_ms as u64))
6864 .await
6865 }
6866
6867 /// Java: `Map<TopicPartition, Long> endOffsets(Collection<TopicPartition>, Duration)`.
6868 pub async fn end_offsets_with_timeout(
6869 &mut self,
6870 partitions: &[TopicPartition],
6871 timeout: Duration,
6872 ) -> Result<HashMap<TopicPartition, i64>, Error> {
6873 // Java's `ListOffsetsRequest.LATEST_TIMESTAMP = -1L`.
6874 self.beginning_or_end_offsets(partitions, -1, timeout).await
6875 }
6876
6877 /// Translates Java's
6878 /// `private Map<TopicPartition, Long> beginningOrEndOffset(Collection<TopicPartition>, long timestamp, Duration timeout)`
6879 /// (`AsyncKafkaConsumer#beginningOrEndOffset`).
6880 async fn beginning_or_end_offsets(
6881 &mut self,
6882 partitions: &[TopicPartition],
6883 timestamp: i64,
6884 timeout: Duration,
6885 ) -> Result<HashMap<TopicPartition, i64>, Error> {
6886 self.ensure_open()?;
6887 if partitions.is_empty() {
6888 return Ok(HashMap::new());
6889 }
6890 let mut timestamps_to_search: HashMap<TopicPartition, i64> = HashMap::new();
6891 for tp in partitions {
6892 timestamps_to_search.insert(tp.clone(), timestamp);
6893 }
6894 let now_ms = self.time.milliseconds();
6895 let deadline_ms = CompletableEvent::calculate_deadline_ms(now_ms, ConsumerUtils::duration_to_millis(timeout));
6896
6897 if timeout.is_zero() {
6898 // Java: `if (timeout.isZero()) { applicationEventHandler.add(listOffsetsEvent); return new HashMap<>(); }`.
6899 let (handle, _receiver, _erased) = CompletableEvent::make_completable_event::<
6900 HashMap<TopicPartition, Option<OffsetAndTimestampInternal>>,
6901 >(deadline_ms);
6902 self.application_event_handler.add(
6903 ApplicationEvent::ListOffsets { handle, timestamps_to_search, require_timestamps: false },
6904 now_ms,
6905 )?;
6906 return Ok(HashMap::new());
6907 }
6908
6909 let (handle, receiver, _erased) = CompletableEvent::make_completable_event::<
6910 HashMap<TopicPartition, Option<OffsetAndTimestampInternal>>,
6911 >(deadline_ms);
6912 // Java's `beginningOffsets` / `endOffsets` do NOT call
6913 // `setActiveTask` — `enable_wakeup=false`.
6914 let result = self
6915 .submit_and_await::<HashMap<TopicPartition, Option<OffsetAndTimestampInternal>>>(
6916 ApplicationEvent::ListOffsets { handle, timestamps_to_search, require_timestamps: false },
6917 receiver,
6918 deadline_ms,
6919 "Timeout expired while waiting for ListOffsets",
6920 false,
6921 )
6922 .await;
6923 match result {
6924 Ok(offsets_map) => {
6925 // Java's `org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer#beginningOrEndOffset`
6926 // returns a map keyed on every requested partition mapped
6927 // to `entry.getValue().offset()`. The Rust translation
6928 // now mirrors that all-or-error contract: every
6929 // requested partition that the bg task surfaced a
6930 // result for is in the output. The bg task uses
6931 // `OffsetAndTimestampInternal` (matching Java) so the
6932 // broker's `timestamp == -1` sentinel for
6933 // `EARLIEST` / `LATEST` no longer maps to `None`.
6934 //
6935 // A `None` entry here means the bg task explicitly
6936 // surfaced "no offset" for the partition — Java's
6937 // null-value semantic — which is impossible on the
6938 // success path with a valid broker response but can
6939 // still appear if the global result is somehow
6940 // partially populated. We preserve Java's "filter null
6941 // values silently" behaviour for parity with
6942 // `OffsetsForTimes`; see COMMENTS.DONE.1.md Issue 6
6943 // for the regression where the `None`→drop filter
6944 // silently elided every entry due to
6945 // OffsetAndTimestamp::with_leader_epoch rejecting
6946 // negative timestamps.
6947 let mut out = HashMap::with_capacity(offsets_map.len());
6948 for (tp, opt) in offsets_map {
6949 if let Some(oat) = opt {
6950 out.insert(tp, oat.offset());
6951 }
6952 }
6953 Ok(out)
6954 },
6955 Err(Error::Timeout(_)) => Err(Error::timeout(format!(
6956 "Failed to get offsets by times in {}ms",
6957 ConsumerUtils::duration_to_millis(timeout)
6958 ))),
6959 Err(err) => Err(err),
6960 }
6961 }
6962
6963 /// Java: `Map<TopicPartition, OffsetAndTimestamp> offsetsForTimes(Map<TopicPartition, Long>)`.
6964 pub async fn offsets_for_times(
6965 &mut self,
6966 timestamps_to_search: HashMap<TopicPartition, i64>,
6967 ) -> Result<HashMap<TopicPartition, Option<OffsetAndTimestamp>>, Error> {
6968 self.offsets_for_times_with_timeout(
6969 timestamps_to_search,
6970 Duration::from_millis(self.default_api_timeout_ms as u64),
6971 )
6972 .await
6973 }
6974
6975 /// Java: `Map<TopicPartition, OffsetAndTimestamp> offsetsForTimes(Map<TopicPartition, Long>, Duration)`
6976 /// (`AsyncKafkaConsumer#offsetsForTimes(Map,Duration)`).
6977 pub async fn offsets_for_times_with_timeout(
6978 &mut self,
6979 timestamps_to_search: HashMap<TopicPartition, i64>,
6980 timeout: Duration,
6981 ) -> Result<HashMap<TopicPartition, Option<OffsetAndTimestamp>>, Error> {
6982 self.ensure_open()?;
6983 // Java's per-entry argument validation: negative targets rejected.
6984 for (tp, ts) in ×tamps_to_search {
6985 if *ts < 0 {
6986 return Err(Error::local_illegal_argument(format!(
6987 "The target time for partition {tp} is {ts}. The target time cannot be negative."
6988 )));
6989 }
6990 }
6991 if timestamps_to_search.is_empty() {
6992 return Ok(HashMap::new());
6993 }
6994 let now_ms = self.time.milliseconds();
6995 let deadline_ms = CompletableEvent::calculate_deadline_ms(now_ms, ConsumerUtils::duration_to_millis(timeout));
6996
6997 if timeout.is_zero() {
6998 // Java: `if (timeout.toMillis() == 0L) { applicationEventHandler.add(...); return listOffsetsEvent.emptyResults(); }`.
6999 let (handle, _receiver, _erased) = CompletableEvent::make_completable_event::<
7000 HashMap<TopicPartition, Option<OffsetAndTimestampInternal>>,
7001 >(deadline_ms);
7002 // `emptyResults()`: every requested partition mapped to `null`.
7003 let empty_results = timestamps_to_search.keys().map(|tp| (tp.clone(), None)).collect();
7004 self.application_event_handler.add(
7005 ApplicationEvent::ListOffsets { handle, timestamps_to_search, require_timestamps: true },
7006 now_ms,
7007 )?;
7008 return Ok(empty_results);
7009 }
7010
7011 let (handle, receiver, _erased) = CompletableEvent::make_completable_event::<
7012 HashMap<TopicPartition, Option<OffsetAndTimestampInternal>>,
7013 >(deadline_ms);
7014 // Java's `offsetsForTimes` does NOT call `setActiveTask` —
7015 // `enable_wakeup=false`.
7016 let result = self
7017 .submit_and_await::<HashMap<TopicPartition, Option<OffsetAndTimestampInternal>>>(
7018 ApplicationEvent::ListOffsets { handle, timestamps_to_search, require_timestamps: true },
7019 receiver,
7020 deadline_ms,
7021 "Timeout expired while waiting for ListOffsets",
7022 false,
7023 )
7024 .await;
7025 match result {
7026 Ok(offsets_map) => {
7027 // Java's `org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer#offsetsForTimes(Map,Duration)`
7028 // keeps a partition with no offset as a `null` value and
7029 // converts each other `OffsetAndTimestampInternal` to the
7030 // public-class `OffsetAndTimestamp` via
7031 // `buildOffsetAndTimestamp()`. The
7032 // `require_timestamps=true` path guarantees the broker
7033 // returns a non-negative timestamp (matching the
7034 // user-supplied target time), so the build should not
7035 // fail in practice. If it does (e.g. broker bug), we
7036 // propagate the IllegalArgument so the user observes
7037 // the broker misbehaviour rather than silently
7038 // dropping the entry.
7039 let mut out = HashMap::with_capacity(offsets_map.len());
7040 for (tp, opt) in offsets_map {
7041 let value = match opt {
7042 Some(oat) => Some(oat.build_offset_and_timestamp()?),
7043 None => None,
7044 };
7045 out.insert(tp, value);
7046 }
7047 Ok(out)
7048 },
7049 Err(Error::Timeout(_)) => Err(Error::timeout(format!(
7050 "Failed to get offsets by times in {}ms",
7051 ConsumerUtils::duration_to_millis(timeout)
7052 ))),
7053 Err(err) => Err(err),
7054 }
7055 }
7056
7057 // ── Topic metadata: partitionsFor / listTopics ────────────────────
7058
7059 /// Java: `List<PartitionInfo> partitionsFor(String topic)`.
7060 pub async fn partitions_for(&mut self, topic: &str) -> Result<Vec<crate::common::PartitionInfo>, Error> {
7061 self.partitions_for_with_timeout(topic, Duration::from_millis(self.default_api_timeout_ms as u64))
7062 .await
7063 }
7064
7065 /// Java: `List<PartitionInfo> partitionsFor(String topic, Duration)`
7066 /// (`AsyncKafkaConsumer#partitionsFor(String,Duration)`).
7067 pub async fn partitions_for_with_timeout(
7068 &mut self,
7069 topic: &str,
7070 timeout: Duration,
7071 ) -> Result<Vec<crate::common::PartitionInfo>, Error> {
7072 self.ensure_open()?;
7073 // Java: `Cluster cluster = this.metadata.fetch();
7074 // List<PartitionInfo> parts = cluster.partitionsForTopic(topic);
7075 // if (!parts.isEmpty()) return parts;`
7076 {
7077 let cluster = self.metadata.metadata_arc().fetch();
7078 let parts = cluster.partitions_for_topic(topic);
7079 if !parts.is_empty() {
7080 return Ok(parts.to_vec());
7081 }
7082 }
7083
7084 if timeout.is_zero() {
7085 // Java: `throw new TimeoutException();`
7086 // in `org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer#partitionsFor(String,Duration)` — the
7087 // class, code (`REQUEST_TIMED_OUT`) and retriable ancestry all match; only the
7088 // message differs. RECORDED DEVIATION
7089 // (definition-of-done.md §7): Java's no-arg constructor leaves
7090 // `getMessage()` null, which `Error` cannot represent — the nearest
7091 // forms are an empty string (rendering as `"TimeoutError: "`) or the
7092 // code's default text, and neither says which call timed out. The
7093 // added text is strictly additive diagnostics; nothing in the
7094 // hierarchy or the wire code changes.
7095 return Err(Error::timeout(format!(
7096 "Timeout of {}ms expired before partitions for topic {topic} could be determined",
7097 ConsumerUtils::duration_to_millis(timeout)
7098 )));
7099 }
7100
7101 let now_ms = self.time.milliseconds();
7102 let deadline_ms = CompletableEvent::calculate_deadline_ms(now_ms, ConsumerUtils::duration_to_millis(timeout));
7103 let (handle, receiver, _erased) =
7104 CompletableEvent::make_completable_event::<HashMap<String, Vec<crate::common::PartitionInfo>>>(deadline_ms);
7105 // Java's `partitionsFor` calls `setActiveTask(future)`
7106 // (`org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer#partitionsFor(String,Duration)`) —
7107 // `enable_wakeup=true`.
7108 let map = self
7109 .submit_and_await::<HashMap<String, Vec<crate::common::PartitionInfo>>>(
7110 ApplicationEvent::TopicMetadata { handle, topic: topic.to_string() },
7111 receiver,
7112 deadline_ms,
7113 "Timeout expired while waiting for TopicMetadata",
7114 true,
7115 )
7116 .await?;
7117 Ok(map.get(topic).cloned().unwrap_or_default())
7118 }
7119
7120 /// Java: `Map<String, List<PartitionInfo>> listTopics()`.
7121 pub async fn list_topics(&mut self) -> Result<HashMap<String, Vec<crate::common::PartitionInfo>>, Error> {
7122 self.list_topics_with_timeout(Duration::from_millis(self.default_api_timeout_ms as u64))
7123 .await
7124 }
7125
7126 /// Java: `Map<String, List<PartitionInfo>> listTopics(Duration)`
7127 /// (`AsyncKafkaConsumer#listTopics(Duration)`).
7128 pub async fn list_topics_with_timeout(
7129 &mut self,
7130 timeout: Duration,
7131 ) -> Result<HashMap<String, Vec<crate::common::PartitionInfo>>, Error> {
7132 self.ensure_open()?;
7133 if timeout.is_zero() {
7134 // Java: `throw new TimeoutException();`
7135 // in `org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer#listTopics(Duration)`. Same recorded
7136 // deviation as `partitions_for_with_timeout` above — Java's message is null, which
7137 // `Error` cannot represent; class, code and ancestry match.
7138 return Err(Error::timeout(format!(
7139 "Timeout of {}ms expired before all topics' metadata could be listed",
7140 ConsumerUtils::duration_to_millis(timeout)
7141 )));
7142 }
7143 let now_ms = self.time.milliseconds();
7144 let deadline_ms = CompletableEvent::calculate_deadline_ms(now_ms, ConsumerUtils::duration_to_millis(timeout));
7145 let (handle, receiver, _erased) =
7146 CompletableEvent::make_completable_event::<HashMap<String, Vec<crate::common::PartitionInfo>>>(deadline_ms);
7147 // Java's `listTopics` calls `setActiveTask(future)`
7148 // — `enable_wakeup=true`.
7149 self.submit_and_await::<HashMap<String, Vec<crate::common::PartitionInfo>>>(
7150 ApplicationEvent::AllTopicsMetadata { handle },
7151 receiver,
7152 deadline_ms,
7153 "Timeout expired while waiting for AllTopicsMetadata",
7154 true,
7155 )
7156 .await
7157 }
7158
7159 // ── Pause / resume ─────────────────────────────────────────────────
7160
7161 /// Java: `void pause(Collection<TopicPartition>)`
7162 /// (`AsyncKafkaConsumer#pause`).
7163 pub async fn pause(&mut self, partitions: &[TopicPartition]) -> Result<(), Error> {
7164 self.ensure_open()?;
7165 if partitions.is_empty() {
7166 return Ok(());
7167 }
7168 let deadline_ms = self.default_api_timeout_deadline_ms();
7169 let set: std::collections::HashSet<TopicPartition> = partitions.iter().cloned().collect();
7170 let (handle, receiver, _erased) = CompletableEvent::make_completable_event::<()>(deadline_ms);
7171 // Java's `pause(...)` does NOT call `setActiveTask` —
7172 // `enable_wakeup=false`.
7173 self.submit_and_await::<()>(
7174 ApplicationEvent::PausePartitions { handle, partitions: set },
7175 receiver,
7176 deadline_ms,
7177 "Timeout expired while waiting for PausePartitions",
7178 false,
7179 )
7180 .await
7181 }
7182
7183 /// Java: `void resume(Collection<TopicPartition>)`
7184 /// (`AsyncKafkaConsumer#resume`).
7185 pub async fn resume(&mut self, partitions: &[TopicPartition]) -> Result<(), Error> {
7186 self.ensure_open()?;
7187 if partitions.is_empty() {
7188 return Ok(());
7189 }
7190 let deadline_ms = self.default_api_timeout_deadline_ms();
7191 let set: std::collections::HashSet<TopicPartition> = partitions.iter().cloned().collect();
7192 let (handle, receiver, _erased) = CompletableEvent::make_completable_event::<()>(deadline_ms);
7193 // Java's `resume(...)` does NOT call `setActiveTask`.
7194 self.submit_and_await::<()>(
7195 ApplicationEvent::ResumePartitions { handle, partitions: set },
7196 receiver,
7197 deadline_ms,
7198 "Timeout expired while waiting for ResumePartitions",
7199 false,
7200 )
7201 .await
7202 }
7203
7204 // ── Enforce rebalance (KIP-848: unsupported) ──────────────────────
7205
7206 /// Java: `void enforceRebalance()` (`AsyncKafkaConsumer#enforceRebalance()`).
7207 ///
7208 /// Both Java overloads log a warning and otherwise no-op under the
7209 /// KIP-848 protocol (the classic protocol implements them via
7210 /// `ConsumerCoordinator`). We match that: log + no-op, return
7211 /// `Ok(())`. No `Error::unsupported_version` since Java does not
7212 /// throw.
7213 pub async fn enforce_rebalance(&mut self) -> Result<(), Error> {
7214 log::warn!("Operation not supported in new consumer group protocol");
7215 Ok(())
7216 }
7217
7218 /// Java: `void enforceRebalance(String reason)`
7219 /// (`AsyncKafkaConsumer#enforceRebalance(String)`). Same log + no-op body as
7220 /// [`Self::enforce_rebalance`]; Java ignores `reason` here too.
7221 pub async fn enforce_rebalance_with_reason(&mut self, _reason: &str) -> Result<(), Error> {
7222 log::warn!("Operation not supported in new consumer group protocol");
7223 Ok(())
7224 }
7225
7226 /// Java: `private void sendPrefetches(Timer timer)`
7227 /// (`AsyncKafkaConsumer#sendPrefetches`).
7228 ///
7229 /// Submits a non-completable `CreateFetchRequests` event so the bg
7230 /// task can pipeline the next fetch round-trip with the user's
7231 /// per-record processing. Errors from a closed bg-task channel are
7232 /// logged-and-swallowed (Java does the same).
7233 fn send_prefetches(&self) {
7234 let now_ms = self.time.milliseconds();
7235 let deadline_ms = self.default_api_timeout_deadline_ms();
7236 let (handle, _receiver, _erased) = CompletableEvent::make_completable_event::<()>(deadline_ms);
7237 // The event is `add`-ed (not `add_and_get`), so the receiver is
7238 // dropped and the handle is fire-and-forget. The bg side's
7239 // completion of the handle then resolves into the dropped
7240 // receiver, which is a no-op.
7241 if let Err(err) = self
7242 .application_event_handler
7243 .add(ApplicationEvent::CreateFetchRequests { handle }, now_ms)
7244 {
7245 log::warn!("send_prefetches: failed to enqueue CreateFetchRequests: {err}");
7246 }
7247 }
7248
7249 // ── Close path ─────────────────────────────────────────────────────
7250 //
7251 // Translates Java's `close()` chain
7252 // in `org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer`. The close sequence runs
7253 // best-effort: each step that throws is logged and the close
7254 // continues so the network thread is always joined. The final error
7255 // (if any) is propagated only when `swallow_error=false`.
7256 //
7257 // The eight Java steps (`AsyncKafkaConsumer#close(Duration,GroupMembershipOperation,boolean)`) map to Rust as:
7258 // 1. `wakeup_trigger.disable()` — Java's
7259 // `wakeupTrigger.disableWakeups()`.
7260 // 2. `auto_commit_on_close(deadline_ms)` — commit-sync of
7261 // `allConsumed()` + `CommitOnClose` event.
7262 // 3. `stop_find_coordinator_on_close()` —
7263 // `StopFindCoordinatorOnClose` event.
7264 // 4. `run_rebalance_callbacks_on_close()` — invoke
7265 // `on_partitions_revoked` / `on_partitions_lost` for the
7266 // currently-assigned partitions (depending on memberEpoch).
7267 // 5. `leave_group_on_close(deadline_ms, operation)` —
7268 // `LeaveGroupOnClose` event.
7269 // 6. `await_pending_async_commits_and_execute_commit_callbacks(...)` —
7270 // drain pending callbacks one last time on the caller's task.
7271 // 7. Drop `application_event_handler` — wakes the bg task so its
7272 // `run_once` loop exits.
7273 // 8. `network_thread_close.signal_close()` + `await_join()` —
7274 // Java's `closeQuietly(consumerNetworkThread)`.
7275
7276 /// Java: `void close()`. Closes the consumer with default timeout.
7277 pub async fn close(&mut self) -> Result<(), Error> {
7278 self.close_internal(
7279 Duration::from_millis(ConsumerUtils::DEFAULT_CLOSE_TIMEOUT_MS as u64),
7280 crate::consumer::GroupMembershipOperation::Default,
7281 false,
7282 )
7283 .await
7284 }
7285
7286 /// Java: `void close(CloseOptions options)`.
7287 pub async fn close_with_options(&mut self, options: crate::consumer::CloseOptions) -> Result<(), Error> {
7288 let timeout = options
7289 .timeout()
7290 .unwrap_or_else(|| Duration::from_millis(ConsumerUtils::DEFAULT_CLOSE_TIMEOUT_MS as u64));
7291 self.close_internal(timeout, options.group_membership_operation(), false).await
7292 }
7293
7294 /// Translates Java's
7295 /// `private void close(Duration timeout,
7296 /// CloseOptions.GroupMembershipOperation membershipOperation,
7297 /// boolean swallowException)`
7298 /// (`AsyncKafkaConsumer#close(Duration,GroupMembershipOperation,boolean)`).
7299 async fn close_internal(
7300 &mut self,
7301 timeout: Duration,
7302 membership_operation: crate::consumer::GroupMembershipOperation,
7303 swallow_error: bool,
7304 ) -> Result<(), Error> {
7305 log::trace!("Closing the Kafka consumer");
7306 if self.is_closed() {
7307 // Java treats double-close as a silent no-op (the closed
7308 // accessors return immediately).
7309 return Ok(());
7310 }
7311
7312 // Step 1: disable wakeups. After this call, in-flight async
7313 // operations cannot be cancelled mid-close (Java
7314 // `wakeupTrigger.disableWakeups()`).
7315 self.wakeup_trigger.disable();
7316
7317 // Java's `createTimerForCloseRequests(timeout)`
7318 // (`org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer#createTimerForCloseRequests`) caps the
7319 // user-supplied timeout at `requestTimeoutMs`. With the default config
7320 // (timeout=30s, request_timeout_ms=30s) the cap is a no-op,
7321 // but a user calling `close(Duration::from_secs(300))` would
7322 // otherwise block the consumer for 5 minutes per
7323 // close-step — Java clips to `request.timeout.ms` so each
7324 // close-step inherits the broker-RPC bound.
7325 let request_timeout_ms = self.config.request_timeout_ms() as i64;
7326 let capped_timeout_ms = std::cmp::min(ConsumerUtils::duration_to_millis(timeout), request_timeout_ms);
7327 let close_start_ms = self.time.milliseconds();
7328 let close_deadline_ms = CompletableEvent::calculate_deadline_ms(close_start_ms, capped_timeout_ms);
7329
7330 // First-error tracking mirrors Java's `AtomicReference<Throwable> firstException`.
7331 let mut first_error: Option<Error> = None;
7332 let record = |slot: &mut Option<Error>, op: &str, err: Error| {
7333 log::error!("{op}: {err}");
7334 if slot.is_none() {
7335 *slot = Some(err);
7336 }
7337 };
7338
7339 // Step 2: auto_commit_on_close.
7340 if let Err(err) = self.auto_commit_on_close(close_deadline_ms).await {
7341 record(&mut first_error, "Failed to auto-commit offsets", err);
7342 }
7343
7344 // Step 3: stop_find_coordinator_on_close.
7345 if let Err(err) = self.stop_find_coordinator_on_close() {
7346 record(&mut first_error, "Failed to stop finding coordinator", err);
7347 }
7348
7349 // Step 4: run_rebalance_callbacks_on_close.
7350 if let Err(err) = self.run_rebalance_callbacks_on_close().await {
7351 record(&mut first_error, "Failed to run rebalance callbacks", err);
7352 }
7353
7354 // Step 5: leave_group_on_close.
7355 if let Err(err) = self
7356 .leave_group_on_close(close_deadline_ms, capped_timeout_ms, membership_operation)
7357 .await
7358 {
7359 record(&mut first_error, "Failed to leave group while closing consumer", err);
7360 }
7361
7362 // Step 6: drain pending async commits one last time.
7363 // Java passes `enable_wakeup=false` here.
7364 if let Err(err) = self
7365 .await_pending_async_commits_and_execute_commit_callbacks(close_deadline_ms, false)
7366 .await
7367 {
7368 record(
7369 &mut first_error,
7370 "Failed invoking asynchronous commit callbacks while closing consumer",
7371 err,
7372 );
7373 }
7374
7375 // Step 7 & 8: shut down the network thread, bounding its cleanup by
7376 // what is left of the close timer — Java's
7377 // `applicationEventHandler.close(Duration.ofMillis(closeTimer.remainingMs()))`
7378 // (`AsyncKafkaConsumer#close(Duration,GroupMembershipOperation,boolean)`).
7379 let remaining_ms = close_deadline_ms.saturating_sub(self.time.milliseconds()).max(0);
7380 self.network_thread_close
7381 .signal_close(Duration::from_millis(remaining_ms as u64));
7382 self.network_thread_close.wakeup();
7383 if let Err(err) = self.network_thread_close.await_join().await {
7384 record(&mut first_error, "Failed shutting down network thread", err);
7385 }
7386
7387 // Final reaper pass (Java `backgroundEventReaper.reap(backgroundEventQueue)`) — drain any background events
7388 // queued during the close path.
7389 {
7390 let now_ms = self.time.milliseconds();
7391 let mut reaper = self.completable_event_reaper.lock().unwrap();
7392 reaper.reap(now_ms);
7393 }
7394
7395 // Java: `closeQuietly(kafkaConsumerMetrics, "kafka consumer metrics",
7396 // firstException)` in `AsyncKafkaConsumer#close(Duration,GroupMembershipOperation,boolean)` — removes the
7397 // consumer-level poll/commit metrics from the registry. `close()` is
7398 // infallible here (no error to fold into `first_error`).
7399 self.kafka_consumer_metrics.close();
7400
7401 // Java: `closeQuietly(asyncConsumerMetrics, "async consumer metrics",
7402 // firstException)` in `AsyncKafkaConsumer#close(Duration,GroupMembershipOperation,boolean)` — removes the
7403 // async-consumer background-task / event-queue sensors from the
7404 // registry. `close()` is infallible here.
7405 self.async_consumer_metrics.close();
7406
7407 // Java (KAFKA-19542): `closeQuietly(fetchMetricsManager, "consumer fetch
7408 // metrics", firstException)` and `closeQuietly(rebalanceCallbackMetricsManager,
7409 // "consumer rebalance callback
7410 // metrics")` in `AsyncKafkaConsumer#close(Duration,GroupMembershipOperation,boolean)` — remove the fetch
7411 // metrics, including the per-topic / per-partition
7412 // ones, and the rebalance-callback latency metrics. Both infallible.
7413 self.fetch_metrics_manager.close();
7414 self.rebalance_callback_metrics_manager.close();
7415
7416 // Java: `closeQuietly(deserializers, "consumer deserializers",
7417 // firstException)` in `AsyncKafkaConsumer#close(Duration,GroupMembershipOperation,boolean)`. The collector
7418 // holds the only reference to the deserializers, and nothing else
7419 // holds the collector once the background task has been joined, so
7420 // closing them through `&mut` cannot fail on sharing in practice;
7421 // `close_deserializers` still reports it rather than skipping the close.
7422 let deserializers_closed = match Arc::get_mut(&mut self.fetch_collector) {
7423 Some(fetch_collector) => fetch_collector.close_deserializers().await,
7424 None => Err(Error::local_illegal_state(
7425 "Cannot close the consumer deserializers: the fetch collector is still shared",
7426 )),
7427 };
7428 if let Err(err) = deserializers_closed {
7429 record(&mut first_error, "Failed to close consumer deserializers", err);
7430 }
7431
7432 // Rust-only: release the rebalance listener. The last callback ran in
7433 // step 4 or 5 and the background task has been joined, so nothing
7434 // invokes it any more. A listener that captured `consumer.handle()`
7435 // holds the handle state, and through it the `SubscriptionState` that
7436 // stores the listener: without this the cycle would keep the listener,
7437 // the subscription state and the metrics registry alive after the
7438 // consumer is dropped. Java's garbage collector frees the same graph,
7439 // so Java's `close()` has no counterpart step.
7440 self.subscriptions
7441 .lock()
7442 .unwrap_or_else(std::sync::PoisonError::into_inner)
7443 .register_rebalance_listener(None);
7444 *self
7445 .rebalance_listener
7446 .lock()
7447 .unwrap_or_else(std::sync::PoisonError::into_inner) = None;
7448
7449 self.closed.store(true, Ordering::Release);
7450 log::debug!("Kafka consumer has been closed");
7451
7452 // Java
7453 // (`AsyncKafkaConsumer#close(Duration,GroupMembershipOperation,boolean)`):
7454 //
7455 // ```java
7456 // Throwable exception = firstException.get();
7457 // if (exception != null && !swallowException) {
7458 // if (exception instanceof InterruptException) {
7459 // throw (InterruptException) exception;
7460 // }
7461 // throw new KafkaException("Failed to close kafka consumer", exception);
7462 // }
7463 // ```
7464 //
7465 // The wrap is what makes `catch (KafkaException e)` around `close()`
7466 // — the canonical Java idiom — reliable: whatever step failed, the
7467 // caller is handed a `KafkaException` with this message and the
7468 // original as its cause. It matters here because `first_error` can
7469 // hold an error that is NOT in the `KafkaException` hierarchy, e.g.
7470 // the `Error::LocalIllegalState` "Consumer background task is no longer
7471 // running." raised through
7472 // `await_pending_async_commits_and_execute_commit_callbacks`.
7473 //
7474 // Java's `InterruptException` pass-through has no counterpart: Rust
7475 // tasks have no thread-interruption mechanism, so no step can record
7476 // that error.
7477 match first_error {
7478 Some(err) if !swallow_error => Err(Error::kafka_message_source("Failed to close kafka consumer", err)),
7479 _ => Ok(()),
7480 }
7481 }
7482
7483 /// Java: `private void autoCommitOnClose(final Timer timer)`
7484 /// (`AsyncKafkaConsumer#autoCommitOnClose`).
7485 async fn auto_commit_on_close(&mut self, deadline_ms: i64) -> Result<(), Error> {
7486 if self.group_id.is_none() {
7487 return Ok(());
7488 }
7489
7490 if self.auto_commit_enabled {
7491 // Java: `commitSyncAllConsumed(timer)` swallows errors and
7492 // logs a warning. Match that — auto-commit failure on close
7493 // does not propagate.
7494 let remaining_ms = self.remaining_ms(deadline_ms).max(0) as u64;
7495 if let Err(err) = self.commit_sync_with_timeout(Duration::from_millis(remaining_ms)).await {
7496 log::warn!("Synchronous auto-commit failed: {err}");
7497 }
7498 }
7499
7500 // Java: `applicationEventHandler.add(new CommitOnCloseEvent())`.
7501 let now_ms = self.time.milliseconds();
7502 let _ = self.application_event_handler.add(ApplicationEvent::CommitOnClose, now_ms);
7503 Ok(())
7504 }
7505
7506 /// Java: `private void stopFindCoordinatorOnClose()`
7507 /// (`AsyncKafkaConsumer#stopFindCoordinatorOnClose`).
7508 fn stop_find_coordinator_on_close(&self) -> Result<(), Error> {
7509 if self.group_id.is_none() {
7510 return Ok(());
7511 }
7512 log::debug!("Stop finding coordinator during consumer close");
7513 let now_ms = self.time.milliseconds();
7514 self.application_event_handler
7515 .add(ApplicationEvent::StopFindCoordinatorOnClose, now_ms)
7516 }
7517
7518 /// Java: `private void runRebalanceCallbacksOnClose()`
7519 /// (`AsyncKafkaConsumer#runRebalanceCallbacksOnClose`).
7520 ///
7521 /// Invokes the user's `on_partitions_revoked` (if `memberEpoch > 0`)
7522 /// or `on_partitions_lost` (if `memberEpoch <= 0`) on the
7523 /// **group-assigned** partitions captured by the most recent
7524 /// reconciliation. The listener runs inline on the caller's task
7525 /// (§31). Errors propagate.
7526 ///
7527 /// # Why `group_assignment_snapshot` and not `subscriptions.assigned_partitions()`
7528 ///
7529 /// Java reads from `groupAssignmentSnapshot.get()`
7530 /// in `AsyncKafkaConsumer#runRebalanceCallbacksOnClose`, which is populated only by the
7531 /// `MemberStateListener.onGroupAssignmentUpdated` callback fired
7532 /// during reconciliation. The snapshot deliberately excludes
7533 /// partitions added via `assign(...)` (manual assignment) so
7534 /// non-group consumers get no callback on close —
7535 /// `AsyncKafkaConsumer#runRebalanceCallbacksOnClose` returns early on an
7536 /// empty snapshot.
7537 ///
7538 /// Reading from `SubscriptionState::assigned_partitions()` would
7539 /// (a) include manual `assign(...)` partitions, and (b) miss the
7540 /// "partition was just revoked but `SubscriptionState` hasn't been
7541 /// updated yet" window the snapshot still covers.
7542 async fn run_rebalance_callbacks_on_close(&mut self) -> Result<(), Error> {
7543 if self.group_id.is_none() {
7544 return Ok(());
7545 }
7546
7547 // Java: `Set<TopicPartition> assignedPartitions = groupAssignmentSnapshot.get();`
7548 let assigned: Vec<TopicPartition> = {
7549 let guard = self.group_assignment_snapshot.lock().unwrap();
7550 guard.iter().cloned().collect()
7551 };
7552 // Java: `if (assignedPartitions.isEmpty()) return;`.
7553 if assigned.is_empty() {
7554 return Ok(());
7555 }
7556
7557 // Java reads `generationId` from the cached `groupMetadata`.
7558 // The Rust translation reads the cached value the same way.
7559 let member_epoch = {
7560 let guard = self.group_metadata.lock().unwrap();
7561 guard.as_ref().map(|gm| gm.generation_id()).unwrap_or(-1)
7562 };
7563
7564 let listener_opt = self.rebalance_listener.lock().unwrap().clone();
7565 let listener = match listener_opt {
7566 Some(l) => l,
7567 // Java's `rebalanceListenerInvoker.invokePartitions*` is a
7568 // no-op when no listener is registered.
7569 None => return Ok(()),
7570 };
7571
7572 // Admits the listener's `ConsumerHandle` calls (see
7573 // `process_background_events_inner`).
7574 let _in_callback = self.offset_commit_callback_invoker.enter_callback();
7575 let result = if member_epoch > 0 {
7576 self.rebalance_listener_invoker
7577 .invoke_partitions_revoked(&listener, &assigned)
7578 .await
7579 } else {
7580 self.rebalance_listener_invoker
7581 .invoke_partitions_lost(&listener, &assigned)
7582 .await
7583 };
7584
7585 // Java: `if (error != null) throw ConsumerUtils.maybeWrapAsKafkaException(error);`
7586 // in `org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer#runRebalanceCallbacksOnClose`. Without the
7587 // wrap the same user-listener error is classified differently depending on the path
7588 // that surfaced it: wrapped on a normal rebalance (via
7589 // `maybe_wrap_as_kafka_error_with_msg`) but raw here, so
7590 // `is_kafka_error()` would answer `true` in one case and `false` in the
7591 // other for one and the same listener failure.
7592 result.map_err(crate::consumer::internals::ConsumerUtils::maybe_wrap_as_kafka_error)
7593 }
7594
7595 /// Java: `private void leaveGroupOnClose(Timer, GroupMembershipOperation)`
7596 /// (`AsyncKafkaConsumer#leaveGroupOnClose`).
7597 async fn leave_group_on_close(
7598 &mut self,
7599 deadline_ms: i64,
7600 timeout_ms: i64,
7601 membership_operation: crate::consumer::GroupMembershipOperation,
7602 ) -> Result<(), Error> {
7603 if self.group_id.is_none() {
7604 return Ok(());
7605 }
7606
7607 log::debug!("Leaving the consumer group during consumer close");
7608 // The erased handle is not kept: once the bg task drops every clone of
7609 // the handle without completing it, the wait below sees that and
7610 // proceeds instead of waiting out its backstop.
7611 let (handle, receiver, _) = CompletableEvent::make_completable_event::<()>(deadline_ms);
7612 // Java's `leaveGroupOnClose` does NOT call `setActiveTask` and
7613 // close has already called `wakeup_trigger.disable()`
7614 // (`AsyncKafkaConsumer#close(Duration,GroupMembershipOperation,boolean)`
7615 // calls `disableWakeups()`) so wakeup is inert in this path —
7616 // `enable_wakeup=false`.
7617 //
7618 // Java waits with a plain `applicationEventHandler.addAndGet(...)`
7619 // (`AsyncKafkaConsumer#leaveGroupOnClose`), without processing background
7620 // events: a queued rebalance callback is not run (close runs the
7621 // listener only through `runRebalanceCallbacksOnClose`, Step 4) and a
7622 // queued `ErrorEvent` neither fails close nor stops this wait. Nothing
7623 // on the background task waits for those events either: the leave on
7624 // close runs no callbacks (`leave_group_on_close` → `run_callbacks =
7625 // false`), and a reconcile or release parked on a listener ack only
7626 // `try_recv`s it across bg-loop iterations, so the shutdown proceeds
7627 // and the unread events are dropped with the consumer, as Java
7628 // abandons them.
7629 //
7630 // The wait is not bounded by `deadline_ms` (see
7631 // `submit_and_await_for_close`), so `close(Duration::ZERO)` still
7632 // leaves the group, as Java's does.
7633 let result = self
7634 .submit_and_await_for_close::<()>(
7635 ApplicationEvent::LeaveGroupOnClose { handle, membership_operation },
7636 receiver,
7637 deadline_ms,
7638 "Timeout expired while waiting for LeaveGroupOnClose",
7639 )
7640 .await;
7641 match result {
7642 Ok(Some(())) => {
7643 log::info!("Completed leaving the group");
7644 Ok(())
7645 },
7646 Ok(None) => {
7647 // The bg task ended without completing the event (Java's
7648 // shutdown fails it with a `TimeoutException`, handled
7649 // below); there is nothing left to wait for.
7650 log::warn!(
7651 "Consumer attempted to leave the group but the background task ended before completing it. \
7652 It will proceed to close."
7653 );
7654 Ok(())
7655 },
7656 // The reaper expired the event at its deadline, or the backstop in
7657 // `submit_and_await_for_close` gave up.
7658 Err(Error::Timeout(_)) => {
7659 // Java's `catch (TimeoutException) { log.warn(...) }` —
7660 // close proceeds.
7661 // Java logs `timer.timeoutMs()` — the *configured* close
7662 // timeout, not the remaining budget (which is 0 on exactly
7663 // this path, and so tells the operator nothing).
7664 log::warn!(
7665 "Consumer attempted to leave the group but couldn't complete it within {} ms. \
7666 It will proceed to close.",
7667 timeout_ms
7668 );
7669 Ok(())
7670 },
7671 Err(err) => Err(err),
7672 }
7673 }
7674}
7675
7676// ── `Consumer<K, V>` trait impl ─────────────────────────────────────────
7677//
7678// Delegates each trait method to the inherent impl. The split is
7679// deliberate: the inherent impl carries the (untyped) ctor, the
7680// internal helpers, and the API used by crate code that holds an
7681// `AsyncKafkaConsumer<K, V>` directly. The trait impl exposes the same
7682// surface through `KafkaConsumer`, which forwards every `Consumer` method
7683// here, and through `DynConsumer` for callers that want type-erased
7684// dispatch.
7685
7686impl<K, V> crate::consumer::Consumer<K, V> for AsyncKafkaConsumer<K, V>
7687where
7688 K: Send + Sync + 'static,
7689 V: Send + Sync + 'static,
7690{
7691 // ── Sync state-read methods ────────────────────────────────────────
7692
7693 fn assignment(&mut self) -> std::collections::HashSet<TopicPartition> {
7694 AsyncKafkaConsumer::assignment(self)
7695 }
7696
7697 fn subscription(&mut self) -> std::collections::HashSet<String> {
7698 AsyncKafkaConsumer::subscription(self)
7699 }
7700
7701 fn paused(&mut self) -> std::collections::HashSet<TopicPartition> {
7702 AsyncKafkaConsumer::paused(self)
7703 }
7704
7705 fn group_metadata(&mut self) -> Arc<dyn ConsumerGroupMetadata> {
7706 AsyncKafkaConsumer::group_metadata(self)
7707 }
7708
7709 async fn current_lag(&mut self, topic_partition: &TopicPartition) -> Result<Option<i64>, Error> {
7710 AsyncKafkaConsumer::current_lag(self, topic_partition).await
7711 }
7712
7713 fn metrics(&self) -> MetricsView {
7714 AsyncKafkaConsumer::metrics(self)
7715 }
7716
7717 fn wakeup(&self) {
7718 AsyncKafkaConsumer::wakeup(self);
7719 }
7720
7721 fn handle(&self) -> ConsumerHandle {
7722 AsyncKafkaConsumer::handle(self)
7723 }
7724
7725 // ── Subscribe / unsubscribe / assign ───────────────────────────────
7726
7727 async fn subscribe_with_topics(&mut self, topics: Vec<String>) -> Result<(), Error> {
7728 AsyncKafkaConsumer::subscribe_with_topics(self, topics).await
7729 }
7730
7731 async fn subscribe_with_topics_listener(
7732 &mut self,
7733 topics: Vec<String>,
7734 listener: Arc<dyn ConsumerRebalanceListener>,
7735 ) -> Result<(), Error> {
7736 AsyncKafkaConsumer::subscribe_with_topics_listener(self, topics, listener).await
7737 }
7738
7739 async fn subscribe_with_pattern(&mut self, pattern: SubscriptionPattern) -> Result<(), Error> {
7740 AsyncKafkaConsumer::subscribe_with_pattern(self, pattern).await
7741 }
7742
7743 async fn subscribe_with_pattern_listener(
7744 &mut self,
7745 pattern: SubscriptionPattern,
7746 listener: Arc<dyn ConsumerRebalanceListener>,
7747 ) -> Result<(), Error> {
7748 AsyncKafkaConsumer::subscribe_with_pattern_listener(self, pattern, listener).await
7749 }
7750
7751 async fn assign(&mut self, partitions: Vec<TopicPartition>) -> Result<(), Error> {
7752 AsyncKafkaConsumer::assign(self, partitions).await
7753 }
7754
7755 async fn unsubscribe(&mut self) -> Result<(), Error> {
7756 AsyncKafkaConsumer::unsubscribe(self).await
7757 }
7758
7759 // ── Poll ───────────────────────────────────────────────────────────
7760
7761 async fn poll(&mut self, timeout: Duration) -> Result<ConsumerRecords<K, V>, Error> {
7762 AsyncKafkaConsumer::poll(self, timeout).await
7763 }
7764
7765 // ── Commit ─────────────────────────────────────────────────────────
7766
7767 async fn commit_sync(&mut self) -> Result<(), Error> {
7768 AsyncKafkaConsumer::commit_sync(self).await
7769 }
7770
7771 async fn commit_sync_with_timeout(&mut self, timeout: Duration) -> Result<(), Error> {
7772 AsyncKafkaConsumer::commit_sync_with_timeout(self, timeout).await
7773 }
7774
7775 async fn commit_sync_with_offsets(
7776 &mut self,
7777 offsets: HashMap<TopicPartition, OffsetAndMetadata>,
7778 ) -> Result<(), Error> {
7779 AsyncKafkaConsumer::commit_sync_with_offsets(self, offsets).await
7780 }
7781
7782 async fn commit_sync_with_offsets_timeout(
7783 &mut self,
7784 offsets: HashMap<TopicPartition, OffsetAndMetadata>,
7785 timeout: Duration,
7786 ) -> Result<(), Error> {
7787 AsyncKafkaConsumer::commit_sync_with_offsets_timeout(self, offsets, timeout).await
7788 }
7789
7790 async fn commit_async(&mut self) -> Result<(), Error> {
7791 AsyncKafkaConsumer::commit_async(self).await
7792 }
7793
7794 async fn commit_async_with_callback(
7795 &mut self,
7796 callback: Arc<dyn crate::consumer::OffsetCommitCallback>,
7797 ) -> Result<(), Error> {
7798 AsyncKafkaConsumer::commit_async_with_callback(self, callback).await
7799 }
7800
7801 async fn commit_async_with_offsets_callback(
7802 &mut self,
7803 offsets: HashMap<TopicPartition, OffsetAndMetadata>,
7804 callback: Arc<dyn crate::consumer::OffsetCommitCallback>,
7805 ) -> Result<(), Error> {
7806 AsyncKafkaConsumer::commit_async_with_offsets_callback(self, offsets, callback).await
7807 }
7808
7809 // ── Seek ───────────────────────────────────────────────────────────
7810
7811 async fn seek_with_offset(&mut self, partition: TopicPartition, offset: i64) -> Result<(), Error> {
7812 AsyncKafkaConsumer::seek_with_offset(self, partition, offset).await
7813 }
7814
7815 async fn seek_with_offset_and_metadata(
7816 &mut self,
7817 partition: TopicPartition,
7818 offset_and_metadata: OffsetAndMetadata,
7819 ) -> Result<(), Error> {
7820 AsyncKafkaConsumer::seek_with_offset_and_metadata(self, partition, offset_and_metadata).await
7821 }
7822
7823 async fn seek_to_beginning(&mut self, partitions: &[TopicPartition]) -> Result<(), Error> {
7824 AsyncKafkaConsumer::seek_to_beginning(self, partitions).await
7825 }
7826
7827 async fn seek_to_end(&mut self, partitions: &[TopicPartition]) -> Result<(), Error> {
7828 AsyncKafkaConsumer::seek_to_end(self, partitions).await
7829 }
7830
7831 // ── Position / committed ───────────────────────────────────────────
7832
7833 async fn position(&mut self, partition: &TopicPartition) -> Result<i64, Error> {
7834 AsyncKafkaConsumer::position(self, partition).await
7835 }
7836
7837 async fn position_with_timeout(&mut self, partition: &TopicPartition, timeout: Duration) -> Result<i64, Error> {
7838 AsyncKafkaConsumer::position_with_timeout(self, partition, timeout).await
7839 }
7840
7841 async fn committed(
7842 &mut self,
7843 partitions: &[TopicPartition],
7844 ) -> Result<HashMap<TopicPartition, Option<OffsetAndMetadata>>, Error> {
7845 AsyncKafkaConsumer::committed(self, partitions).await
7846 }
7847
7848 async fn committed_with_timeout(
7849 &mut self,
7850 partitions: &[TopicPartition],
7851 timeout: Duration,
7852 ) -> Result<HashMap<TopicPartition, Option<OffsetAndMetadata>>, Error> {
7853 AsyncKafkaConsumer::committed_with_timeout(self, partitions, timeout).await
7854 }
7855
7856 // ── Topic metadata ────────────────────────────────────────────────
7857
7858 async fn partitions_for(&mut self, topic: &str) -> Result<Vec<crate::common::PartitionInfo>, Error> {
7859 AsyncKafkaConsumer::partitions_for(self, topic).await
7860 }
7861
7862 async fn partitions_for_with_timeout(
7863 &mut self,
7864 topic: &str,
7865 timeout: Duration,
7866 ) -> Result<Vec<crate::common::PartitionInfo>, Error> {
7867 AsyncKafkaConsumer::partitions_for_with_timeout(self, topic, timeout).await
7868 }
7869
7870 async fn list_topics(&mut self) -> Result<HashMap<String, Vec<crate::common::PartitionInfo>>, Error> {
7871 AsyncKafkaConsumer::list_topics(self).await
7872 }
7873
7874 async fn list_topics_with_timeout(
7875 &mut self,
7876 timeout: Duration,
7877 ) -> Result<HashMap<String, Vec<crate::common::PartitionInfo>>, Error> {
7878 AsyncKafkaConsumer::list_topics_with_timeout(self, timeout).await
7879 }
7880
7881 async fn offsets_for_times(
7882 &mut self,
7883 timestamps_to_search: HashMap<TopicPartition, i64>,
7884 ) -> Result<HashMap<TopicPartition, Option<OffsetAndTimestamp>>, Error> {
7885 AsyncKafkaConsumer::offsets_for_times(self, timestamps_to_search).await
7886 }
7887
7888 async fn offsets_for_times_with_timeout(
7889 &mut self,
7890 timestamps_to_search: HashMap<TopicPartition, i64>,
7891 timeout: Duration,
7892 ) -> Result<HashMap<TopicPartition, Option<OffsetAndTimestamp>>, Error> {
7893 AsyncKafkaConsumer::offsets_for_times_with_timeout(self, timestamps_to_search, timeout).await
7894 }
7895
7896 async fn beginning_offsets(
7897 &mut self,
7898 partitions: &[TopicPartition],
7899 ) -> Result<HashMap<TopicPartition, i64>, Error> {
7900 AsyncKafkaConsumer::beginning_offsets(self, partitions).await
7901 }
7902
7903 async fn beginning_offsets_with_timeout(
7904 &mut self,
7905 partitions: &[TopicPartition],
7906 timeout: Duration,
7907 ) -> Result<HashMap<TopicPartition, i64>, Error> {
7908 AsyncKafkaConsumer::beginning_offsets_with_timeout(self, partitions, timeout).await
7909 }
7910
7911 async fn end_offsets(&mut self, partitions: &[TopicPartition]) -> Result<HashMap<TopicPartition, i64>, Error> {
7912 AsyncKafkaConsumer::end_offsets(self, partitions).await
7913 }
7914
7915 async fn end_offsets_with_timeout(
7916 &mut self,
7917 partitions: &[TopicPartition],
7918 timeout: Duration,
7919 ) -> Result<HashMap<TopicPartition, i64>, Error> {
7920 AsyncKafkaConsumer::end_offsets_with_timeout(self, partitions, timeout).await
7921 }
7922
7923 // ── Pause / resume ─────────────────────────────────────────────────
7924
7925 async fn pause(&mut self, partitions: &[TopicPartition]) -> Result<(), Error> {
7926 AsyncKafkaConsumer::pause(self, partitions).await
7927 }
7928
7929 async fn resume(&mut self, partitions: &[TopicPartition]) -> Result<(), Error> {
7930 AsyncKafkaConsumer::resume(self, partitions).await
7931 }
7932
7933 // ── Lifecycle ──────────────────────────────────────────────────────
7934
7935 async fn enforce_rebalance(&mut self) -> Result<(), Error> {
7936 AsyncKafkaConsumer::enforce_rebalance(self).await
7937 }
7938
7939 async fn enforce_rebalance_with_reason(&mut self, reason: &str) -> Result<(), Error> {
7940 AsyncKafkaConsumer::enforce_rebalance_with_reason(self, reason).await
7941 }
7942
7943 async fn close(&mut self) -> Result<(), Error> {
7944 AsyncKafkaConsumer::close(self).await
7945 }
7946
7947 async fn close_with_options(&mut self, options: crate::consumer::CloseOptions) -> Result<(), Error> {
7948 AsyncKafkaConsumer::close_with_options(self, options).await
7949 }
7950}
7951
7952#[cfg(test)]
7953mod tests {
7954 //! State-read tests for the Phase 11 commit (2/N) ctor.
7955 //!
7956 //! Most Java tests in `AsyncKafkaConsumerTest.java` exercise
7957 //! end-to-end behavior (poll / commit / close). The handful that
7958 //! fit commit (2/N) are state-read tests:
7959 //!
7960 //! - `testAssignmentEmpty` (no subscribe → empty assignment),
7961 //! - `testSubscriptionEmpty`,
7962 //! - `testWakeupIsCallableFromMultipleTasks`,
7963 //! - `testClientIdReturnsConfigValue`,
7964 //! - `testGroupMetadataReturnsStubForGrouplessConsumer`.
7965 //!
7966 //! Subscribe / unsubscribe / assign tests land in commit (3/N).
7967 //! Poll / commit / close tests land in commits (4/N)-(7/N).
7968
7969 use std::collections::HashSet;
7970 use std::sync::atomic::AtomicI64;
7971
7972 use tokio::sync::mpsc;
7973
7974 use crate::common::internals::ClusterResourceListeners;
7975 use crate::common::serialization::Deserializer;
7976 use crate::consumer::internals::AutoOffsetResetStrategy;
7977 use crate::consumer::internals::events::ApplicationEventEnvelope;
7978 use crate::consumer::internals::events::CompletableEventReaper;
7979
7980 use super::*;
7981
7982 /// Minimal `Vec<u8>` deserializer for tests — equivalent to Java's
7983 /// `ByteArrayDeserializer`. Returns the input bytes unchanged.
7984 struct TestBytesDeserializer;
7985
7986 impl Deserializer<Vec<u8>> for TestBytesDeserializer {
7987 async fn deserialize(&self, _topic: &str, data: &[u8]) -> Result<Vec<u8>, Error> {
7988 Ok(data.to_vec())
7989 }
7990 }
7991
7992 /// Test-side handles handed back from [`make_test_consumer_with_channels`].
7993 /// Holding these lets the test impersonate the bg task: drain the
7994 /// app-event channel to act on `add_and_get`-style events, and push
7995 /// background events into the consumer's bg-event channel for
7996 /// `process_background_events` to drain.
7997 struct ConsumerTestHandles {
7998 /// Receiver for app-side events. Held by the test instead of the
7999 /// (absent) bg task.
8000 app_event_rx: mpsc::UnboundedReceiver<ApplicationEventEnvelope>,
8001 /// Sender for bg-side events, used to enqueue events the bg task
8002 /// would normally post.
8003 bg_event_tx: mpsc::UnboundedSender<BackgroundEventEnvelope>,
8004 /// Handle on the shared `SubscriptionState` so tests can inspect /
8005 /// pre-populate it.
8006 subscriptions: Arc<Mutex<SubscriptionState>>,
8007 /// The close timeout (millis) `close()` handed the bg task through
8008 /// `NetworkThreadCloseHandle::signal_close`; `-1` until signalled.
8009 signal_close_timeout_ms: Arc<AtomicI64>,
8010 /// Set to `true` whenever the consumer's bg-task wakeup fn is invoked
8011 /// (Phase 41 Issue 3 observability). Lets a `&mut self`-level component
8012 /// test assert that the ack-send path in `process_background_events`
8013 /// pokes the bg wakeup `Notify`.
8014 ///
8015 /// NOTE: this flag stands in for the *user-facing* wakeup
8016 /// (`NetworkThreadCloseHandle::wakeup` → `WakeupTrigger::wakeup`).
8017 /// The ack path must NOT fire that one — see
8018 /// `process_background_events_ack_pokes_bg_notify_not_user_wakeup`.
8019 bg_wakeup_called: Arc<AtomicBool>,
8020 /// The application-event `Notify` the bg loop parks on — the correct
8021 /// target of the §31 step-4 ack poke.
8022 event_notify: Arc<tokio::sync::Notify>,
8023 }
8024
8025 /// Core test-consumer builder: constructs a consumer whose `group.id`
8026 /// and `group.instance.id` are the given values and whose
8027 /// `OffsetCommitCallbackInvoker` runs `invoker_interceptors` — the chain
8028 /// whose `onCommit` an async commit fires (Java's `MockConsumerInterceptor`
8029 /// from `interceptor.classes`, which production does not load yet) — and
8030 /// returns it together with the test-side channel handles needed to act as
8031 /// the bg task during a test.
8032 ///
8033 /// The group-metadata slot is seeded through the SAME
8034 /// `initialize_group_metadata` helper the production ctor uses
8035 /// (definition-of-done.md #12), so the fixture and production cannot
8036 /// diverge: `group_id: Some(..)` starts with UNKNOWN generation / member
8037 /// id plus the configured `group.instance.id`; `group_id: None` builds an
8038 /// assignment-only consumer whose slot stays empty.
8039 fn make_test_consumer_with_group_and_invoker_interceptors(
8040 group_id: Option<&str>,
8041 group_instance_id: Option<&str>,
8042 invoker_interceptors: Vec<Box<dyn crate::consumer::ConsumerInterceptor<Vec<u8>, Vec<u8>>>>,
8043 ) -> (AsyncKafkaConsumer<Vec<u8>, Vec<u8>>, ConsumerTestHandles) {
8044 let mut config = ConsumerConfig {
8045 bootstrap_servers: vec!["localhost:9092".to_string()],
8046 ..ConsumerConfig::defaults()
8047 };
8048 config.client_id = "test-client".to_string();
8049 config.group_id = group_id.map(str::to_string);
8050 config.group_instance_id = group_instance_id.map(str::to_string);
8051 let client_id: Arc<str> = Arc::from(config.client_id.as_str());
8052
8053 let subs = Arc::new(Mutex::new(SubscriptionState::new(AutoOffsetResetStrategy::EARLIEST)));
8054 let metadata = Arc::new(ConsumerMetadata::with_config(
8055 &config,
8056 Arc::clone(&subs),
8057 ClusterResourceListeners::new(),
8058 ));
8059 let request_managers = Arc::new(std::sync::Mutex::new(RequestManagers::new(
8060 None, None, None, None, None, None, None,
8061 )));
8062 // Test stand-in for the bg task's app-event receiver. Tests hold
8063 // this `app_event_rx` and pull events off it themselves.
8064 let (app_handler_tx, app_event_rx) = mpsc::unbounded_channel::<ApplicationEventEnvelope>();
8065 // Held by the returned handles so tests can assert on the REAL poke
8066 // primitive the bg loop parks on, rather than a stub flag.
8067 let event_notify = Arc::new(tokio::sync::Notify::new());
8068 let app_handler = Arc::new(ApplicationEventHandler::new(app_handler_tx, Arc::clone(&event_notify)));
8069 let reaper = Arc::new(std::sync::Mutex::new(CompletableEventReaper::new()));
8070 let max_time = Arc::new(AtomicI64::new(0));
8071 let wakeup = WakeupTrigger::new();
8072
8073 // Stub join handle — spawn a noop task. Tests do not assert on
8074 // the bg task's behavior in this commit.
8075 let join_handle: JoinHandle<()> = tokio::spawn(async move {});
8076 // Records the timeout `close()` hands the bg task (-1 = never signalled).
8077 let signal_close_timeout_ms = Arc::new(AtomicI64::new(-1));
8078 let signal_close_slot = Arc::clone(&signal_close_timeout_ms);
8079 let wakeup_called = Arc::new(AtomicBool::new(false));
8080 let wakeup_flag = Arc::clone(&wakeup_called);
8081 // Production's `wakeup_fn` fires the `WakeupTrigger` (see the ctor);
8082 // mirror that here as well as setting the flag, so a test asserting
8083 // "no wakeup is pending" really exercises what the app would observe.
8084 let wakeup_trigger_for_fn = wakeup.clone();
8085 let close_handle = NetworkThreadCloseHandle::new(
8086 Box::new(move |timeout: Duration| {
8087 signal_close_slot.store(timeout.as_millis() as i64, Ordering::Release);
8088 }),
8089 Box::new(move || {
8090 wakeup_flag.store(true, Ordering::Release);
8091 wakeup_trigger_for_fn.wakeup();
8092 }),
8093 join_handle,
8094 );
8095
8096 let (bg_event_tx, bg_rx) = mpsc::unbounded_channel::<BackgroundEventEnvelope>();
8097 let interceptors = Arc::new(Mutex::new(ConsumerInterceptors::<Vec<u8>, Vec<u8>>::new(Vec::new())));
8098 let offset_commit_callback_invoker =
8099 Arc::new(OffsetCommitCallbackInvoker::<Vec<u8>, Vec<u8>>::new(ConsumerInterceptors::<
8100 Vec<u8>,
8101 Vec<u8>,
8102 >::new(
8103 invoker_interceptors
8104 )));
8105 let deserializers = Arc::new(Deserializers::<Vec<u8>, Vec<u8>>::new(
8106 Box::new(TestBytesDeserializer),
8107 Box::new(TestBytesDeserializer),
8108 ));
8109 let rebalance_listener_invoker = ConsumerRebalanceListenerInvoker::new(Arc::clone(&subs));
8110
8111 let fetch_buffer = Arc::new(FetchBuffer::new());
8112 let fetch_config = crate::consumer::internals::FetchConfig::new(
8113 1,
8114 50 * 1024 * 1024,
8115 500,
8116 1024 * 1024,
8117 500,
8118 true,
8119 "",
8120 IsolationLevel::ReadUncommitted,
8121 );
8122 let time: Arc<dyn Time> = Arc::new(crate::common::utils::SystemTime);
8123 let (metrics, fetch_metrics_manager) =
8124 AsyncKafkaConsumer::<Vec<u8>, Vec<u8>>::create_fetch_metrics_manager(&config, Arc::clone(&time));
8125 let kafka_consumer_metrics = Arc::new(KafkaConsumerMetrics::new(Arc::clone(&metrics)));
8126 let async_consumer_metrics = Arc::new(AsyncConsumerMetrics::new(
8127 Arc::clone(&metrics),
8128 crate::consumer::internals::ConsumerUtils::CONSUMER_METRIC_GROUP,
8129 ));
8130 let rebalance_callback_metrics_manager = Arc::new(RebalanceCallbackMetricsManager::new(&metrics));
8131 let background_event_queue_size = Arc::new(AtomicI64::new(0));
8132 let fetch_collector = Arc::new(FetchCollector::<Vec<u8>, Vec<u8>>::new(
8133 Arc::clone(&metadata),
8134 Arc::clone(&subs),
8135 fetch_config,
8136 deserializers,
8137 Arc::clone(&fetch_metrics_manager),
8138 Arc::new(crate::common::utils::SystemTime),
8139 ));
8140
8141 // Build the state-notifier + shared slots once (Phase-12
8142 // Issue 2): the test fixture mirrors the production ctor by
8143 // constructing the slots locally and threading the same Arcs
8144 // through `state_notifier` and the components struct. Tests
8145 // that need the listener registered on a membership manager
8146 // call `consumer.state_notifier()` and pass the Arc to
8147 // `AbstractMembershipManager::register_state_listener`.
8148 //
8149 // Seed the group-metadata slot through the SAME helper the
8150 // production ctor calls (definition-of-done.md #12): a fixture
8151 // consumer with a `group.id` starts with UNKNOWN generation / member
8152 // id plus the configured `group.instance.id`, exactly as production
8153 // does; a groupless fixture starts with an empty slot.
8154 let group_metadata_slot: Arc<Mutex<Option<Arc<ConsumerGroupMetadataImpl>>>> =
8155 Arc::new(Mutex::new(initialize_group_metadata(group_id, group_instance_id)));
8156 let group_assignment_snapshot_slot: Arc<Mutex<HashSet<TopicPartition>>> = Arc::new(Mutex::new(HashSet::new()));
8157 let state_notifier = Arc::new(ConsumerStateNotifier::new(
8158 Arc::clone(&group_metadata_slot),
8159 Arc::clone(&group_assignment_snapshot_slot),
8160 Arc::new(AtomicBool::new(false)),
8161 ));
8162
8163 let positions_validator = Arc::new(PositionsValidator::new(Arc::clone(&subs), Arc::clone(&metadata)));
8164 let components = AsyncKafkaConsumerComponents {
8165 config,
8166 client_id,
8167 group_id: group_id.map(str::to_string),
8168 positions_validator,
8169 subscriptions: Arc::clone(&subs),
8170 metadata,
8171 request_managers,
8172 background_event_rx: bg_rx,
8173 application_event_handler: app_handler,
8174 completable_event_reaper: reaper,
8175 max_time_to_wait_ms: max_time,
8176 wakeup_trigger: wakeup,
8177 network_thread_close: close_handle,
8178 fetch_buffer,
8179 fetch_collector,
8180 metrics,
8181 kafka_consumer_metrics,
8182 async_consumer_metrics,
8183 // Java's test constructor takes `mock(FetchMetricsManager.class)` /
8184 // `mock(RebalanceCallbackMetricsManager.class)` here; the real
8185 // managers over the fixture's registry stand in for the mocks.
8186 fetch_metrics_manager,
8187 rebalance_callback_metrics_manager,
8188 background_event_queue_size,
8189 rebalance_listener_invoker,
8190 offset_commit_callback_invoker,
8191 interceptors,
8192 isolation_level: IsolationLevel::ReadUncommitted,
8193 time,
8194 group_metadata: group_metadata_slot,
8195 group_assignment_snapshot: group_assignment_snapshot_slot,
8196 state_notifier,
8197 };
8198 (
8199 AsyncKafkaConsumer::<Vec<u8>, Vec<u8>>::with_components(components),
8200 ConsumerTestHandles {
8201 app_event_rx,
8202 bg_event_tx,
8203 subscriptions: subs,
8204 signal_close_timeout_ms,
8205 bg_wakeup_called: wakeup_called,
8206 event_notify,
8207 },
8208 )
8209 }
8210
8211 /// Builds a consumer along with the test-side channel handles needed to
8212 /// act as the bg task during a test. Thin wrapper over
8213 /// [`make_test_consumer_with_group`] with the default `group.id`
8214 /// `"test-group"` and no `group.instance.id`.
8215 fn make_test_consumer_with_channels() -> (AsyncKafkaConsumer<Vec<u8>, Vec<u8>>, ConsumerTestHandles) {
8216 make_test_consumer_with_group(Some("test-group"), None)
8217 }
8218
8219 /// [`make_test_consumer_with_channels`] whose `OffsetCommitCallbackInvoker`
8220 /// runs `invoker_interceptors`.
8221 fn make_test_consumer_with_channels_and_invoker_interceptors(
8222 invoker_interceptors: Vec<Box<dyn crate::consumer::ConsumerInterceptor<Vec<u8>, Vec<u8>>>>,
8223 ) -> (AsyncKafkaConsumer<Vec<u8>, Vec<u8>>, ConsumerTestHandles) {
8224 make_test_consumer_with_group_and_invoker_interceptors(Some("test-group"), None, invoker_interceptors)
8225 }
8226
8227 /// [`make_test_consumer_with_group_and_invoker_interceptors`] with no
8228 /// interceptors.
8229 fn make_test_consumer_with_group(
8230 group_id: Option<&str>,
8231 group_instance_id: Option<&str>,
8232 ) -> (AsyncKafkaConsumer<Vec<u8>, Vec<u8>>, ConsumerTestHandles) {
8233 make_test_consumer_with_group_and_invoker_interceptors(group_id, group_instance_id, Vec::new())
8234 }
8235
8236 /// Backwards-compat alias for the existing state-read tests.
8237 fn make_test_consumer() -> AsyncKafkaConsumer<Vec<u8>, Vec<u8>> {
8238 make_test_consumer_with_channels().0
8239 }
8240
8241 /// Construct a test consumer with `group_id=None` from the start
8242 /// (Issue 26: mirrors Java's `assignor-only` consumer constructed
8243 /// without `group.id`). Delegates to [`make_test_consumer_with_group`]
8244 /// with `None, None`, so the `group_id` field, the config, and the
8245 /// seeded group-metadata slot (empty, per `initialize_group_metadata`)
8246 /// are all consistent from the start — no post-construction mutation.
8247 fn make_test_consumer_without_group_id() -> (AsyncKafkaConsumer<Vec<u8>, Vec<u8>>, ConsumerTestHandles) {
8248 make_test_consumer_with_group(None, None)
8249 }
8250
8251 #[tokio::test]
8252 async fn assignment_is_empty_before_subscribe() {
8253 let consumer = make_test_consumer();
8254 assert!(consumer.assignment().is_empty());
8255 }
8256
8257 #[tokio::test]
8258 async fn subscription_is_empty_before_subscribe() {
8259 let consumer = make_test_consumer();
8260 assert!(consumer.subscription().is_empty());
8261 }
8262
8263 #[tokio::test]
8264 async fn paused_is_empty_before_subscribe() {
8265 let consumer = make_test_consumer();
8266 assert!(consumer.paused().is_empty());
8267 }
8268
8269 #[tokio::test]
8270 async fn client_id_returns_config_value() {
8271 let consumer = make_test_consumer();
8272 assert_eq!(consumer.client_id(), "test-client");
8273 }
8274
8275 #[tokio::test]
8276 async fn group_metadata_returns_seeded_placeholders_for_freshly_constructed_consumer() {
8277 // The production ctor seeds the group-metadata slot at construction
8278 // (Issue C1; Java's `org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer` constructor →
8279 // `AsyncKafkaConsumer#initializeGroupMetadata(String,Optional)`), so a freshly constructed consumer with a
8280 // `group.id` — before any heartbeat — already returns the configured group id with the
8281 // UNKNOWN generation / member-id placeholders, NOT a groupless stub.
8282 let consumer = make_test_consumer();
8283 let meta = consumer.group_metadata();
8284 assert_eq!(meta.group_id(), "test-group");
8285 assert_eq!(meta.generation_id(), -1); // JoinGroupRequest.UNKNOWN_GENERATION_ID
8286 assert_eq!(meta.member_id(), ""); // JoinGroupRequest.UNKNOWN_MEMBER_ID
8287 assert_eq!(meta.group_instance_id(), None);
8288 }
8289
8290 /// Java parity: `memberStateListener.onMemberEpochUpdated` →
8291 /// `updateGroupMetadata` populates the cached metadata
8292 /// (`AsyncKafkaConsumer#memberStateListener`, `AsyncKafkaConsumer#updateGroupMetadata`).
8293 #[tokio::test]
8294 async fn state_notifier_populates_group_metadata_on_epoch_update() {
8295 let consumer = make_test_consumer();
8296 let notifier = consumer.state_notifier();
8297 // Pre-condition: the slot is seeded with the UNKNOWN generation id
8298 // (-1) at construction (Issue C1), not empty.
8299 assert_eq!(consumer.group_metadata().generation_id(), -1);
8300
8301 notifier.on_member_epoch_updated(Some(42), "member-id-xyz");
8302
8303 let meta = consumer.group_metadata();
8304 assert_eq!(meta.group_id(), "test-group");
8305 assert_eq!(meta.generation_id(), 42);
8306 assert_eq!(meta.member_id(), "member-id-xyz");
8307 }
8308
8309 /// Java: `memberEpoch.ifPresent(...)` short-circuits when None —
8310 /// the cache is NOT cleared. Mirrors Java's
8311 /// `updateGroupMetadata(Optional.empty(), …)` no-op behavior.
8312 #[tokio::test]
8313 async fn state_notifier_with_none_epoch_does_not_modify_cache() {
8314 let consumer = make_test_consumer();
8315 let notifier = consumer.state_notifier();
8316 notifier.on_member_epoch_updated(Some(7), "m1");
8317 notifier.on_member_epoch_updated(None, "m1");
8318 // None did not overwrite — the epoch 7 entry survives.
8319 assert_eq!(consumer.group_metadata().generation_id(), 7);
8320 }
8321
8322 /// Java parity: `memberStateListener.onGroupAssignmentUpdated` →
8323 /// `setGroupAssignmentSnapshot(partitions)` updates the snapshot
8324 /// (`AsyncKafkaConsumer#memberStateListener`, `AsyncKafkaConsumer#setGroupAssignmentSnapshot`).
8325 #[tokio::test]
8326 async fn state_notifier_updates_group_assignment_snapshot() {
8327 let consumer = make_test_consumer();
8328 let notifier = consumer.state_notifier();
8329 let tp0 = TopicPartition::new("t".to_string(), 0);
8330 let tp1 = TopicPartition::new("t".to_string(), 1);
8331 let mut set: HashSet<TopicPartition> = HashSet::new();
8332 set.insert(tp0.clone());
8333 set.insert(tp1.clone());
8334
8335 notifier.on_group_assignment_updated(&set);
8336
8337 let snap = consumer.group_assignment_snapshot.lock().unwrap().clone();
8338 assert_eq!(snap.len(), 2);
8339 assert!(snap.contains(&tp0));
8340 assert!(snap.contains(&tp1));
8341 }
8342
8343 /// Phase-12 Issue 2 regression: when the consumer's `state_notifier`
8344 /// Arc is registered on a `ConsumerMembershipManager` listener list
8345 /// (the production wiring) and a heartbeat-response-style member-epoch
8346 /// update is dispatched on the listener side, the app-side
8347 /// `group_metadata()` observes the new epoch.
8348 ///
8349 /// Pre-fix this test would fail because the production ctor built two
8350 /// separate notifiers — one registered on the membership manager, one
8351 /// reachable through `consumer.state_notifier()` — and writes to one
8352 /// did not reach the other's backing slot.
8353 #[tokio::test]
8354 async fn issue_2_state_notifier_writes_visible_through_consumer_after_registration() {
8355 use crate::consumer::internals::MemberStateListener;
8356
8357 let consumer = make_test_consumer();
8358 let notifier = consumer.state_notifier();
8359
8360 // Mimic the production ctor's
8361 // `membership.abstract_mm.register_state_listener(state_notifier)`
8362 // call: take the Arc out of the consumer, hand it to the
8363 // listener-registration site, and dispatch a fake
8364 // `on_member_epoch_updated`. The app side must see the change.
8365 //
8366 // `register_state_listener` takes `Arc<dyn MemberStateListener>`.
8367 // We don't need a real `ConsumerMembershipManager` here; the
8368 // contract is "register_state_listener obtains an
8369 // Arc<dyn MemberStateListener> and may invoke its methods at any
8370 // time" — the test substitutes for that registration site by
8371 // calling the trait method directly on the Arc-erased listener.
8372 let listener: Arc<dyn MemberStateListener> = notifier;
8373 listener.on_member_epoch_updated(Some(99), "member-from-listener");
8374
8375 // App-side observes the change through the same shared slot the
8376 // listener wrote into — proving the `state_notifier` and
8377 // `group_metadata` Arcs are wired as a single source of truth
8378 // (Java `org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer#groupMetadata`,
8379 // `AsyncKafkaConsumer#memberStateListener`).
8380 let meta = consumer.group_metadata();
8381 assert_eq!(meta.generation_id(), 99);
8382 assert_eq!(meta.member_id(), "member-from-listener");
8383 }
8384
8385 #[tokio::test]
8386 async fn wakeup_triggers_token_cancellation_and_bg_wakeup() {
8387 let consumer = make_test_consumer();
8388 let token = consumer.wakeup_trigger.current_token();
8389 assert!(!token.is_cancelled());
8390 consumer.wakeup();
8391 assert!(token.is_cancelled(), "wakeup() must cancel the current token");
8392 }
8393
8394 /// A [`ConsumerHandle`] obtained from the consumer fires the SAME
8395 /// wakeup state as `wakeup()` — proving the shareable handle is a
8396 /// faithful, `Send`-able stand-in for the cross-task `wakeup()`
8397 /// pattern (the safe replacement for the deleted unsafe test helper).
8398 #[tokio::test]
8399 async fn handle_wakeup_cancels_current_token() {
8400 let consumer = make_test_consumer();
8401 let token = consumer.wakeup_trigger.current_token();
8402 assert!(!token.is_cancelled());
8403
8404 // The handle is moved into another task — no reference to the
8405 // consumer crosses the task boundary.
8406 let handle = consumer.handle();
8407 let joined = tokio::spawn(async move {
8408 handle.wakeup();
8409 });
8410 joined.await.expect("waker task");
8411
8412 assert!(token.is_cancelled(), "handle().wakeup() must cancel the current token");
8413 }
8414
8415 /// Phase 41c / §31 deadlock regression (blocker b): a reentrant
8416 /// `ConsumerHandle` op submitted from another task (standing in for a
8417 /// rebalance-listener body) routes an `ApplicationEvent` through the bg
8418 /// pipeline and completes — proving the handle's no-drain await is NOT
8419 /// frozen. The op is `pause`, which (like every bg-routed handle op)
8420 /// goes through `submit_and_await`. We act as the bg task by reading
8421 /// the app-event channel and completing the event's handle; the handle
8422 /// op must then return promptly.
8423 #[tokio::test]
8424 async fn handle_reentrant_op_completes_through_bg_pipeline() {
8425 let (consumer, mut handles) = make_test_consumer_with_channels();
8426 let handle = consumer.handle();
8427 // The test body stands in for the listener: the consumer is "running
8428 // a callback" for as long as this marker lives.
8429 let _in_callback = consumer.enter_callback_scope();
8430
8431 let tp = TopicPartition::new("t".to_string(), 0);
8432 // Submit the reentrant op from another task — exactly how a
8433 // captured handle is used from inside a listener.
8434 let op = tokio::spawn(async move { handle.pause(std::slice::from_ref(&tp)).await });
8435
8436 // Act as the bg task: pull the PausePartitions envelope and
8437 // complete it. (The real bg loop keeps spinning during a callback
8438 // after Phase 41b, so this event is serviced rather than stranded.)
8439 let env = handles.app_event_rx.recv().await.expect("PausePartitions envelope must arrive");
8440 match env.event {
8441 ApplicationEvent::PausePartitions { handle, partitions } => {
8442 assert_eq!(partitions.len(), 1);
8443 handle.complete(());
8444 },
8445 other => panic!("expected PausePartitions, got {}", other.type_name()),
8446 }
8447
8448 op.await
8449 .expect("handle op task ok")
8450 .expect("reentrant pause completes — no deadlock");
8451 // The consumer must outlive the handle/op (shared Arc state).
8452 drop(consumer);
8453 }
8454
8455 // ─── The ConsumerHandle gate: async ops only inside a callback ───
8456
8457 /// Asserts a handle op made outside a callback was rejected with Java's
8458 /// `ConcurrentModificationException` message and submitted nothing to
8459 /// the bg task.
8460 fn assert_rejected_outside_callback<T: std::fmt::Debug>(
8461 label: &str,
8462 result: Result<T, Error>,
8463 app_event_rx: &mut mpsc::UnboundedReceiver<ApplicationEventEnvelope>,
8464 ) {
8465 let err = result.expect_err(label);
8466 assert!(
8467 err.is_local_concurrent_modification_error(),
8468 "{label}: unexpected error {err:?}"
8469 );
8470 assert_eq!(err.message(), HANDLE_OUTSIDE_CALLBACK_MSG, "{label}");
8471 assert!(
8472 app_event_rx.try_recv().is_err(),
8473 "{label}: a rejected op must not submit an application event"
8474 );
8475 }
8476
8477 /// Completes every `PausePartitions` / `ResumePartitions` event, standing
8478 /// in for the bg task while a handle op is in flight.
8479 fn spawn_pause_resume_completer(
8480 mut app_event_rx: mpsc::UnboundedReceiver<ApplicationEventEnvelope>,
8481 ) -> JoinHandle<()> {
8482 tokio::spawn(async move {
8483 while let Some(env) = app_event_rx.recv().await {
8484 match env.event {
8485 ApplicationEvent::PausePartitions { handle, .. }
8486 | ApplicationEvent::ResumePartitions { handle, .. } => {
8487 handle.complete(());
8488 },
8489 other => panic!("unexpected event {}", other.type_name()),
8490 }
8491 }
8492 })
8493 }
8494
8495 /// Every async operation of a [`ConsumerHandle`] obtained from an
8496 /// `AsyncKafkaConsumer` fails with Java's `ConcurrentModificationException`
8497 /// when no `ConsumerRebalanceListener` / `OffsetCommitCallback` is
8498 /// running, before anything reaches the bg task: the handle is the
8499 /// in-callback path only (`Consumer::handle` docs). This is the Rust
8500 /// counterpart of Java's `acquire()` for the one path `&mut self` cannot
8501 /// guard.
8502 #[tokio::test]
8503 async fn handle_async_ops_outside_a_callback_fail_with_concurrent_modification() {
8504 let (consumer, mut handles) = make_test_consumer_with_channels();
8505 let handle = consumer.handle();
8506 let rx = &mut handles.app_event_rx;
8507 let tp = TopicPartition::new("t".to_string(), 0);
8508 let tps = std::slice::from_ref(&tp);
8509 let timeout = Duration::from_secs(5);
8510 let offsets = singleton_offsets(tp.clone(), 1);
8511 let cb = || -> Arc<dyn crate::consumer::OffsetCommitCallback> { Arc::new(NoopCallback) };
8512
8513 assert_rejected_outside_callback("seek_with_offset", handle.seek_with_offset(tp.clone(), 1).await, rx);
8514 assert_rejected_outside_callback(
8515 "seek_with_offset_and_metadata",
8516 handle
8517 .seek_with_offset_and_metadata(tp.clone(), OffsetAndMetadata::new(1).expect("offset"))
8518 .await,
8519 rx,
8520 );
8521 assert_rejected_outside_callback("seek_to_beginning", handle.seek_to_beginning(tps).await, rx);
8522 assert_rejected_outside_callback("seek_to_end", handle.seek_to_end(tps).await, rx);
8523 assert_rejected_outside_callback("pause", handle.pause(tps).await, rx);
8524 assert_rejected_outside_callback("resume", handle.resume(tps).await, rx);
8525 assert_rejected_outside_callback("position", handle.position(&tp).await, rx);
8526 assert_rejected_outside_callback("position_with_timeout", handle.position_with_timeout(&tp, timeout).await, rx);
8527 assert_rejected_outside_callback("current_lag", handle.current_lag(&tp).await, rx);
8528 assert_rejected_outside_callback("committed", handle.committed(tps).await, rx);
8529 assert_rejected_outside_callback(
8530 "committed_with_timeout",
8531 handle.committed_with_timeout(tps, timeout).await,
8532 rx,
8533 );
8534 assert_rejected_outside_callback("partitions_for", handle.partitions_for("t").await, rx);
8535 assert_rejected_outside_callback(
8536 "partitions_for_with_timeout",
8537 handle.partitions_for_with_timeout("t", timeout).await,
8538 rx,
8539 );
8540 assert_rejected_outside_callback("list_topics", handle.list_topics().await, rx);
8541 assert_rejected_outside_callback(
8542 "list_topics_with_timeout",
8543 handle.list_topics_with_timeout(timeout).await,
8544 rx,
8545 );
8546 assert_rejected_outside_callback("beginning_offsets", handle.beginning_offsets(tps).await, rx);
8547 assert_rejected_outside_callback(
8548 "beginning_offsets_with_timeout",
8549 handle.beginning_offsets_with_timeout(tps, timeout).await,
8550 rx,
8551 );
8552 assert_rejected_outside_callback("end_offsets", handle.end_offsets(tps).await, rx);
8553 assert_rejected_outside_callback(
8554 "end_offsets_with_timeout",
8555 handle.end_offsets_with_timeout(tps, timeout).await,
8556 rx,
8557 );
8558 let timestamps = HashMap::from([(tp.clone(), 0_i64)]);
8559 assert_rejected_outside_callback("offsets_for_times", handle.offsets_for_times(timestamps.clone()).await, rx);
8560 assert_rejected_outside_callback(
8561 "offsets_for_times_with_timeout",
8562 handle.offsets_for_times_with_timeout(timestamps, timeout).await,
8563 rx,
8564 );
8565 assert_rejected_outside_callback("commit_sync", handle.commit_sync().await, rx);
8566 assert_rejected_outside_callback(
8567 "commit_sync_with_timeout",
8568 handle.commit_sync_with_timeout(timeout).await,
8569 rx,
8570 );
8571 assert_rejected_outside_callback(
8572 "commit_sync_with_offsets",
8573 handle.commit_sync_with_offsets(offsets.clone()).await,
8574 rx,
8575 );
8576 assert_rejected_outside_callback(
8577 "commit_sync_with_offsets_timeout",
8578 handle.commit_sync_with_offsets_timeout(offsets.clone(), timeout).await,
8579 rx,
8580 );
8581 assert_rejected_outside_callback("commit_async", handle.commit_async().await, rx);
8582 assert_rejected_outside_callback(
8583 "commit_async_with_callback",
8584 handle.commit_async_with_callback(cb()).await,
8585 rx,
8586 );
8587 assert_rejected_outside_callback(
8588 "commit_async_with_offsets_callback",
8589 handle.commit_async_with_offsets_callback(offsets, cb()).await,
8590 rx,
8591 );
8592 assert!(
8593 consumer.commit_state.last_pending_async_commit().is_none(),
8594 "a rejected commit_async must not register a pending async commit"
8595 );
8596 }
8597
8598 /// The handle's non-waiting methods are not gated: `wakeup()` is the
8599 /// cross-thread path Java's `wakeup()` is, and the state reads do not
8600 /// touch anything the owning thread mutates unprotected.
8601 #[tokio::test]
8602 async fn handle_wakeup_and_sync_reads_work_outside_a_callback() {
8603 let (consumer, _handles) = make_test_consumer_with_channels();
8604 let handle = consumer.handle();
8605 let tp = TopicPartition::new("t".to_string(), 0);
8606 consumer
8607 .subscriptions
8608 .lock()
8609 .unwrap()
8610 .assign_from_user(HashSet::from([tp.clone()]))
8611 .unwrap();
8612
8613 assert_eq!(handle.assignment(), HashSet::from([tp.clone()]));
8614 assert!(handle.paused().is_empty());
8615 assert_eq!(handle.group_metadata().group_id(), consumer.group_metadata().group_id());
8616
8617 let token = consumer.wakeup_trigger.current_token();
8618 handle.wakeup();
8619 assert!(token.is_cancelled(), "wakeup() is callable from any thread at any time");
8620 }
8621
8622 /// A listener fired by the consumer may use the handle: inside
8623 /// `on_partitions_revoked` the op goes through the bg pipeline and
8624 /// completes; as soon as the listener has returned the same op is
8625 /// rejected again, because the consumer is no longer running a
8626 /// callback.
8627 #[tokio::test]
8628 async fn handle_async_ops_work_inside_a_rebalance_listener_and_fail_after_it_returns() {
8629 use crate::consumer::internals::ConsumerRebalanceListenerMethodName;
8630 use async_trait::async_trait;
8631 use tokio::sync::oneshot;
8632
8633 struct PausingListener {
8634 handle: ConsumerHandle,
8635 outcome: std::sync::Mutex<Option<Result<(), Error>>>,
8636 }
8637 #[async_trait]
8638 impl ConsumerRebalanceListener for PausingListener {
8639 async fn on_partitions_revoked(&self, partitions: &[TopicPartition]) -> Result<(), Error> {
8640 *self.outcome.lock().unwrap() = Some(self.handle.pause(partitions).await);
8641 Ok(())
8642 }
8643 async fn on_partitions_assigned(&self, _: &[TopicPartition]) -> Result<(), Error> {
8644 Ok(())
8645 }
8646 }
8647
8648 let (mut consumer, handles) = make_test_consumer_with_channels();
8649 let ConsumerTestHandles { app_event_rx, bg_event_tx, .. } = handles;
8650 let handle = consumer.handle();
8651 let listener = Arc::new(PausingListener { handle: handle.clone(), outcome: std::sync::Mutex::new(None) });
8652 *consumer.rebalance_listener.lock().unwrap() =
8653 Some(Arc::clone(&listener) as Arc<dyn ConsumerRebalanceListener>);
8654 let completer = spawn_pause_resume_completer(app_event_rx);
8655 let tp = TopicPartition::new("t".to_string(), 0);
8656
8657 let (ack_tx, ack_rx) = oneshot::channel::<Result<(), Error>>();
8658 bg_event_tx
8659 .send(BackgroundEventEnvelope {
8660 event: BackgroundEvent::PartitionsRemoved {
8661 method_name: ConsumerRebalanceListenerMethodName::OnPartitionsRevoked,
8662 partitions: vec![tp.clone()],
8663 ack: ack_tx,
8664 ack_through_application_events: false,
8665 },
8666 enqueued_ms: 0,
8667 })
8668 .expect("bg event sent");
8669 consumer.process_background_events().await.expect("listener ran");
8670 ack_rx.await.expect("ack").expect("listener returned Ok");
8671 match listener.outcome.lock().unwrap().take() {
8672 Some(Ok(())) => {},
8673 other => panic!("the handle op inside the listener must complete, got {other:?}"),
8674 }
8675
8676 // The listener has returned: the consumer runs no callback.
8677 let err = handle.pause(std::slice::from_ref(&tp)).await.expect_err("outside the listener");
8678 assert!(err.is_local_concurrent_modification_error(), "unexpected error {err:?}");
8679 completer.abort();
8680 }
8681
8682 /// `ConsumerHandle::current_lag` inside `on_partitions_revoked` submits a
8683 /// `CurrentLag` event carrying the consumer's isolation level and returns
8684 /// what the bg task answers, as the owning consumer's `current_lag` does;
8685 /// after the listener returns it is rejected like every async handle op.
8686 #[tokio::test]
8687 async fn handle_current_lag_works_inside_a_rebalance_listener() {
8688 use crate::consumer::internals::ConsumerRebalanceListenerMethodName;
8689 use async_trait::async_trait;
8690 use tokio::sync::oneshot;
8691
8692 struct LagListener {
8693 handle: ConsumerHandle,
8694 lags: std::sync::Mutex<Vec<Result<Option<i64>, Error>>>,
8695 }
8696 #[async_trait]
8697 impl ConsumerRebalanceListener for LagListener {
8698 async fn on_partitions_revoked(&self, partitions: &[TopicPartition]) -> Result<(), Error> {
8699 for partition in partitions {
8700 let lag = self.handle.current_lag(partition).await;
8701 self.lags.lock().unwrap().push(lag);
8702 }
8703 Ok(())
8704 }
8705 async fn on_partitions_assigned(&self, _: &[TopicPartition]) -> Result<(), Error> {
8706 Ok(())
8707 }
8708 }
8709
8710 let (mut consumer, handles) = make_test_consumer_with_channels();
8711 let ConsumerTestHandles { mut app_event_rx, bg_event_tx, .. } = handles;
8712 let isolation_level = consumer.isolation_level;
8713 let handle = consumer.handle();
8714 let listener = Arc::new(LagListener { handle: handle.clone(), lags: std::sync::Mutex::new(Vec::new()) });
8715 *consumer.rebalance_listener.lock().unwrap() =
8716 Some(Arc::clone(&listener) as Arc<dyn ConsumerRebalanceListener>);
8717 let known = TopicPartition::new("t".to_string(), 0);
8718 let unknown = TopicPartition::new("t".to_string(), 1);
8719 let completer = {
8720 let known = known.clone();
8721 tokio::spawn(async move {
8722 while let Some(env) = app_event_rx.recv().await {
8723 match env.event {
8724 ApplicationEvent::CurrentLag { handle, partition, isolation_level: level } => {
8725 assert_eq!(isolation_level, level, "the handle carries the consumer's isolation level");
8726 handle.complete((partition == known).then_some(7));
8727 },
8728 other => panic!("unexpected event {}", other.type_name()),
8729 }
8730 }
8731 })
8732 };
8733
8734 let (ack_tx, ack_rx) = oneshot::channel::<Result<(), Error>>();
8735 bg_event_tx
8736 .send(BackgroundEventEnvelope {
8737 event: BackgroundEvent::PartitionsRemoved {
8738 method_name: ConsumerRebalanceListenerMethodName::OnPartitionsRevoked,
8739 partitions: vec![known.clone(), unknown],
8740 ack: ack_tx,
8741 ack_through_application_events: false,
8742 },
8743 enqueued_ms: 0,
8744 })
8745 .expect("bg event sent");
8746 consumer.process_background_events().await.expect("listener ran");
8747 ack_rx.await.expect("ack").expect("listener returned Ok");
8748 let lags: Vec<Option<i64>> = std::mem::take(&mut *listener.lags.lock().unwrap())
8749 .into_iter()
8750 .map(|lag| lag.expect("current_lag inside the listener"))
8751 .collect();
8752 assert_eq!(vec![Some(7), None], lags, "the known lag, then an unknown one");
8753
8754 // The listener has returned: the consumer runs no callback.
8755 let err = handle.current_lag(&known).await.expect_err("outside the listener");
8756 assert!(err.is_local_concurrent_modification_error(), "unexpected error {err:?}");
8757 completer.abort();
8758 }
8759
8760 /// PR #263 review (Copilot): a leave's `on_partitions_revoked` returns its
8761 /// ack through the application-event queue, behind every event the
8762 /// listener submitted — here a `ConsumerHandle` seek whose wait the
8763 /// listener abandoned with `tokio::time::timeout` — as Java's
8764 /// `invokeRebalanceCallbackAndNotifyBackgroundThread` enqueues
8765 /// `ConsumerRebalanceListenerCallbackCompletedEvent` after the listener
8766 /// returns. The ack is not sent directly, which would let the bg task
8767 /// apply the leave before the still-queued seek. A reconcile or release
8768 /// callback still gets its ack directly, with nothing enqueued.
8769 #[tokio::test]
8770 async fn leave_callback_ack_is_enqueued_after_events_the_listener_submitted() {
8771 use crate::consumer::internals::ConsumerRebalanceListenerMethodName;
8772 use async_trait::async_trait;
8773 use tokio::sync::oneshot;
8774
8775 struct AbandoningListener {
8776 handle: ConsumerHandle,
8777 }
8778 #[async_trait]
8779 impl ConsumerRebalanceListener for AbandoningListener {
8780 async fn on_partitions_revoked(&self, partitions: &[TopicPartition]) -> Result<(), Error> {
8781 // The seek is enqueued on the first poll; the zero timeout
8782 // then abandons the wait with the event still queued.
8783 let wait =
8784 tokio::time::timeout(Duration::ZERO, self.handle.seek_with_offset(partitions[0].clone(), 42));
8785 assert!(wait.await.is_err(), "the listener stopped waiting for the seek");
8786 Ok(())
8787 }
8788 async fn on_partitions_assigned(&self, _: &[TopicPartition]) -> Result<(), Error> {
8789 Ok(())
8790 }
8791 }
8792
8793 let (mut consumer, handles) = make_test_consumer_with_channels();
8794 let ConsumerTestHandles { mut app_event_rx, bg_event_tx, .. } = handles;
8795 let listener = Arc::new(AbandoningListener { handle: consumer.handle() });
8796 *consumer.rebalance_listener.lock().unwrap() = Some(listener as Arc<dyn ConsumerRebalanceListener>);
8797 let tp = TopicPartition::new("t".to_string(), 0);
8798
8799 let send_callback = |ack_through_application_events: bool| {
8800 let (ack_tx, ack_rx) = oneshot::channel::<Result<(), Error>>();
8801 bg_event_tx
8802 .send(BackgroundEventEnvelope {
8803 event: BackgroundEvent::PartitionsRemoved {
8804 method_name: ConsumerRebalanceListenerMethodName::OnPartitionsRevoked,
8805 partitions: vec![tp.clone()],
8806 ack: ack_tx,
8807 ack_through_application_events,
8808 },
8809 enqueued_ms: 0,
8810 })
8811 .expect("bg event sent");
8812 ack_rx
8813 };
8814
8815 // A leave's callback.
8816 let mut ack_rx = send_callback(true);
8817 consumer.process_background_events().await.expect("listener ran");
8818 assert!(
8819 matches!(ack_rx.try_recv(), Err(oneshot::error::TryRecvError::Empty)),
8820 "a leave's ack must not be sent directly"
8821 );
8822 match app_event_rx.try_recv().expect("the listener's seek is queued").event {
8823 ApplicationEvent::SeekUnvalidated { partition, offset, .. } => {
8824 assert_eq!((partition, offset), (tp.clone(), 42));
8825 },
8826 other => panic!("expected the listener's seek first, got {}", other.type_name()),
8827 }
8828 match app_event_rx.try_recv().expect("the callback-completed event is queued").event {
8829 ApplicationEvent::ConsumerRebalanceListenerCallbackCompleted { method_name, error, ack } => {
8830 assert_eq!(method_name, ConsumerRebalanceListenerMethodName::OnPartitionsRevoked);
8831 assert!(error.is_none(), "the listener returned Ok");
8832 // The processor completes the carried future.
8833 ack.send(Ok(())).expect("receiver alive");
8834 assert!(matches!(ack_rx.try_recv(), Ok(Ok(()))), "the event carries this callback's ack");
8835 },
8836 other => panic!(
8837 "expected the callback-completed event after the seek, got {}",
8838 other.type_name()
8839 ),
8840 }
8841 assert!(app_event_rx.try_recv().is_err(), "nothing else queued");
8842
8843 // A reconcile / release callback: the ack is sent directly.
8844 let mut ack_rx = send_callback(false);
8845 consumer.process_background_events().await.expect("listener ran");
8846 assert!(matches!(ack_rx.try_recv(), Ok(Ok(()))), "the ack is sent directly");
8847 assert!(
8848 matches!(
8849 app_event_rx.try_recv().expect("the listener's seek is queued").event,
8850 ApplicationEvent::SeekUnvalidated { .. }
8851 ),
8852 "only the listener's seek is queued"
8853 );
8854 assert!(app_event_rx.try_recv().is_err(), "no callback-completed event for a direct ack");
8855 }
8856
8857 /// PR #263 review (Critic pr2-c2 finding 1): when the caller's future is
8858 /// dropped while a leave's `on_partitions_revoked` is still running, the
8859 /// ack is not dropped unsent: the callback-completed event is still
8860 /// enqueued behind the op the listener had submitted, carrying an
8861 /// `Error::Interrupt`. Java's analog is a listener cut short by an
8862 /// `InterruptException`, which `AsyncKafkaConsumer.invokeRebalanceCallbacks`
8863 /// catches into the event's error before
8864 /// `invokeRebalanceCallbackAndNotifyBackgroundThread` enqueues the
8865 /// completed event. Without it the bg task would see a closed ack and
8866 /// apply the leave ahead of the listener's still-queued seek.
8867 #[tokio::test]
8868 async fn leave_callback_ack_is_enqueued_after_the_listener_ops_when_the_caller_is_cancelled() {
8869 use crate::consumer::internals::ConsumerRebalanceListenerMethodName;
8870 use async_trait::async_trait;
8871 use std::future::Future;
8872 use std::task::Poll;
8873 use tokio::sync::oneshot;
8874
8875 struct SeekingListener {
8876 handle: ConsumerHandle,
8877 }
8878 #[async_trait]
8879 impl ConsumerRebalanceListener for SeekingListener {
8880 async fn on_partitions_revoked(&self, partitions: &[TopicPartition]) -> Result<(), Error> {
8881 // Nothing completes the seek, so the listener is still
8882 // awaiting it when the caller's future is dropped.
8883 self.handle.seek_with_offset(partitions[0].clone(), 42).await
8884 }
8885 async fn on_partitions_assigned(&self, _: &[TopicPartition]) -> Result<(), Error> {
8886 Ok(())
8887 }
8888 }
8889
8890 let (mut consumer, handles) = make_test_consumer_with_channels();
8891 let ConsumerTestHandles { mut app_event_rx, bg_event_tx, .. } = handles;
8892 let listener = Arc::new(SeekingListener { handle: consumer.handle() });
8893 *consumer.rebalance_listener.lock().unwrap() = Some(listener as Arc<dyn ConsumerRebalanceListener>);
8894 let tp = TopicPartition::new("t".to_string(), 0);
8895 let (ack_tx, mut ack_rx) = oneshot::channel::<Result<(), Error>>();
8896 bg_event_tx
8897 .send(BackgroundEventEnvelope {
8898 event: BackgroundEvent::PartitionsRemoved {
8899 method_name: ConsumerRebalanceListenerMethodName::OnPartitionsRevoked,
8900 partitions: vec![tp.clone()],
8901 ack: ack_tx,
8902 ack_through_application_events: true,
8903 },
8904 enqueued_ms: 0,
8905 })
8906 .expect("bg event sent");
8907
8908 // Poll the caller's future once — far enough for the listener to
8909 // submit its seek and park on it — then drop it.
8910 {
8911 let mut drain = Box::pin(consumer.process_background_events());
8912 std::future::poll_fn(|cx| {
8913 assert!(drain.as_mut().poll(cx).is_pending(), "the listener is waiting on its seek");
8914 Poll::Ready(())
8915 })
8916 .await;
8917 }
8918
8919 match app_event_rx.try_recv().expect("the listener's seek is queued").event {
8920 ApplicationEvent::SeekUnvalidated { partition, offset, .. } => {
8921 assert_eq!((partition, offset), (tp.clone(), 42));
8922 },
8923 other => panic!("expected the listener's seek first, got {}", other.type_name()),
8924 }
8925 match app_event_rx
8926 .try_recv()
8927 .expect("the callback-completed event is still queued after the cancelled callback")
8928 .event
8929 {
8930 ApplicationEvent::ConsumerRebalanceListenerCallbackCompleted { method_name, error, ack } => {
8931 assert_eq!(method_name, ConsumerRebalanceListenerMethodName::OnPartitionsRevoked);
8932 let error = error.expect("a cancelled callback completes with an error");
8933 assert!(
8934 error.is_interrupt_error(),
8935 "Java's error for an interrupted callback, got {error:?}"
8936 );
8937 assert_eq!(
8938 error.to_string(),
8939 Error::Interrupt(crate::common::errors::InterruptError::new(
8940 "The ConsumerRebalanceListener.onPartitionsRevoked callback was interrupted: the call \
8941 running it was cancelled"
8942 ))
8943 .to_string()
8944 );
8945 ack.send(Err(error)).expect("receiver alive");
8946 assert!(matches!(ack_rx.try_recv(), Ok(Err(_))), "the event carries this callback's ack");
8947 },
8948 other => panic!(
8949 "expected the callback-completed event after the seek, got {}",
8950 other.type_name()
8951 ),
8952 }
8953 assert!(app_event_rx.try_recv().is_err(), "nothing else queued");
8954 }
8955
8956 /// An `OffsetCommitCallback` drained by the consumer's own `commit()`
8957 /// (Java's `executeCallbacks()` before every commit) may use the handle;
8958 /// after the drain the handle is gated again.
8959 #[tokio::test]
8960 async fn handle_async_ops_work_inside_an_offset_commit_callback_and_fail_after_it_returns() {
8961 struct PausingCallback {
8962 handle: ConsumerHandle,
8963 tp: TopicPartition,
8964 outcome: std::sync::Mutex<Option<Result<(), Error>>>,
8965 }
8966 #[async_trait::async_trait]
8967 impl crate::consumer::OffsetCommitCallback for PausingCallback {
8968 async fn on_complete(
8969 &self,
8970 _offsets: &HashMap<TopicPartition, OffsetAndMetadata>,
8971 _error: Option<&Error>,
8972 ) -> Result<(), Error> {
8973 *self.outcome.lock().unwrap() = Some(self.handle.pause(std::slice::from_ref(&self.tp)).await);
8974 Ok(())
8975 }
8976 }
8977
8978 let (mut consumer, handles) = make_test_consumer_with_channels();
8979 let handle = consumer.handle();
8980 let tp = TopicPartition::new("t".to_string(), 0);
8981 let callback =
8982 Arc::new(PausingCallback { handle: handle.clone(), tp: tp.clone(), outcome: std::sync::Mutex::new(None) });
8983 consumer.offset_commit_callback_invoker.enqueue_user_callback_invocation(
8984 Arc::clone(&callback) as Arc<dyn crate::consumer::OffsetCommitCallback>,
8985 singleton_offsets(tp.clone(), 1),
8986 None,
8987 );
8988 let completer = spawn_pause_resume_completer(handles.app_event_rx);
8989
8990 // An empty-offsets commit_sync runs the queued callbacks and then
8991 // waits on an already-completed future, submitting nothing.
8992 consumer
8993 .commit_sync_with_offsets(HashMap::new())
8994 .await
8995 .expect("commit runs the callback");
8996 match callback.outcome.lock().unwrap().take() {
8997 Some(Ok(())) => {},
8998 other => panic!("the handle op inside the callback must complete, got {other:?}"),
8999 }
9000
9001 let err = handle.pause(std::slice::from_ref(&tp)).await.expect_err("outside the callback");
9002 assert!(err.is_local_concurrent_modification_error(), "unexpected error {err:?}");
9003 completer.abort();
9004 }
9005
9006 /// The gate is checked when the operation's future first runs, not when
9007 /// it is created: a future built inside a callback but awaited after the
9008 /// callback returned is rejected, and one built outside but awaited inside
9009 /// is admitted. Pins the `ConsumerHandle` docs.
9010 #[tokio::test]
9011 async fn handle_gate_is_checked_when_the_future_runs() {
9012 let (consumer, handles) = make_test_consumer_with_channels();
9013 let handle = consumer.handle();
9014 let tp = TopicPartition::new("t".to_string(), 0);
9015 let completer = spawn_pause_resume_completer(handles.app_event_rx);
9016
9017 let scope = consumer.enter_callback_scope();
9018 let built_inside = handle.pause(std::slice::from_ref(&tp));
9019 drop(scope);
9020 let err = built_inside.await.expect_err("awaited after the callback returned");
9021 assert!(err.is_local_concurrent_modification_error(), "unexpected error {err:?}");
9022
9023 let built_outside = handle.pause(std::slice::from_ref(&tp));
9024 let _scope = consumer.enter_callback_scope();
9025 built_outside.await.expect("awaited inside the callback");
9026 completer.abort();
9027 }
9028
9029 /// The gate's scope is the consumer, not the calling task: while the
9030 /// consumer runs a callback, a handle op from any other task is admitted
9031 /// (and serialised through the bg task like the callback's own). This
9032 /// pins the documented limit of the gate rather than a guarantee.
9033 #[tokio::test]
9034 async fn handle_ops_from_another_task_are_admitted_while_a_callback_runs() {
9035 let (consumer, handles) = make_test_consumer_with_channels();
9036 let handle = consumer.handle();
9037 let tp = TopicPartition::new("t".to_string(), 0);
9038 let completer = spawn_pause_resume_completer(handles.app_event_rx);
9039
9040 let in_callback = consumer.enter_callback_scope();
9041 let other_task = tokio::spawn({
9042 let handle = handle.clone();
9043 let tp = tp.clone();
9044 async move { handle.resume(std::slice::from_ref(&tp)).await }
9045 });
9046 other_task.await.expect("task ok").expect("admitted while the callback runs");
9047 drop(in_callback);
9048
9049 let other_task = tokio::spawn(async move { handle.resume(std::slice::from_ref(&tp)).await });
9050 let err = other_task
9051 .await
9052 .expect("task ok")
9053 .expect_err("rejected once the callback returned");
9054 assert!(err.is_local_concurrent_modification_error(), "unexpected error {err:?}");
9055 completer.abort();
9056 }
9057
9058 /// Phase 12.5 Issue 7 regression: the production ctor must register
9059 /// the `CommitRequestManager` as a `MemberStateListener` on the
9060 /// `ConsumerMembershipManager`, so heartbeat-driven member-epoch
9061 /// updates propagate the broker-assigned UUID into the commit
9062 /// manager's internal `MemberInfo`. Without this registration the
9063 /// `OffsetCommitRequest` goes out with the default empty member id
9064 /// and the broker rejects it with `UNKNOWN_MEMBER_ID`.
9065 ///
9066 /// Java parity: `RequestManagers.supplier` (KIP-848 / `consumer`
9067 /// group protocol arm) — Java registers two listeners on
9068 /// `membershipManager`, the first being `commitRequestManager`
9069 /// itself.
9070 ///
9071 /// The test exercises the actual production ctor path
9072 /// (`AsyncKafkaConsumer::new`) against a refuses-connection broker —
9073 /// the same pattern as `tests/consumer/async_kafka_consumer_test.rs`
9074 /// — then drives `update_member_epoch(...)` on the membership
9075 /// manager's `MembershipInner` directly. This fans out to all
9076 /// registered listeners; we assert that the `CommitRequestManager`'s
9077 /// `member_info.member_id` updated to match the membership
9078 /// manager's auto-generated UUID. Reverting the listener
9079 /// registration in `with_components` makes this test fail with
9080 /// an empty `member_id`.
9081 /// Java wraps the whole `AsyncKafkaConsumer` constructor body in
9082 /// `catch (Throwable t) { ... throw new KafkaException("Failed to construct
9083 /// kafka consumer", t); }`, so every
9084 /// construction failure reaches the caller with that exact message and the
9085 /// underlying failure as its cause.
9086 ///
9087 /// Propagating the inner error raw would change both the class (an
9088 /// `IllegalArgumentException` from address parsing answers `false` to
9089 /// `is_kafka_error()`) and the message — and "Failed to construct kafka
9090 /// consumer" is the string users match on.
9091 #[tokio::test(flavor = "multi_thread")]
9092 async fn constructor_failure_is_wrapped_as_failed_to_construct_kafka_consumer() {
9093 use std::collections::HashMap;
9094
9095 use crate::common::serialization::Deserializer;
9096
9097 struct TestStringDeserializer;
9098 impl Deserializer<String> for TestStringDeserializer {
9099 async fn deserialize(&self, _topic: &str, data: &[u8]) -> Result<String, Error> {
9100 String::from_utf8(data.to_vec()).map_err(|e| Error::serialization(format!("invalid utf-8: {}", e)))
9101 }
9102 }
9103
9104 // A bootstrap address with no port passes `ConsumerConfig` parsing but
9105 // fails `parse_and_validate_addresses` inside the constructor body —
9106 // i.e. inside Java's `try`.
9107 let props = HashMap::from([("bootstrap.servers".to_string(), "no-port-here".to_string())]);
9108 let config = ConsumerConfig::new(&props).expect("config itself validates");
9109
9110 let err = AsyncKafkaConsumer::<String, String>::new(
9111 config,
9112 Box::new(TestStringDeserializer),
9113 Box::new(TestStringDeserializer),
9114 )
9115 .expect_err("an unparseable bootstrap address must fail construction");
9116
9117 // Java's message, verbatim.
9118 assert_eq!("Failed to construct kafka consumer", err.message());
9119 // Java throws a `KafkaException`, so the hierarchy predicate must agree.
9120 assert!(err.is_kafka_error(), "must be a Kafka error: {err:?}");
9121 assert!(
9122 matches!(err, Error::KafkaError(_)),
9123 "the raw ConfigException must not escape unwrapped: {err:?}"
9124 );
9125 // The original failure is the cause (Java's second constructor arg).
9126 let source = std::error::Error::source(&err).expect("the underlying failure must be the cause");
9127 assert!(
9128 source.to_string().contains("Invalid url in bootstrap.servers"),
9129 "cause must be the address-parse failure, got: {source}"
9130 );
9131 }
9132
9133 /// Critic 92, Issue 2: built as the C / Python bindings build it — outside
9134 /// any runtime, so every resolution attempt is started from the consumer's
9135 /// own `kafka-consumer-io` runtime — `close_with_options(1 s)` returns within
9136 /// Java's bound (1 s, plus at most 2 s waiting on the resolver) while slow
9137 /// lookups are in flight. With the resolution on that runtime's blocking pool,
9138 /// three 5 s lookups held `close()` for ~15 s.
9139 #[test]
9140 fn test_close_is_bounded_while_a_bootstrap_resolution_is_in_flight() {
9141 use std::collections::HashMap;
9142
9143 use crate::ClientUtils;
9144 use crate::common::serialization::StringDeserializer;
9145 use crate::consumer::{CloseOptions, Consumer, KafkaConsumer};
9146
9147 let servers: Vec<String> = (0..3)
9148 .map(|i| format!("host{i}{}:9092", ClientUtils::SLOW_TEST_HOST_SUFFIX))
9149 .collect();
9150 let props = HashMap::from([
9151 (ConsumerConfig::BOOTSTRAP_SERVERS_CONFIG.to_string(), servers.join(",")),
9152 (
9153 ConsumerConfig::BOOTSTRAP_RESOLVE_TIMEOUT_MS_CONFIG.to_string(),
9154 "120000".to_string(),
9155 ),
9156 (ConsumerConfig::GROUP_PROTOCOL_CONFIG.to_string(), "consumer".to_string()),
9157 (ConsumerConfig::GROUP_ID_CONFIG.to_string(), "test-group".to_string()),
9158 ]);
9159 let mut consumer = KafkaConsumer::<String, String>::new(
9160 ConsumerConfig::new(&props).unwrap(),
9161 StringDeserializer,
9162 StringDeserializer,
9163 )
9164 .expect("a positive timeout defers resolution, so construction succeeds");
9165
9166 let runtime = tokio::runtime::Builder::new_multi_thread().enable_all().build().unwrap();
9167 let elapsed = runtime.block_on(async move {
9168 consumer
9169 .subscribe_with_topics(vec!["topic".to_string()])
9170 .await
9171 .expect("subscribe");
9172 consumer
9173 .poll(Duration::from_millis(100))
9174 .await
9175 .expect("poll while bootstrapping");
9176 let started = std::time::Instant::now();
9177 consumer
9178 .close_with_options(CloseOptions::new_with_timeout(Duration::from_secs(1)))
9179 .await
9180 .expect("close");
9181 started.elapsed()
9182 });
9183 assert!(elapsed < Duration::from_secs(5), "close_with_options(1 s) took {elapsed:?}");
9184 }
9185
9186 /// A config built inside the crate from `ConsumerConfig::defaults()` skips
9187 /// `ConsumerConfig::new`, so the constructor derives the `client.id` with
9188 /// the same `maybeOverrideClientId` rule instead of sending an empty id.
9189 #[tokio::test(flavor = "multi_thread")]
9190 async fn default_built_config_gets_generated_client_id() {
9191 use crate::common::serialization::Deserializer;
9192
9193 struct TestStringDeserializer;
9194 impl Deserializer<String> for TestStringDeserializer {
9195 async fn deserialize(&self, _topic: &str, data: &[u8]) -> Result<String, Error> {
9196 String::from_utf8(data.to_vec()).map_err(|e| Error::serialization(format!("invalid utf-8: {}", e)))
9197 }
9198 }
9199
9200 // A refused localhost port, so the ctor does not block on network IO.
9201 let config = ConsumerConfig {
9202 bootstrap_servers: vec!["127.0.0.1:1".to_string()],
9203 group_id: Some("g".to_string()),
9204 ..ConsumerConfig::defaults()
9205 };
9206 assert_eq!(
9207 config.client_id(),
9208 "",
9209 "precondition: a default-built config leaves client.id unset"
9210 );
9211
9212 let mut consumer = AsyncKafkaConsumer::<String, String>::new(
9213 config,
9214 Box::new(TestStringDeserializer),
9215 Box::new(TestStringDeserializer),
9216 )
9217 .expect("ctor should succeed against a refused broker");
9218
9219 let client_id = consumer.client_id().to_string();
9220 let n = client_id
9221 .strip_prefix("consumer-g-")
9222 .unwrap_or_else(|| panic!("unexpected generated id {client_id:?}"));
9223 assert!(n.parse::<i32>().unwrap() >= 1, "unexpected generated id {client_id:?}");
9224
9225 consumer.close().await.expect("close should succeed");
9226 }
9227
9228 /// Audit finding FC-03: a closed and dropped production consumer is freed,
9229 /// together with its metrics registry, subscription state, metadata and the
9230 /// rebalance listener it was given. Java's back-references (the
9231 /// `kafka-metrics-count` gauge capturing the registry, in the `Metrics`
9232 /// constructor; `addClusterUpdateListener(this)`, in the
9233 /// `OffsetsRequestManager` constructor) are
9234 /// collected by the garbage collector once the application drops the
9235 /// consumer; translated as strong `Arc`s they formed cycles that kept every
9236 /// consumer alive for the life of the process, so a C listener's
9237 /// `user_data_destroy` never ran.
9238 #[tokio::test(flavor = "multi_thread")]
9239 async fn closed_and_dropped_consumer_is_freed() {
9240 use std::collections::HashMap;
9241 use std::sync::Weak;
9242
9243 use crate::common::serialization::Deserializer;
9244
9245 struct TestStringDeserializer;
9246 impl Deserializer<String> for TestStringDeserializer {
9247 async fn deserialize(&self, _topic: &str, data: &[u8]) -> Result<String, Error> {
9248 String::from_utf8(data.to_vec()).map_err(|e| Error::serialization(format!("invalid utf-8: {}", e)))
9249 }
9250 }
9251
9252 struct Fc03Listener;
9253 #[async_trait::async_trait]
9254 impl ConsumerRebalanceListener for Fc03Listener {
9255 async fn on_partitions_revoked(&self, _partitions: &[TopicPartition]) -> Result<(), Error> {
9256 Ok(())
9257 }
9258 async fn on_partitions_assigned(&self, _partitions: &[TopicPartition]) -> Result<(), Error> {
9259 Ok(())
9260 }
9261 }
9262
9263 // A refused localhost port, so the ctor does not block on network IO.
9264 let props = HashMap::from([
9265 ("bootstrap.servers".to_string(), "127.0.0.1:1".to_string()),
9266 ("group.id".to_string(), "fc-03-group".to_string()),
9267 ("group.protocol".to_string(), "consumer".to_string()),
9268 ]);
9269 let mut consumer = AsyncKafkaConsumer::<String, String>::new(
9270 ConsumerConfig::new(&props).expect("config validates"),
9271 Box::new(TestStringDeserializer),
9272 Box::new(TestStringDeserializer),
9273 )
9274 .expect("ctor should succeed against a refused broker");
9275
9276 let listener: Arc<dyn ConsumerRebalanceListener> = Arc::new(Fc03Listener);
9277 let listener_weak = Arc::downgrade(&listener);
9278 consumer
9279 .subscribe_with_topics_listener(vec!["fc-03-topic".to_string()], listener)
9280 .await
9281 .expect("subscribe should succeed");
9282
9283 let subscriptions: Weak<Mutex<SubscriptionState>> = Arc::downgrade(&consumer.subscriptions);
9284 let metadata: Weak<ConsumerMetadata> = Arc::downgrade(&consumer.metadata);
9285 let metrics: Weak<Metrics> = Arc::downgrade(&consumer.metrics);
9286 // Every registered `KafkaMetric` lives in the registry's shared map, so a
9287 // metric outliving the consumer means the registry itself leaked.
9288 let registered: Vec<(crate::common::MetricName, Weak<crate::common::metrics::KafkaMetric>)> = consumer
9289 .metrics
9290 .metrics()
9291 .snapshot()
9292 .iter()
9293 .map(|(name, metric)| (name.clone(), Arc::downgrade(metric)))
9294 .collect();
9295 assert!(!registered.is_empty(), "precondition: the consumer registers metrics");
9296
9297 consumer.close().await.expect("close should succeed");
9298 drop(consumer);
9299
9300 // Allow any detached continuation still finishing after close to drop
9301 // its clones.
9302 let all_freed = || {
9303 listener_weak.upgrade().is_none()
9304 && subscriptions.upgrade().is_none()
9305 && metadata.upgrade().is_none()
9306 && metrics.upgrade().is_none()
9307 && registered.iter().all(|(_, m)| m.upgrade().is_none())
9308 };
9309 let deadline = std::time::Instant::now() + Duration::from_secs(10);
9310 while !all_freed() && std::time::Instant::now() < deadline {
9311 tokio::time::sleep(Duration::from_millis(20)).await;
9312 }
9313
9314 assert!(
9315 listener_weak.upgrade().is_none(),
9316 "rebalance listener leaked after close + drop"
9317 );
9318 assert!(subscriptions.upgrade().is_none(), "SubscriptionState leaked after close + drop");
9319 assert!(metadata.upgrade().is_none(), "ConsumerMetadata leaked after close + drop");
9320 assert!(metrics.upgrade().is_none(), "Metrics leaked after close + drop");
9321 let leaked: Vec<String> = registered
9322 .iter()
9323 .filter(|(_, m)| m.upgrade().is_some())
9324 .map(|(name, _)| format!("{}/{}", name.group(), name.name()))
9325 .collect();
9326 assert!(leaked.is_empty(), "metrics registry leaked after close + drop: {leaked:?}");
9327 }
9328
9329 /// `client.rack` reaches the membership manager as Java's
9330 /// `GroupRebalanceConfig.rackId` (`GroupRebalanceConfig#rackId`),
9331 /// trimmed, and an empty one (the default) is no rack. Before Critic 96 F3
9332 /// the constructor always passed `None`, so a KIP-848 heartbeat never
9333 /// carried the consumer's rack.
9334 #[tokio::test(flavor = "multi_thread")]
9335 async fn client_rack_reaches_the_membership_manager() {
9336 use std::collections::HashMap;
9337
9338 use crate::common::serialization::Deserializer;
9339
9340 struct TestStringDeserializer;
9341 impl Deserializer<String> for TestStringDeserializer {
9342 async fn deserialize(&self, _topic: &str, data: &[u8]) -> Result<String, Error> {
9343 String::from_utf8(data.to_vec()).map_err(|e| Error::serialization(format!("invalid utf-8: {}", e)))
9344 }
9345 }
9346
9347 for (configured, expected) in [(Some(" rack-1 "), Some("rack-1")), (None, None), (Some(""), None)] {
9348 let mut props = HashMap::from([
9349 ("bootstrap.servers".to_string(), "127.0.0.1:1".to_string()),
9350 ("group.id".to_string(), "rack-group".to_string()),
9351 ("group.protocol".to_string(), "consumer".to_string()),
9352 ("enable.auto.commit".to_string(), "false".to_string()),
9353 ]);
9354 if let Some(rack) = configured {
9355 props.insert(ConsumerConfig::CLIENT_RACK_CONFIG.to_string(), rack.to_string());
9356 }
9357 let mut consumer = AsyncKafkaConsumer::<String, String>::new(
9358 ConsumerConfig::new(&props).expect("config validates"),
9359 Box::new(TestStringDeserializer),
9360 Box::new(TestStringDeserializer),
9361 )
9362 .expect("ctor should succeed against a refused broker");
9363 let membership = consumer
9364 .request_managers
9365 .lock()
9366 .unwrap()
9367 .consumer_membership
9368 .clone()
9369 .expect("group.id set → membership must be Some");
9370 assert_eq!(expected, membership.rack_id(), "client.rack = {configured:?}");
9371 consumer.close().await.expect("close");
9372 }
9373 }
9374
9375 /// `metadata.cluster.check.enable` (KIP-1242) reaches the consumer's
9376 /// `NetworkClient`, as `ClientUtils.createNetworkClient` passes it
9377 /// Goes through the REAL constructor (DoD #12)
9378 /// against an in-process fake broker and observes the wire: the bootstrap
9379 /// connection's ApiVersions carries no cluster id (not known yet), and the
9380 /// connection to the partition leader (node 0, opened to reset the assigned
9381 /// partition's position) carries `ClusterId`/`NodeId` only when the check
9382 /// is enabled — the default. Java's broker-backed analogue is
9383 /// `ClientRebootstrapTest.testRebootstrapOnMetadataClusterCheckFail`.
9384 #[tokio::test(flavor = "multi_thread")]
9385 async fn metadata_cluster_check_enable_reaches_the_network_client() {
9386 use std::collections::HashMap;
9387
9388 use crate::common::serialization::Deserializer;
9389 use crate::test_fake_broker::{FAKE_BROKER_NODE_ID, FakeBroker, ReceivedApiVersions};
9390
9391 struct TestStringDeserializer;
9392 impl Deserializer<String> for TestStringDeserializer {
9393 async fn deserialize(&self, _topic: &str, data: &[u8]) -> Result<String, Error> {
9394 String::from_utf8(data.to_vec()).map_err(|e| Error::serialization(format!("invalid utf-8: {}", e)))
9395 }
9396 }
9397
9398 const TOPIC: &str = "cluster-check-topic";
9399 for (configured, check) in [(None, true), (Some("false"), false), (Some("true"), true)] {
9400 let broker = FakeBroker::start("fake-cluster", TOPIC).await;
9401 // No group.id: the position comes from a ListOffsets to the leader,
9402 // the request that opens the second connection.
9403 let mut props = HashMap::from([("bootstrap.servers".to_string(), broker.bootstrap())]);
9404 if let Some(value) = configured {
9405 props.insert("metadata.cluster.check.enable".to_string(), value.to_string());
9406 }
9407 let mut consumer = AsyncKafkaConsumer::<String, String>::new(
9408 ConsumerConfig::new(&props).expect("config validates"),
9409 Box::new(TestStringDeserializer),
9410 Box::new(TestStringDeserializer),
9411 )
9412 .expect("constructs against the fake broker");
9413 consumer
9414 .assign(vec![TopicPartition::new(TOPIC.to_string(), 0)])
9415 .await
9416 .expect("assign");
9417 let deadline = tokio::time::Instant::now() + Duration::from_secs(30);
9418 while broker.api_versions_requests().len() < 2 && tokio::time::Instant::now() < deadline {
9419 let _ = consumer.poll(Duration::from_millis(50)).await;
9420 }
9421 let seen = broker.api_versions_requests();
9422 assert_eq!(2, seen.len(), "metadata.cluster.check.enable = {configured:?}: {seen:?}");
9423 let unchecked = ReceivedApiVersions { version: 5, cluster_id: None, node_id: -1 };
9424 assert_eq!(unchecked, seen[0], "bootstrap connection, check = {check}");
9425 let expected_leader = if check {
9426 ReceivedApiVersions {
9427 version: 5,
9428 cluster_id: Some("fake-cluster".to_string()),
9429 node_id: FAKE_BROKER_NODE_ID,
9430 }
9431 } else {
9432 unchecked
9433 };
9434 assert_eq!(expected_leader, seen[1], "leader connection, check = {check}");
9435 consumer
9436 .close_with_options(crate::consumer::CloseOptions::new_with_timeout(Duration::ZERO))
9437 .await
9438 .expect("close");
9439 }
9440 }
9441
9442 /// Java validates `group.instance.id` again in the constructor
9443 /// (`GroupRebalanceConfig#GroupRebalanceConfig(AbstractConfig,ProtocolType)`), independently of `client.id`, so an
9444 /// id that `ConsumerConfig` accepted (explicit `client.id`) still fails
9445 /// construction with the constructor wrap and `JoinGroupRequest`'s message
9446 /// as the cause.
9447 #[tokio::test(flavor = "multi_thread")]
9448 async fn invalid_group_instance_id_fails_construction_with_explicit_client_id() {
9449 use std::collections::HashMap;
9450
9451 use crate::common::serialization::Deserializer;
9452
9453 struct TestStringDeserializer;
9454 impl Deserializer<String> for TestStringDeserializer {
9455 async fn deserialize(&self, _topic: &str, data: &[u8]) -> Result<String, Error> {
9456 String::from_utf8(data.to_vec()).map_err(|e| Error::serialization(format!("invalid utf-8: {}", e)))
9457 }
9458 }
9459
9460 let props = HashMap::from([
9461 ("bootstrap.servers".to_string(), "localhost:9092".to_string()),
9462 ("client.id".to_string(), "my-consumer".to_string()),
9463 ("group.id".to_string(), "test-group".to_string()),
9464 ("group.instance.id".to_string(), "bad/id".to_string()),
9465 ]);
9466 let config = ConsumerConfig::new(&props).expect("config itself validates");
9467
9468 let err = AsyncKafkaConsumer::<String, String>::new(
9469 config,
9470 Box::new(TestStringDeserializer),
9471 Box::new(TestStringDeserializer),
9472 )
9473 .expect_err("an invalid group.instance.id must fail construction");
9474
9475 assert_eq!("Failed to construct kafka consumer", err.message());
9476 let source = std::error::Error::source(&err).expect("the validation failure must be the cause");
9477 let Some(Error::InvalidConfiguration(cause)) = err.kafka_error().and_then(|e| e.source()) else {
9478 panic!("cause must be InvalidConfiguration, got: {source:?}");
9479 };
9480 assert_eq!(
9481 "Group instance id is invalid: 'bad/id' contains one or more characters other than ASCII \
9482 alphanumerics, '.', '_' and '-'",
9483 cause.message()
9484 );
9485 }
9486
9487 /// A channel-builder failure (here an unreadable truststore) is Java's
9488 /// `ConfigException` (`SslFactory#configure`), the cause of the
9489 /// "Failed to construct kafka consumer" wrap.
9490 #[tokio::test(flavor = "multi_thread")]
9491 async fn channel_builder_failure_is_a_config_error() {
9492 use std::collections::HashMap;
9493
9494 use crate::common::serialization::Deserializer;
9495
9496 struct TestStringDeserializer;
9497 impl Deserializer<String> for TestStringDeserializer {
9498 async fn deserialize(&self, _topic: &str, data: &[u8]) -> Result<String, Error> {
9499 String::from_utf8(data.to_vec()).map_err(|e| Error::serialization(format!("invalid utf-8: {}", e)))
9500 }
9501 }
9502
9503 let props = HashMap::from([
9504 ("bootstrap.servers".to_string(), "localhost:9092".to_string()),
9505 ("security.protocol".to_string(), "SSL".to_string()),
9506 (
9507 "ssl.truststore.location".to_string(),
9508 "/nonexistent/kafka-clients-test/truststore.pem".to_string(),
9509 ),
9510 ]);
9511 let config = ConsumerConfig::new(&props).expect("config itself validates");
9512
9513 let err = AsyncKafkaConsumer::<String, String>::new(
9514 config,
9515 Box::new(TestStringDeserializer),
9516 Box::new(TestStringDeserializer),
9517 )
9518 .expect_err("an unreadable truststore must fail construction");
9519
9520 assert_eq!("Failed to construct kafka consumer", err.message());
9521 let cause = err
9522 .kafka_error()
9523 .and_then(|e| e.source())
9524 .expect("the channel-builder failure is the cause");
9525 assert!(cause.is_config_error(), "cause must be a ConfigError, got: {cause:?}");
9526 assert!(cause.is_kafka_error(), "{cause:?}");
9527 assert!(
9528 cause.message().starts_with("Failed to create channel builder: "),
9529 "{}",
9530 cause.message()
9531 );
9532 }
9533
9534 #[tokio::test(flavor = "multi_thread")]
9535 async fn issue_7_commit_request_manager_registered_as_member_state_listener() {
9536 use std::collections::HashMap;
9537
9538 use crate::common::serialization::Deserializer;
9539
9540 // Local string deserializer matching the smoke-test pattern.
9541 struct TestStringDeserializer;
9542 impl Deserializer<String> for TestStringDeserializer {
9543 async fn deserialize(&self, _topic: &str, data: &[u8]) -> Result<String, Error> {
9544 String::from_utf8(data.to_vec()).map_err(|e| Error::serialization(format!("invalid utf-8: {}", e)))
9545 }
9546 }
9547
9548 // Build a `ConsumerConfig` with `group.protocol=consumer` and a
9549 // `group.id` so the membership + commit managers are constructed
9550 // by the ctor (gated paths). Point at a refused localhost port
9551 // so the ctor does not block on network IO.
9552 let props = HashMap::from([
9553 ("bootstrap.servers".to_string(), "127.0.0.1:1".to_string()),
9554 ("group.id".to_string(), "issue-7-group".to_string()),
9555 ("group.protocol".to_string(), "consumer".to_string()),
9556 ("client.id".to_string(), "issue-7-client".to_string()),
9557 ("auto.offset.reset".to_string(), "earliest".to_string()),
9558 ("enable.auto.commit".to_string(), "false".to_string()),
9559 ]);
9560 let config = ConsumerConfig::new(&props).expect("config validates");
9561
9562 let mut consumer = AsyncKafkaConsumer::<String, String>::new(
9563 config,
9564 Box::new(TestStringDeserializer),
9565 Box::new(TestStringDeserializer),
9566 )
9567 .expect("ctor should succeed against a refused broker");
9568
9569 // Reach into the production `RequestManagers` slot. Both
9570 // `consumer_membership` and `commit` MUST be `Some` because
9571 // `group.id` was set.
9572 let (membership_arc, commit_arc) = {
9573 let rm_guard = consumer.request_managers.lock().expect("rm not poisoned");
9574 (
9575 rm_guard
9576 .consumer_membership
9577 .as_ref()
9578 .expect("group.id set → membership must be Some")
9579 .clone(),
9580 rm_guard.commit.as_ref().expect("group.id set → commit must be Some").clone(),
9581 )
9582 };
9583
9584 // Pre-condition: `member_info.member_id` defaults to "" — the
9585 // broker-assigned UUID has not been observed yet. If the listener
9586 // was already invoked somehow during ctor we'd see the membership
9587 // manager's UUID here.
9588 assert_eq!(
9589 commit_arc.member_info_for_test().member_id,
9590 "",
9591 "before epoch update, member_id should be the default empty string"
9592 );
9593
9594 // Capture the membership manager's auto-generated UUID. This is
9595 // the value the listener will pass to
9596 // `on_member_epoch_updated(epoch, member_id)`.
9597 let expected_member_id = {
9598 let inner_guard = membership_arc.abstract_mm.inner.lock().expect("membership inner not poisoned");
9599 inner_guard.member_id.clone()
9600 };
9601 assert!(
9602 !expected_member_id.is_empty(),
9603 "membership manager must have auto-generated a UUID"
9604 );
9605
9606 // Drive `update_member_epoch(...)` directly on the inner. This
9607 // is the same fan-out that Java's heartbeat-response handler
9608 // triggers via `MembershipManager.updateMemberEpoch(int)`.
9609 // `update_member_epoch` calls `notify_epoch_change(Some(epoch))`
9610 // which iterates `state_updates_listeners` and invokes each
9611 // listener's `on_member_epoch_updated(epoch, &self.member_id)`.
9612 {
9613 let mut inner_guard = membership_arc.abstract_mm.inner.lock().expect("membership inner not poisoned");
9614 inner_guard.update_member_epoch(42);
9615 }
9616
9617 // Post-condition: if `CommitRequestManager` was registered as a
9618 // `MemberStateListener`, its `member_info` now carries the
9619 // membership manager's UUID + epoch.
9620 let post = commit_arc.member_info_for_test();
9621 assert_eq!(
9622 post.member_id, expected_member_id,
9623 "Issue 7: CommitRequestManager.member_id must equal membership.member_id after \
9624 update_member_epoch — this proves the listener registration in new_with_components \
9625 routed the epoch update through to the commit manager"
9626 );
9627 assert_eq!(
9628 post.member_epoch,
9629 Some(42),
9630 "epoch should match the value passed to update_member_epoch"
9631 );
9632
9633 // Clean shutdown.
9634 consumer.close().await.expect("close should succeed");
9635 }
9636
9637 /// Stand-in for Java's `testAssignmentReturnsAssignedPartitions`
9638 /// once `assign` lands in commit (3/N) — for now we mutate
9639 /// `SubscriptionState` directly to verify the read path.
9640 #[tokio::test]
9641 async fn assignment_reads_subscription_state() {
9642 let consumer = make_test_consumer();
9643 let tp = TopicPartition::new("t".to_string(), 0);
9644 {
9645 let mut subs = consumer.subscriptions.lock().unwrap();
9646 let mut assigned: HashSet<TopicPartition> = HashSet::new();
9647 assigned.insert(tp.clone());
9648 subs.assign_from_user(assigned).unwrap();
9649 }
9650 let assignment = consumer.assignment();
9651 assert_eq!(assignment.len(), 1);
9652 assert!(assignment.contains(&tp));
9653 }
9654
9655 // ─── Subscribe / unsubscribe / assign tests (commit (3/N)) ───
9656 //
9657 // These tests stand in for Java's
9658 // `AsyncKafkaConsumerTest.testSubscribeGeneratesEvent` /
9659 // `testSubscribePatternGeneratesEvent` /
9660 // `testUnsubscribeGeneratesUnsubscribeEvent` /
9661 // `testAssign*` / etc. Where Java's tests use Mockito to immediately
9662 // satisfy `applicationEventHandler.addAndGet(...)`, the Rust tests
9663 // spawn a small helper task that drains one envelope off the
9664 // app-event channel and `complete`s the handle inside it.
9665 //
9666 // Skipped Java tests for this commit (rationale per row):
9667 // - `testAssignOnNullTopicPartition` / `testAssignOnNullTopicInPartition`
9668 // — Rust's type system makes the `null` case unrepresentable:
9669 // `Vec<TopicPartition>` cannot contain a `null` slot, and
9670 // `TopicPartition` requires an owned `String` topic.
9671 // - `testSubscribeToNullTopicCollection` /
9672 // `testSubscriptionOnNullTopic` — same.
9673 // - `testReaperInvokedInUnsubscribe` — depends on `backgroundEventReaper.reap(time)`
9674 // wiring inside `process_background_events`; the reaper hookup
9675 // itself lands later (Phase 11 commit (4/N)). Deferred to that
9676 // commit.
9677 // - `testGroupMetadataIsResetAfterUnsubscribe` — depends on the
9678 // `MemberStateListener` (commit (7/N)) that populates
9679 // `group_metadata`. Deferred to that commit.
9680 // - `testUnsubscribeWithoutGroupId` — depends on a no-group ctor
9681 // path which is built in commit (7/N) via `KafkaConsumer::new`.
9682 // Deferred.
9683 // - `testSubscribePatternAgainstBrokerNotSupportingRegex` —
9684 // end-to-end against a `MockClient`; depends on the poll path
9685 // (commit (4/N)).
9686 // - `testReaperInvokedInPoll` / similar poll-only flows — commit
9687 // (4/N).
9688
9689 /// Helper: spawn a task that takes the next envelope off the
9690 /// app-event channel and completes its `handle` with `Ok(())`.
9691 /// Mirrors Java's
9692 /// `completeTopicSubscriptionChangeEventSuccessfully()` / etc.
9693 fn auto_complete_next_event(
9694 mut rx: mpsc::UnboundedReceiver<ApplicationEventEnvelope>,
9695 ) -> tokio::task::JoinHandle<Option<ApplicationEventEnvelope>> {
9696 tokio::spawn(async move {
9697 let env = rx.recv().await?;
9698 complete_event(&env);
9699 Some(env)
9700 })
9701 }
9702
9703 /// Same as [`auto_complete_next_event`] but keeps completing every event
9704 /// that arrives until the channel closes — for tests that make more than
9705 /// one blocking call.
9706 fn auto_complete_all_events(
9707 mut rx: mpsc::UnboundedReceiver<ApplicationEventEnvelope>,
9708 ) -> tokio::task::JoinHandle<()> {
9709 tokio::spawn(async move {
9710 while let Some(env) = rx.recv().await {
9711 complete_event(&env);
9712 }
9713 })
9714 }
9715
9716 /// Completes the handle carried by an application event, so the app-side
9717 /// `add_and_get` resolves without a background task.
9718 fn complete_event(env: &ApplicationEventEnvelope) {
9719 match &env.event {
9720 ApplicationEvent::TopicSubscriptionChange { handle, .. } => {
9721 handle.complete(());
9722 },
9723 ApplicationEvent::TopicPatternSubscriptionChange { handle, .. } => {
9724 handle.complete(());
9725 },
9726 ApplicationEvent::TopicRe2JPatternSubscriptionChange { handle, .. } => {
9727 handle.complete(());
9728 },
9729 ApplicationEvent::AssignmentChange { handle, .. } => {
9730 handle.complete(());
9731 },
9732 ApplicationEvent::Unsubscribe { handle } => {
9733 handle.complete(());
9734 },
9735 ApplicationEvent::SeekUnvalidated { handle, .. } => {
9736 handle.complete(());
9737 },
9738 ApplicationEvent::ResetOffset { handle, .. } => {
9739 handle.complete(());
9740 },
9741 ApplicationEvent::PausePartitions { handle, .. } => {
9742 handle.complete(());
9743 },
9744 ApplicationEvent::ResumePartitions { handle, .. } => {
9745 handle.complete(());
9746 },
9747 _ => {
9748 // Unknown variant — leave the handle un-completed; the
9749 // test's `add_and_get` will time out and the
9750 // assertion will be a clear failure.
9751 },
9752 }
9753 }
9754
9755 /// A stand-in background task for the 4.4 tests below: it applies the
9756 /// subscription, unsubscribe and pause events to the shared
9757 /// `SubscriptionState` and completes them, as Java's
9758 /// `complete*EventSuccessfully` mock answers do, and records the name of
9759 /// every event it sees.
9760 fn apply_and_complete_events(
9761 mut rx: mpsc::UnboundedReceiver<ApplicationEventEnvelope>,
9762 subscriptions: Arc<Mutex<SubscriptionState>>,
9763 ) -> (tokio::task::JoinHandle<()>, Arc<Mutex<Vec<&'static str>>>) {
9764 let seen = Arc::new(Mutex::new(Vec::new()));
9765 let seen_in_task = Arc::clone(&seen);
9766 let task = tokio::spawn(async move {
9767 while let Some(env) = rx.recv().await {
9768 seen_in_task.lock().unwrap().push(env.event.type_name());
9769 match &env.event {
9770 ApplicationEvent::TopicSubscriptionChange { topics, listener, .. } => {
9771 subscriptions
9772 .lock()
9773 .unwrap()
9774 .subscribe_with_topics(topics.clone(), listener.clone())
9775 .unwrap();
9776 },
9777 ApplicationEvent::Unsubscribe { .. } => subscriptions.lock().unwrap().unsubscribe(),
9778 ApplicationEvent::PausePartitions { partitions, .. } => {
9779 let mut subs = subscriptions.lock().unwrap();
9780 for tp in partitions {
9781 subs.pause(tp).unwrap();
9782 }
9783 },
9784 _ => {},
9785 }
9786 complete_event(&env);
9787 }
9788 });
9789 (task, seen)
9790 }
9791
9792 /// Translated from `AsyncKafkaConsumerTest.testPauseFlagPreservedForRetainedPartitionAcrossRebalance`
9793 /// (36aab4fddd): a paused partition stays paused across a reconciliation
9794 /// that keeps it assigned, and a newly added one starts unpaused.
9795 #[tokio::test]
9796 async fn test_pause_flag_preserved_for_retained_partition_across_rebalance() {
9797 let (mut consumer, handles) = make_test_consumer_with_channels();
9798 let subscriptions = Arc::clone(&handles.subscriptions);
9799 let (drainer, _seen) = apply_and_complete_events(handles.app_event_rx, Arc::clone(&subscriptions));
9800 let tp0 = TopicPartition::new("topic1", 0);
9801 let tp1 = TopicPartition::new("topic1", 1);
9802 let tp2 = TopicPartition::new("topic1", 2);
9803
9804 struct NoopListener;
9805 #[async_trait::async_trait]
9806 impl ConsumerRebalanceListener for NoopListener {
9807 async fn on_partitions_revoked(&self, _partitions: &[TopicPartition]) -> Result<(), Error> {
9808 Ok(())
9809 }
9810 async fn on_partitions_assigned(&self, _partitions: &[TopicPartition]) -> Result<(), Error> {
9811 Ok(())
9812 }
9813 }
9814 consumer
9815 .subscribe_with_topics_listener(vec!["topic1".to_string()], Arc::new(NoopListener))
9816 .await
9817 .unwrap();
9818
9819 // Simulate a rebalance that reconciled a new assignment of tp0 and tp1.
9820 subscriptions
9821 .lock()
9822 .unwrap()
9823 .assign_from_subscribed_awaiting_callback(&[tp0.clone(), tp1.clone()], &[tp0.clone(), tp1.clone()])
9824 .unwrap();
9825 assert_eq!(HashSet::from([tp0.clone(), tp1.clone()]), consumer.assignment());
9826
9827 // Pause tp0, the partition that will be retained across the rebalance.
9828 consumer.pause(std::slice::from_ref(&tp0)).await.unwrap();
9829 assert_eq!(HashSet::from([tp0.clone()]), consumer.paused());
9830
9831 // Reconcile a new assignment that retains tp0, revokes tp1, and adds tp2.
9832 subscriptions
9833 .lock()
9834 .unwrap()
9835 .assign_from_subscribed_awaiting_callback(&[tp0.clone(), tp2.clone()], std::slice::from_ref(&tp2))
9836 .unwrap();
9837
9838 // tp0 is retained across the rebalance, so its pause state must be
9839 // preserved. The newly added tp2 starts unpaused.
9840 assert_eq!(HashSet::from([tp0.clone(), tp2]), consumer.assignment());
9841 assert_eq!(
9842 HashSet::from([tp0]),
9843 consumer.paused(),
9844 "Partition that remain assigned should keep the pause state"
9845 );
9846 drainer.abort();
9847 }
9848
9849 /// Translated from `AsyncKafkaConsumerTest.testUnsubscribeDoesNotCommitOffsetsEvenWithAutoCommitEnabled`
9850 /// (KAFKA-20119, 40e9fcd742; a5137f7c38 only stubs Java's close-time
9851 /// commit, which this fixture's drainer completes anyway): with
9852 /// `enable.auto.commit` on, `unsubscribe()` sends an `Unsubscribe` event and
9853 /// no commit event of any kind.
9854 #[tokio::test]
9855 async fn test_unsubscribe_does_not_commit_offsets_even_with_auto_commit_enabled() {
9856 let (mut consumer, handles) = make_test_consumer_with_channels();
9857 assert!(
9858 consumer.auto_commit_enabled,
9859 "the fixture's enable.auto.commit is Java's default, true"
9860 );
9861 let (drainer, seen) = apply_and_complete_events(handles.app_event_rx, Arc::clone(&handles.subscriptions));
9862
9863 // Subscribe to a topic.
9864 consumer.subscribe_with_topics(vec!["topic".to_string()]).await.unwrap();
9865 // Focus on the unsubscribe behavior.
9866 seen.lock().unwrap().clear();
9867
9868 // This should NOT commit offsets even though auto-commit is enabled.
9869 consumer.unsubscribe().await.unwrap();
9870
9871 let events = seen.lock().unwrap().clone();
9872 assert!(events.contains(&"Unsubscribe"), "UnsubscribeEvent was sent: {events:?}");
9873 for commit in ["CommitSync", "CommitAsync", "CommitOnClose"] {
9874 assert!(!events.contains(&commit), "no {commit} event despite auto-commit: {events:?}");
9875 }
9876 assert!(consumer.subscription().is_empty());
9877 drainer.abort();
9878 }
9879
9880 /// Java: `testSubscribeGeneratesEvent`.
9881 #[tokio::test]
9882 async fn subscribe_generates_topic_subscription_change_event() {
9883 let (mut consumer, handles) = make_test_consumer_with_channels();
9884 let completer = auto_complete_next_event(handles.app_event_rx);
9885 consumer.subscribe_with_topics(vec!["topic1".to_string()]).await.expect("ok");
9886 let env = completer.await.expect("task ok").expect("event received");
9887 assert!(matches!(env.event, ApplicationEvent::TopicSubscriptionChange { .. }));
9888 }
9889
9890 /// Java: `testSubscribeToRe2JPatternGeneratesEvent`.
9891 #[tokio::test]
9892 async fn subscribe_subscription_pattern_generates_event() {
9893 let (mut consumer, handles) = make_test_consumer_with_channels();
9894 let completer = auto_complete_next_event(handles.app_event_rx);
9895 consumer
9896 .subscribe_with_pattern(SubscriptionPattern::new("t*"))
9897 .await
9898 .expect("ok");
9899 let env = completer.await.expect("task ok").expect("event received");
9900 assert!(matches!(env.event, ApplicationEvent::TopicRe2JPatternSubscriptionChange { .. }));
9901 }
9902
9903 /// Java: `testSubscribeToRe2JPatternValidation`
9904 /// — empty pattern rejected with the EXACT Java error message
9905 /// `"Topic pattern to subscribe to cannot be empty"`. Issue 5
9906 /// (Critic batch-1): substring assertion strengthened to exact match
9907 /// per DoD §3 (error message content is part of the behavioural
9908 /// contract).
9909 ///
9910 /// SKIP: null-pattern case — unrepresentable in
9911 /// Rust because `subscribe_with_pattern` takes `SubscriptionPattern`
9912 /// by value, not `Option<SubscriptionPattern>`.
9913 ///
9914 /// SKIP: null-listener case — unrepresentable in
9915 /// Rust because `subscribe_with_pattern_listener` takes
9916 /// `Arc<dyn ConsumerRebalanceListener>`, not
9917 /// `Option<Arc<dyn ConsumerRebalanceListener>>`.
9918 #[tokio::test]
9919 async fn subscribe_subscription_pattern_rejects_empty() {
9920 let (mut consumer, _handles) = make_test_consumer_with_channels();
9921 let err = consumer
9922 .subscribe_with_pattern(SubscriptionPattern::new(""))
9923 .await
9924 .expect_err("must err");
9925 match err {
9926 Error::LocalIllegalArgument(msg) => {
9927 assert_eq!(msg.message(), "Topic pattern to subscribe to cannot be empty");
9928 },
9929 other => panic!("expected IllegalArgument, got {other:?}"),
9930 }
9931 }
9932
9933 /// Java: `testSubscribeToRe2JPatternValidation` —
9934 /// `assertDoesNotThrow(() -> consumer.subscribe(new SubscriptionPattern("t*")))`.
9935 /// The valid-pattern arm of the same Java test.
9936 #[tokio::test]
9937 async fn subscribe_subscription_pattern_accepts_valid_pattern() {
9938 let (mut consumer, handles) = make_test_consumer_with_channels();
9939 let completer = auto_complete_next_event(handles.app_event_rx);
9940 consumer
9941 .subscribe_with_pattern(SubscriptionPattern::new("t*"))
9942 .await
9943 .expect("valid pattern must not throw");
9944 let env = completer.await.expect("task ok").expect("event received");
9945 assert!(matches!(env.event, ApplicationEvent::TopicRe2JPatternSubscriptionChange { .. }));
9946 }
9947
9948 /// Java: `testUnsubscribeGeneratesUnsubscribeEvent`.
9949 #[tokio::test]
9950 async fn unsubscribe_generates_unsubscribe_event() {
9951 let (mut consumer, handles) = make_test_consumer_with_channels();
9952 let completer = auto_complete_next_event(handles.app_event_rx);
9953 consumer.unsubscribe().await.expect("ok");
9954 let env = completer.await.expect("task ok").expect("event received");
9955 assert!(matches!(env.event, ApplicationEvent::Unsubscribe { .. }));
9956 }
9957
9958 /// Java: `testSubscribeToEmptyListActsAsUnsubscribe`.
9959 #[tokio::test]
9960 async fn subscribe_to_empty_list_acts_as_unsubscribe() {
9961 let (mut consumer, handles) = make_test_consumer_with_channels();
9962 let completer = auto_complete_next_event(handles.app_event_rx);
9963 consumer.subscribe_with_topics(Vec::new()).await.expect("ok");
9964 let env = completer.await.expect("task ok").expect("event received");
9965 assert!(matches!(env.event, ApplicationEvent::Unsubscribe { .. }));
9966 }
9967
9968 /// Java: `testSubscriptionOnEmptyTopic` — blank topic rejected.
9969 #[tokio::test]
9970 async fn subscribe_rejects_blank_topic() {
9971 let (mut consumer, _handles) = make_test_consumer_with_channels();
9972 let err = consumer
9973 .subscribe_with_topics(vec![" ".to_string()])
9974 .await
9975 .expect_err("must err");
9976 assert!(matches!(err, Error::LocalIllegalArgument(_)));
9977 }
9978
9979 /// Java: `testAssign`. Asserts the
9980 /// `AssignmentChangeEvent` is enqueued. The "clears subscription"
9981 /// half of Java's assertion (`consumer.subscription().isEmpty()`)
9982 /// is exercised by `assign_clears_subscription_after_event_completes`
9983 /// below — splitting the assertions clarifies what each test
9984 /// actually verifies (Issue 7 from Critic batch-1).
9985 #[tokio::test]
9986 async fn assign_generates_assignment_change_event() {
9987 let (mut consumer, handles) = make_test_consumer_with_channels();
9988 let completer = auto_complete_next_event(handles.app_event_rx);
9989 let tp = TopicPartition::new("foo".to_string(), 3);
9990 consumer.assign(vec![tp.clone()]).await.expect("ok");
9991 let env = completer.await.expect("task ok").expect("event received");
9992 match env.event {
9993 ApplicationEvent::AssignmentChange { partitions, .. } => {
9994 assert!(partitions.contains(&tp));
9995 },
9996 other => panic!("expected AssignmentChange, got {}", other.type_name()),
9997 }
9998 }
9999
10000 /// Java: `testAssign` — second-half assertion:
10001 /// `assertTrue(consumer.subscription().isEmpty())` AND
10002 /// `assertTrue(consumer.assignment().contains(tp))`. The bg-task
10003 /// `assign_from_user(...)` is applied directly on the
10004 /// `SubscriptionState` here to mirror Java's `MockClient` arm at
10005 /// `completeAssignmentChangeEventSuccessfully()`.
10006 #[tokio::test]
10007 async fn assign_clears_subscription_after_event_completes() {
10008 let (mut consumer, handles) = make_test_consumer_with_channels();
10009 let tp = TopicPartition::new("foo".to_string(), 3);
10010 let subs = Arc::clone(&handles.subscriptions);
10011 // Mirror Java's `completeAssignmentChangeEventSuccessfully`
10012 // helper: drain the event and apply `assignFromUser` on the
10013 // shared `SubscriptionState` BEFORE completing the handle.
10014 let completer = {
10015 let tp = tp.clone();
10016 let mut rx = handles.app_event_rx;
10017 tokio::spawn(async move {
10018 while let Some(env) = rx.recv().await {
10019 if let ApplicationEvent::AssignmentChange { handle, partitions, .. } = env.event {
10020 let mut s = subs.lock().unwrap();
10021 let mut set = HashSet::new();
10022 for p in partitions {
10023 set.insert(p);
10024 }
10025 s.assign_from_user(set).expect("assign ok");
10026 drop(s);
10027 handle.complete(());
10028 return Some(tp);
10029 }
10030 }
10031 None
10032 })
10033 };
10034 consumer.assign(vec![tp.clone()]).await.expect("ok");
10035 let _ = completer.await.expect("task ok").expect("event received");
10036 assert!(consumer.subscription().is_empty(), "subscription must be empty after assign");
10037 assert!(
10038 consumer.assignment().contains(&tp),
10039 "assignment must contain the assigned partition"
10040 );
10041 }
10042
10043 /// Java: `testAssignOnEmptyTopicPartition` — empty collection
10044 /// acts as `unsubscribe()`.
10045 #[tokio::test]
10046 async fn assign_on_empty_acts_as_unsubscribe() {
10047 let (mut consumer, handles) = make_test_consumer_with_channels();
10048 let completer = auto_complete_next_event(handles.app_event_rx);
10049 consumer.assign(Vec::new()).await.expect("ok");
10050 let env = completer.await.expect("task ok").expect("event received");
10051 assert!(matches!(env.event, ApplicationEvent::Unsubscribe { .. }));
10052 }
10053
10054 /// Java: `testAssignOnEmptyTopicInPartition` — blank topic rejected.
10055 #[tokio::test]
10056 async fn assign_rejects_blank_topic_in_partition() {
10057 let (mut consumer, _handles) = make_test_consumer_with_channels();
10058 let tp = TopicPartition::new(" ".to_string(), 0);
10059 let err = consumer.assign(vec![tp]).await.expect_err("must err");
10060 assert!(matches!(err, Error::LocalIllegalArgument(_)));
10061 }
10062
10063 /// Sanity check: subscribe stores the listener app-side so
10064 /// `process_background_events` can pick it up.
10065 #[tokio::test]
10066 async fn subscribe_topics_listener_stores_listener() {
10067 use async_trait::async_trait;
10068 struct DummyListener;
10069 #[async_trait]
10070 impl ConsumerRebalanceListener for DummyListener {
10071 async fn on_partitions_revoked(&self, _: &[TopicPartition]) -> Result<(), Error> {
10072 Ok(())
10073 }
10074 async fn on_partitions_assigned(&self, _: &[TopicPartition]) -> Result<(), Error> {
10075 Ok(())
10076 }
10077 async fn on_partitions_lost(&self, _: &[TopicPartition]) -> Result<(), Error> {
10078 Ok(())
10079 }
10080 }
10081 let (mut consumer, handles) = make_test_consumer_with_channels();
10082 let completer = auto_complete_all_events(handles.app_event_rx);
10083 let listener: Arc<dyn ConsumerRebalanceListener> = Arc::new(DummyListener);
10084 consumer
10085 .subscribe_with_topics_listener(vec!["t".to_string()], Arc::clone(&listener))
10086 .await
10087 .expect("ok");
10088 let stored = consumer.rebalance_listener.lock().unwrap().clone();
10089 assert!(stored.is_some(), "listener must be stored on subscribe_with_topics_listener");
10090
10091 // Java keeps ONE slot: a listener-less `subscribe(topics)` calls
10092 // `registerRebalanceListener(Optional.empty())`
10093 // (`org.apache.kafka.clients.consumer.internals.SubscriptionState#subscribe(Set,Optional)`), so it must CLEAR
10094 // the app-side mirror too — otherwise `leave_group_on_close` would invoke the
10095 // replaced listener, and (through the C FFI) its `user_data_destroy`
10096 // hook would be withheld until the consumer is dropped.
10097 consumer.subscribe_with_topics(vec!["t2".to_string()]).await.expect("ok");
10098 let stored = consumer.rebalance_listener.lock().unwrap().clone();
10099 assert!(
10100 stored.is_none(),
10101 "a listener-less subscribe must clear the previously stored listener"
10102 );
10103 completer.abort();
10104 }
10105
10106 /// The other half of the single-slot invariant: `unsubscribe()` must
10107 /// **keep** the registered listener.
10108 ///
10109 /// `SubscriptionState#unsubscribe`
10110 /// clears the subscription, group subscription, assignment, assigned topic
10111 /// ids, pattern and subscription type and bumps `assignmentId` — it does
10112 /// not touch `rebalanceListener`, which is only ever written by the three
10113 /// `subscribe(...)` overloads and `subscribeToShareGroup`. Neither
10114 /// does `AsyncKafkaConsumer#unsubscribe`.
10115 ///
10116 /// Rust duplicates Java's one slot (bg-side `SubscriptionState`, app-side
10117 /// mirror, because §31 invokes the callback on the caller's task) and the
10118 /// **mirror** is what `process_background_events` reads. So the observable
10119 /// consequence of clearing it here is a `PartitionsRemoved` event (AK
10120 /// 4.3.1's rename of `ConsumerRebalanceListenerCallbackNeeded`, revoke/lost
10121 /// shape unchanged) enqueued by the bg task while
10122 /// the registration was still live, but drained by the app after
10123 /// `unsubscribe()` returned, silently taking the `None => Ok(())` arm and
10124 /// skipping the user's `on_partitions_revoked`. That behaviour — not just
10125 /// the field state — is what this test pins, together with the fact that a
10126 /// later listener-less `subscribe(...)` (Java's
10127 /// `registerRebalanceListener(Optional.empty())`) is what ends the
10128 /// registration.
10129 #[tokio::test]
10130 async fn unsubscribe_keeps_the_registered_listener() {
10131 use crate::consumer::internals::ConsumerRebalanceListenerMethodName;
10132 use async_trait::async_trait;
10133 use std::sync::atomic::AtomicUsize;
10134 use tokio::sync::oneshot;
10135
10136 struct RecordingListener {
10137 revoked: AtomicUsize,
10138 }
10139 #[async_trait]
10140 impl ConsumerRebalanceListener for RecordingListener {
10141 async fn on_partitions_assigned(&self, _: &[TopicPartition]) -> Result<(), Error> {
10142 Ok(())
10143 }
10144 async fn on_partitions_revoked(&self, _: &[TopicPartition]) -> Result<(), Error> {
10145 self.revoked.fetch_add(1, Ordering::SeqCst);
10146 Ok(())
10147 }
10148 async fn on_partitions_lost(&self, _: &[TopicPartition]) -> Result<(), Error> {
10149 Ok(())
10150 }
10151 }
10152
10153 /// Enqueues the callback-needed event the bg task would raise and
10154 /// drains it, returning the ack result.
10155 async fn drive_revoked_callback(
10156 consumer: &mut AsyncKafkaConsumer<Vec<u8>, Vec<u8>>,
10157 bg_event_tx: &mpsc::UnboundedSender<BackgroundEventEnvelope>,
10158 ) -> Result<(), Error> {
10159 let (ack_tx, ack_rx) = oneshot::channel::<Result<(), Error>>();
10160 bg_event_tx
10161 .send(BackgroundEventEnvelope {
10162 event: BackgroundEvent::PartitionsRemoved {
10163 method_name: ConsumerRebalanceListenerMethodName::OnPartitionsRevoked,
10164 partitions: vec![TopicPartition::new("t".to_string(), 0)],
10165 ack: ack_tx,
10166 ack_through_application_events: false,
10167 },
10168 enqueued_ms: 0,
10169 })
10170 .expect("send ok");
10171 consumer.process_background_events().await.expect("drain ok");
10172 ack_rx.await.expect("ack received")
10173 }
10174
10175 let (mut consumer, handles) = make_test_consumer_with_channels();
10176 let completer = auto_complete_all_events(handles.app_event_rx);
10177 let listener: Arc<RecordingListener> = Arc::new(RecordingListener { revoked: AtomicUsize::new(0) });
10178 let erased: Arc<dyn ConsumerRebalanceListener> = Arc::clone(&listener) as Arc<dyn ConsumerRebalanceListener>;
10179
10180 consumer
10181 .subscribe_with_topics_listener(vec!["t".to_string()], Arc::clone(&erased))
10182 .await
10183 .expect("subscribe ok");
10184 // The fixture has no background task, so apply the registration the
10185 // `ApplicationEventProcessor` would perform for
10186 // `TopicSubscriptionChange` — that is the bg-side half of the slot.
10187 handles
10188 .subscriptions
10189 .lock()
10190 .unwrap()
10191 .subscribe_with_topics(["t".to_string()].into_iter().collect(), Some(Arc::clone(&erased)))
10192 .expect("subscribe_with_topics ok");
10193
10194 consumer.unsubscribe().await.expect("unsubscribe ok");
10195 // ...and the bg-side half of `Unsubscribe`.
10196 handles.subscriptions.lock().unwrap().unsubscribe();
10197
10198 // Both copies of Java's single slot must still hold the listener.
10199 assert!(
10200 handles.subscriptions.lock().unwrap().rebalance_listener().is_some(),
10201 "SubscriptionState::unsubscribe must not clear the listener (Java parity)"
10202 );
10203 assert!(
10204 consumer.rebalance_listener.lock().unwrap().is_some(),
10205 "the app-side mirror must track SubscriptionState's slot across unsubscribe()"
10206 );
10207
10208 // Observable behaviour: a callback the bg task raised while the
10209 // registration was live is still delivered to the user.
10210 assert!(drive_revoked_callback(&mut consumer, &handles.bg_event_tx).await.is_ok());
10211 assert_eq!(
10212 1,
10213 listener.revoked.load(Ordering::SeqCst),
10214 "a rebalance callback drained after unsubscribe() must still reach the retained listener"
10215 );
10216
10217 // A listener-less `subscribe(...)` is what ends the registration
10218 // (`registerRebalanceListener(Optional.empty())`), on both copies.
10219 consumer
10220 .subscribe_with_topics(vec!["t2".to_string()])
10221 .await
10222 .expect("subscribe ok");
10223 handles
10224 .subscriptions
10225 .lock()
10226 .unwrap()
10227 .subscribe_with_topics(["t2".to_string()].into_iter().collect(), None)
10228 .expect("subscribe_with_topics ok");
10229 assert!(
10230 handles.subscriptions.lock().unwrap().rebalance_listener().is_none(),
10231 "a listener-less subscribe clears SubscriptionState's slot"
10232 );
10233 assert!(
10234 consumer.rebalance_listener.lock().unwrap().is_none(),
10235 "...and the app-side mirror with it"
10236 );
10237 assert!(drive_revoked_callback(&mut consumer, &handles.bg_event_tx).await.is_ok());
10238 assert_eq!(
10239 1,
10240 listener.revoked.load(Ordering::SeqCst),
10241 "the de-registered listener must not be invoked again"
10242 );
10243
10244 completer.abort();
10245 }
10246
10247 // ─── Phase 11 commit (8/N) Java test translations ───
10248 //
10249 // The block below mirrors Java's `AsyncKafkaConsumerTest` fixture +
10250 // state-read + subscribe / unsubscribe tests. Many Java tests are
10251 // already covered by the inline tests above (commits 2–7); each
10252 // Java test that is _skipped_ carries a `// SKIP: <reason>` rationale
10253 // here per DoD §3.
10254 //
10255 // Skipped Java tests (commit 8 batch):
10256 // - testCommitInRebalanceCallback — TRANSLATED as
10257 // `test_commit_in_rebalance_callback` (§31 regression pair, below).
10258 // - testRecordBackgroundEventQueueSizeAndBackgroundEventQueueTime —
10259 // TRANSLATED (Phase M7) as
10260 // `test_record_background_event_queue_size_and_time` (inline below).
10261 // Drains a bg event under a `MockTime` advanced by 10 ms, then
10262 // reads the values via the public `metrics()` accessor (M7).
10263 // - testEmptyStreamRebalanceData, testStreamRebalanceData,
10264 // testCloseInvokesStreamsRebalanceListenerOnTasksRevokedWhenMemberEpochPositive,
10265 // testCloseInvokesStreamsRebalanceListenerOnAllTasksLostWhenMemberEpochZeroOrNegative,
10266 // testCloseWrapsStreamsRebalanceListenerException — PLAN
10267 // deferral #2 (Streams out of milestone scope per
10268 // consumer-threading.md §20).
10269 // - testGroupRemoteAssignorInClassicProtocol — PLAN deferral #3
10270 // (classic-protocol out of scope per §20).
10271 // - testFailConstructor — PLAN deferral #5 (Supplier-based ctor
10272 // failure paths don't translate; the equivalent error path is
10273 // observed via the `Error` returned from `KafkaConsumer::new` on
10274 // bad config).
10275 // - testGroupMetadataIsResetAfterUnsubscribe
10276 // — translated below as
10277 // `group_metadata_is_reset_after_unsubscribe`. Per Issue 21
10278 // fixup, `unsubscribe()` now calls
10279 // `state_notifier.reset_group_metadata()` after the unsubscribe
10280 // event completes, mirroring Java `unsubscribe()`'s
10281 // `resetGroupMetadata()`.
10282 // - testSubscribeToNullTopicCollection, testSubscriptionOnNullTopic,
10283 // testAssignOnNullTopicPartition, testAssignOnNullTopicInPartition
10284 // — Rust's type system makes `null` cases unrepresentable.
10285 // - testBeginningOffsetsFailsIfNullPartitions,
10286 // testOffsetsForTimesOnNullPartitions — same.
10287 // - Any ConcurrentModificationException-asserting test — PLAN
10288 // deferral #4 (Rust's `&mut self` makes the guard redundant; no
10289 // such test method names found in the Java file).
10290 // - testReaperInvokedInClose / testReaperInvokedInUnsubscribe /
10291 // testReaperInvokedInPoll — deferred; the reap call itself is
10292 // wired (`close_internal` line 2675, `process_background_events`),
10293 // but the test would require a mocked reaper to observe the
10294 // invocation. The bg-task `reaper.reap(time)` is exercised via
10295 // `ConsumerNetworkThreadTest` (Phase 10 commit 8/N).
10296 // - testSubscribePatternAgainstBrokerNotSupportingRegex — end-to-end
10297 // against a `MockClient`; requires response routing for
10298 // FindCoordinator + Heartbeat to be wired (deferred to Phase 12.5
10299 // per `Phase-12/RESPONSE-ROUTING-AUDIT.md`).
10300
10301 /// The closed flag alone, without running `close()`, makes an async
10302 /// method return the EXACT message `"This consumer has already been
10303 /// closed."`. Java's `testFailOnClosedConsumer` asserts it on
10304 /// `assignment()`, which in Rust panics instead (CLAUDE.md §12.5): that
10305 /// test is translated by `assignment_panics_after_close`, and every
10306 /// guarded method by `every_guarded_method_returns_illegal_state_after_close`.
10307 #[tokio::test]
10308 async fn fail_on_closed_consumer_exact_message() {
10309 let (mut consumer, _handles) = make_test_consumer_with_channels();
10310 consumer.closed.store(true, Ordering::Release);
10311 let err = consumer.commit_sync().await.expect_err("must err");
10312 match err {
10313 Error::LocalIllegalState(msg) => {
10314 assert_eq!(msg.message(), "This consumer has already been closed.");
10315 },
10316 other => panic!("expected IllegalState, got {other:?}"),
10317 }
10318 }
10319
10320 /// The group-id guard of `commit_sync()` on a groupless consumer
10321 /// (`return_error_if_group_id_not_defined`, Java's
10322 /// `throwIfGroupIdNotDefined()`) returns Java's `InvalidGroupIdException`
10323 /// message verbatim. Java's `testGroupMetadataAfterCreationWithGroupIdIsNull`
10324 /// itself asserts on
10325 /// `groupMetadata()`, which in Rust panics with the same message; that
10326 /// test is translated by `group_metadata_panics_without_group_id`.
10327 ///
10328 /// This test asserts on `commit_sync()`, not `group_metadata()`, so per
10329 /// the fixture refactor it keeps its post-construction `group_id = None`
10330 /// (it exercises the commit group-id guard, not the metadata slot).
10331 #[tokio::test]
10332 async fn group_metadata_groupless_commit_sync_emits_exact_java_message() {
10333 use crate::common::protocol::Errors;
10334 let (mut consumer, _handles) = make_test_consumer_with_channels();
10335 consumer.group_id = None;
10336 let err = consumer.commit_sync().await.expect_err("must err");
10337 assert_eq!(err.error(), Errors::InvalidGroupId, "expected InvalidGroupId, got {err:?}");
10338 assert_eq!(
10339 err.message(),
10340 "To use the group management or offset commit APIs, you must provide a valid \
10341 group.id in the consumer configuration."
10342 );
10343 }
10344
10345 /// Java: `testGroupMetadataAfterCreationWithGroupIdIsNotNull`
10346 /// On a consumer with
10347 /// `group.id` set, the initial `group_metadata()` returns:
10348 /// - `group_id` = configured value
10349 /// - `generation_id` = `UNKNOWN_GENERATION_ID` (-1)
10350 /// - `member_id` = `UNKNOWN_MEMBER_ID` ("")
10351 /// - `group_instance_id` = `None`
10352 #[tokio::test]
10353 async fn group_metadata_after_creation_with_group_id() {
10354 let consumer = make_test_consumer();
10355 let meta = consumer.group_metadata();
10356 assert_eq!(meta.group_id(), "test-group");
10357 assert_eq!(meta.generation_id(), -1);
10358 assert_eq!(meta.member_id(), "");
10359 assert_eq!(meta.group_instance_id(), None);
10360 }
10361
10362 /// Java: `testGroupMetadataAfterCreationWithGroupIdIsNotNullAndGroupInstanceIdSet`
10363 /// This is the Issue-C1
10364 /// regression: a freshly constructed consumer with `group.instance.id`
10365 /// set surfaces it from `group_metadata()` immediately — the production
10366 /// ctor seeds the slot at construction with the configured instance id
10367 /// (Java's `AsyncKafkaConsumer` constructor → `AsyncKafkaConsumer#initializeConsumerGroupMetadata`), alongside the
10368 /// UNKNOWN generation / member-id placeholders. No heartbeat / notifier
10369 /// call is needed.
10370 ///
10371 /// Pre-C1 the slot was `None` at construction and the getter returned a
10372 /// groupless stub with `group_instance_id = None`, so a static member's
10373 /// `TxnOffsetCommitRequest` omitted `GroupInstanceId` until the first
10374 /// heartbeat (`transaction_manager.rs:1832`).
10375 #[tokio::test]
10376 async fn group_metadata_with_instance_id() {
10377 let (consumer, _handles) = make_test_consumer_with_group(Some("test-group"), Some("groupInstanceId1"));
10378 let meta = consumer.group_metadata();
10379 assert_eq!(meta.group_id(), "test-group");
10380 assert_eq!(meta.group_instance_id(), Some("groupInstanceId1"));
10381 assert_eq!(meta.generation_id(), -1); // JoinGroupRequest.UNKNOWN_GENERATION_ID
10382 assert_eq!(meta.member_id(), ""); // JoinGroupRequest.UNKNOWN_MEMBER_ID
10383 }
10384
10385 /// Java: `testGroupMetadataUpdate`.
10386 /// The captured
10387 /// `MemberStateListener.onMemberEpochUpdated(Optional.of(42), "memberId")`
10388 /// updates the cached `group_metadata` to reflect the new epoch +
10389 /// member id, while carrying the group id / instance id over from the
10390 /// old value (Java `AsyncKafkaConsumer#updateGroupMetadata` maps over the old).
10391 #[tokio::test]
10392 async fn group_metadata_update_via_member_state_listener() {
10393 let consumer = make_test_consumer();
10394 let notifier = consumer.state_notifier();
10395 let old = consumer.group_metadata();
10396 assert_eq!(old.generation_id(), -1, "pre-condition: unknown generation");
10397 notifier.on_member_epoch_updated(Some(42), "memberId");
10398 let new = consumer.group_metadata();
10399 assert_eq!(new.group_id(), old.group_id());
10400 assert_eq!(new.member_id(), "memberId");
10401 assert_eq!(new.generation_id(), 42);
10402 assert_eq!(new.group_instance_id(), old.group_instance_id());
10403 }
10404
10405 /// Java: `testGroupMetadataIsResetAfterUnsubscribe`
10406 /// After `unsubscribe()`
10407 /// returns, the cached `group_metadata()` carries the original
10408 /// `group_id` + `group_instance_id` but the `generation_id` /
10409 /// `member_id` slots are reset to
10410 /// `JoinGroupRequest.UNKNOWN_GENERATION_ID` (-1) /
10411 /// `JoinGroupRequest.UNKNOWN_MEMBER_ID` ("").
10412 ///
10413 /// Mirrors `AsyncKafkaConsumer#resetGroupMetadata`
10414 /// called from `unsubscribe()`; Issue 21 fixup.
10415 #[tokio::test]
10416 async fn group_metadata_is_reset_after_unsubscribe() {
10417 let (mut consumer, handles) = make_test_consumer_with_channels();
10418 // Pre-condition: populate `group_metadata` cache as if a heartbeat
10419 // landed (Java has the bg-task `MemberStateListener` populate it
10420 // during the lifecycle; we simulate the same here).
10421 let notifier = consumer.state_notifier();
10422 notifier.on_member_epoch_updated(Some(42), "memberId");
10423 let pre = consumer.group_metadata();
10424 assert_eq!(pre.generation_id(), 42, "pre-condition: cache populated");
10425 assert_eq!(pre.member_id(), "memberId");
10426 assert_eq!(pre.group_id(), "test-group");
10427
10428 // Drive unsubscribe to completion.
10429 let completer = auto_complete_next_event(handles.app_event_rx);
10430 consumer.unsubscribe().await.expect("ok");
10431 let _ = completer.await.expect("task ok").expect("event received");
10432
10433 // Post-condition: group_metadata is reset — generation_id back to
10434 // -1, member_id back to "", but group_id preserved.
10435 let post = consumer.group_metadata();
10436 assert_eq!(post.generation_id(), -1, "generation_id reset to UNKNOWN");
10437 assert_eq!(post.member_id(), "", "member_id reset to UNKNOWN");
10438 assert_eq!(post.group_id(), "test-group", "group_id preserved");
10439 assert_eq!(post.group_instance_id(), None, "group_instance_id preserved");
10440 }
10441
10442 /// Issue C1 (strengthens `testGroupMetadataUpdate`,
10443 /// `AsyncKafkaConsumerTest#testGroupMetadataUpdate`): an epoch update PRESERVES the
10444 /// configured `group.instance.id`. Java's `updateGroupMetadata` maps over
10445 /// the old value and reuses `oldGroupMetadata.groupInstanceId()`
10446 /// (`AsyncKafkaConsumer#updateGroupMetadata`, specifically the
10447 /// `oldGroupMetadata.groupInstanceId()` argument). Java's own
10448 /// test can only assert `old == new` on an EMPTY instance id; seeding a
10449 /// non-empty instance id proves the instance id survives an epoch update —
10450 /// the exact value a static member's `TxnOffsetCommitRequest` carries.
10451 #[tokio::test]
10452 async fn group_metadata_epoch_update_preserves_instance_id() {
10453 let (consumer, _handles) = make_test_consumer_with_group(Some("test-group"), Some("groupInstanceId1"));
10454 let old = consumer.group_metadata();
10455 assert_eq!(old.group_instance_id(), Some("groupInstanceId1"), "pre-condition: seeded");
10456 assert_eq!(old.generation_id(), -1, "pre-condition: unknown generation");
10457
10458 let notifier = consumer.state_notifier();
10459 notifier.on_member_epoch_updated(Some(42), "memberId");
10460
10461 let new = consumer.group_metadata();
10462 assert_eq!(new.group_id(), old.group_id());
10463 assert_eq!(new.member_id(), "memberId");
10464 assert_eq!(new.generation_id(), 42);
10465 // The instance id carried over from the old value, unchanged.
10466 assert_eq!(new.group_instance_id(), old.group_instance_id());
10467 assert_eq!(new.group_instance_id(), Some("groupInstanceId1"));
10468 }
10469
10470 /// Issue C1 (strengthens `testGroupMetadataIsResetAfterUnsubscribe`,
10471 /// `AsyncKafkaConsumerTest#testGroupMetadataIsResetAfterUnsubscribe`): the unsubscribe reset PRESERVES
10472 /// the configured `group.instance.id`. `resetGroupMetadata()` maps old →
10473 /// `initializeConsumerGroupMetadata(old.groupId(), old.groupInstanceId())`
10474 /// (`AsyncKafkaConsumer#resetGroupMetadata`), so the instance id survives the
10475 /// reset. Java's own test uses an empty instance id, so it cannot tell
10476 /// "reset drops it" from "reset keeps it"; a non-empty instance id proves
10477 /// it is kept.
10478 #[tokio::test]
10479 async fn group_metadata_reset_after_unsubscribe_preserves_instance_id() {
10480 let (mut consumer, handles) = make_test_consumer_with_group(Some("test-group"), Some("groupInstanceId1"));
10481 // Populate the cache as if a heartbeat landed.
10482 let notifier = consumer.state_notifier();
10483 notifier.on_member_epoch_updated(Some(42), "memberId");
10484 let pre = consumer.group_metadata();
10485 assert_eq!(pre.generation_id(), 42, "pre-condition: cache populated");
10486 assert_eq!(pre.group_instance_id(), Some("groupInstanceId1"));
10487
10488 // Drive unsubscribe to completion.
10489 let completer = auto_complete_next_event(handles.app_event_rx);
10490 consumer.unsubscribe().await.expect("ok");
10491 let _ = completer.await.expect("task ok").expect("event received");
10492
10493 // Post-condition: generation / member reset to UNKNOWN, but group id
10494 // AND instance id preserved.
10495 let post = consumer.group_metadata();
10496 assert_eq!(post.generation_id(), -1, "generation_id reset to UNKNOWN");
10497 assert_eq!(post.member_id(), "", "member_id reset to UNKNOWN");
10498 assert_eq!(post.group_id(), "test-group", "group_id preserved");
10499 assert_eq!(
10500 post.group_instance_id(),
10501 Some("groupInstanceId1"),
10502 "group_instance_id preserved across reset"
10503 );
10504 }
10505
10506 /// Issue C1 behaviour change: on a consumer WITHOUT `group.id`, an
10507 /// epoch update leaves the slot empty. Java's `updateGroupMetadata`
10508 /// (`AsyncKafkaConsumer#updateGroupMetadata`) is an `Optional.map` over the old
10509 /// value, so an empty slot stays empty and `group_metadata()` keeps
10510 /// panicking with Java's `InvalidGroupIdException` message. This differs from the pre-C1 lazy-init
10511 /// notifier, which would have written a fresh entry on the first update —
10512 /// hence its own test.
10513 #[tokio::test]
10514 async fn group_metadata_epoch_update_on_groupless_consumer_is_a_noop() {
10515 let (consumer, _handles) = make_test_consumer_without_group_id();
10516 // Pre-condition: the slot is empty (no `group.id` → seeded `None`).
10517 assert!(consumer.group_metadata.lock().unwrap().is_none(), "pre-condition: empty slot");
10518
10519 let notifier = consumer.state_notifier();
10520 notifier.on_member_epoch_updated(Some(42), "m");
10521
10522 // The slot stayed `None` (map-over-old short-circuit) ...
10523 assert!(
10524 consumer.group_metadata.lock().unwrap().is_none(),
10525 "epoch update must not initialize the slot for a groupless consumer"
10526 );
10527 // ... and the getter still refuses a groupless consumer, as Java's
10528 // `groupMetadata()` throws `InvalidGroupIdException`
10529 // (`AsyncKafkaConsumer.java` `throwIfGroupIdNotDefined`).
10530 let panic = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| consumer.group_metadata()))
10531 .expect_err("group_metadata without group.id must panic");
10532 assert_eq!(
10533 panic.downcast_ref::<String>().map(String::as_str),
10534 Some(GROUP_ID_NOT_DEFINED_MSG)
10535 );
10536 }
10537
10538 /// Builds a consumer and closes it, answering the close-path events.
10539 async fn closed_test_consumer() -> AsyncKafkaConsumer<Vec<u8>, Vec<u8>> {
10540 closed_test_consumer_and_handle().await.0
10541 }
10542
10543 /// As [`closed_test_consumer`], also returning a handle taken before
10544 /// `close()`, as a listener captures one.
10545 async fn closed_test_consumer_and_handle() -> (AsyncKafkaConsumer<Vec<u8>, Vec<u8>>, ConsumerHandle) {
10546 let (mut consumer, mut handles) = make_test_consumer_with_channels();
10547 let handle = consumer.handle();
10548 tokio::spawn(async move {
10549 while let Some(env) = handles.app_event_rx.recv().await {
10550 match env.event {
10551 ApplicationEvent::LeaveGroupOnClose { handle, .. } => {
10552 handle.complete(());
10553 },
10554 ApplicationEvent::CommitSync { handle, offsets_ready, .. }
10555 | ApplicationEvent::CommitAsync { handle, offsets_ready, .. } => {
10556 offsets_ready.complete(());
10557 handle.complete(HashMap::new());
10558 },
10559 _ => {},
10560 }
10561 }
10562 });
10563 crate::consumer::Consumer::close(&mut consumer).await.expect("close");
10564 (consumer, handle)
10565 }
10566
10567 /// Asserts that `result` is the closed consumer's `IllegalStateError`,
10568 /// with Java's message.
10569 fn assert_closed_error<T: std::fmt::Debug>(method: &str, result: Result<T, Error>) {
10570 match result {
10571 Err(Error::LocalIllegalState(error)) => {
10572 assert_eq!(error.message(), CONSUMER_CLOSED_MSG, "{method}");
10573 },
10574 other => panic!("{method}: expected the closed consumer's IllegalStateError, got {other:?}"),
10575 }
10576 }
10577
10578 /// A rebalance listener that does nothing.
10579 struct NoopRebalanceListener;
10580 #[async_trait::async_trait]
10581 impl ConsumerRebalanceListener for NoopRebalanceListener {
10582 async fn on_partitions_revoked(&self, _partitions: &[TopicPartition]) -> Result<(), Error> {
10583 Ok(())
10584 }
10585 async fn on_partitions_assigned(&self, _partitions: &[TopicPartition]) -> Result<(), Error> {
10586 Ok(())
10587 }
10588 }
10589
10590 /// Java: `testFailOnClosedConsumer`, over every method Java guards with
10591 /// `acquireAndEnsureOpen()`: once `close()` has returned, each returns the
10592 /// closed consumer's `IllegalStateError`. The arguments are valid, so no
10593 /// check Java makes ahead of the guard (`seek`'s negative offset) answers
10594 /// first. The guarded sync getters panic instead
10595 /// (`assignment_panics_after_close` and its siblings). The unguarded
10596 /// `metrics`, `wakeup` and `enforce_rebalance` keep working, and a second
10597 /// `close()` returns without error, as Java's.
10598 #[tokio::test]
10599 async fn every_guarded_method_returns_illegal_state_after_close() {
10600 use crate::consumer::Consumer as C;
10601 let mut consumer = closed_test_consumer().await;
10602 let tp = TopicPartition::new("t", 0);
10603 let tps = [tp.clone()];
10604 let offsets = || HashMap::from([(tp.clone(), OffsetAndMetadata::new(1).unwrap())]);
10605 let timestamps = || HashMap::from([(tp.clone(), 0_i64)]);
10606 let timeout = Duration::from_secs(1);
10607 let listener = || Arc::new(NoopRebalanceListener) as Arc<dyn ConsumerRebalanceListener>;
10608 let callback = || Arc::new(NoopCallback) as Arc<dyn crate::consumer::OffsetCommitCallback>;
10609 let topics = || vec!["t".to_string()];
10610 let pattern = || SubscriptionPattern::new("t.*");
10611 let c = &mut consumer;
10612
10613 assert_closed_error("subscribe_with_topics", C::subscribe_with_topics(c, topics()).await);
10614 assert_closed_error(
10615 "subscribe_with_topics_listener",
10616 C::subscribe_with_topics_listener(c, topics(), listener()).await,
10617 );
10618 assert_closed_error("subscribe_with_pattern", C::subscribe_with_pattern(c, pattern()).await);
10619 assert_closed_error(
10620 "subscribe_with_pattern_listener",
10621 C::subscribe_with_pattern_listener(c, pattern(), listener()).await,
10622 );
10623 assert_closed_error("assign", C::assign(c, vec![tp.clone()]).await);
10624 assert_closed_error("unsubscribe", C::unsubscribe(c).await);
10625 assert_closed_error("poll", C::poll(c, timeout).await);
10626 assert_closed_error("commit_sync", C::commit_sync(c).await);
10627 assert_closed_error("commit_sync_with_timeout", C::commit_sync_with_timeout(c, timeout).await);
10628 assert_closed_error("commit_sync_with_offsets", C::commit_sync_with_offsets(c, offsets()).await);
10629 assert_closed_error(
10630 "commit_sync_with_offsets_timeout",
10631 C::commit_sync_with_offsets_timeout(c, offsets(), timeout).await,
10632 );
10633 assert_closed_error("commit_async", C::commit_async(c).await);
10634 assert_closed_error("commit_async_with_callback", C::commit_async_with_callback(c, callback()).await);
10635 assert_closed_error(
10636 "commit_async_with_offsets_callback",
10637 C::commit_async_with_offsets_callback(c, offsets(), callback()).await,
10638 );
10639 assert_closed_error("seek_with_offset", C::seek_with_offset(c, tp.clone(), 1).await);
10640 assert_closed_error(
10641 "seek_with_offset_and_metadata",
10642 C::seek_with_offset_and_metadata(c, tp.clone(), OffsetAndMetadata::new(1).unwrap()).await,
10643 );
10644 assert_closed_error("seek_to_beginning", C::seek_to_beginning(c, &tps).await);
10645 assert_closed_error("seek_to_end", C::seek_to_end(c, &tps).await);
10646 assert_closed_error("position", C::position(c, &tp).await);
10647 assert_closed_error("position_with_timeout", C::position_with_timeout(c, &tp, timeout).await);
10648 assert_closed_error("committed", C::committed(c, &tps).await);
10649 assert_closed_error("committed_with_timeout", C::committed_with_timeout(c, &tps, timeout).await);
10650 assert_closed_error("current_lag", C::current_lag(c, &tp).await);
10651 assert_closed_error("partitions_for", C::partitions_for(c, "t").await);
10652 assert_closed_error(
10653 "partitions_for_with_timeout",
10654 C::partitions_for_with_timeout(c, "t", timeout).await,
10655 );
10656 assert_closed_error("list_topics", C::list_topics(c).await);
10657 assert_closed_error("list_topics_with_timeout", C::list_topics_with_timeout(c, timeout).await);
10658 assert_closed_error("offsets_for_times", C::offsets_for_times(c, timestamps()).await);
10659 assert_closed_error(
10660 "offsets_for_times_with_timeout",
10661 C::offsets_for_times_with_timeout(c, timestamps(), timeout).await,
10662 );
10663 assert_closed_error("beginning_offsets", C::beginning_offsets(c, &tps).await);
10664 assert_closed_error(
10665 "beginning_offsets_with_timeout",
10666 C::beginning_offsets_with_timeout(c, &tps, timeout).await,
10667 );
10668 assert_closed_error("end_offsets", C::end_offsets(c, &tps).await);
10669 assert_closed_error("end_offsets_with_timeout", C::end_offsets_with_timeout(c, &tps, timeout).await);
10670 assert_closed_error("pause", C::pause(c, &tps).await);
10671 assert_closed_error("resume", C::resume(c, &tps).await);
10672
10673 // Java leaves these outside `acquireAndEnsureOpen()`.
10674 let _ = C::metrics(c);
10675 C::wakeup(c);
10676 C::enforce_rebalance(c).await.expect("enforce_rebalance only logs, as Java's");
10677 C::close(c).await.expect("a second close() is a no-op, as Java's");
10678 }
10679
10680 /// A [`ConsumerHandle`] taken before `close()` refuses the closed
10681 /// consumer as the consumer does: called outside a callback, where an
10682 /// open consumer's handle returns `ConcurrentModificationError`
10683 /// (`handle_async_ops_outside_a_callback_fail_with_concurrent_modification`),
10684 /// each async operation
10685 /// returns the closed consumer's `IllegalStateError` instead, as Java's
10686 /// `acquire()` passes when no thread is inside the consumer and
10687 /// `ensureOpen()` then throws. `wakeup` stays callable.
10688 #[tokio::test]
10689 async fn handle_async_methods_return_illegal_state_after_close() {
10690 let (_consumer, handle) = closed_test_consumer_and_handle().await;
10691 let ConsumerHandleInner::Async(state) = &handle.inner else {
10692 unreachable!("an AsyncKafkaConsumer hands out an Async handle")
10693 };
10694 assert_eq!(
10695 state.callbacks_in_progress.load(Ordering::Acquire),
10696 0,
10697 "the calls below are made outside a callback"
10698 );
10699 let tp = TopicPartition::new("t", 0);
10700 let tps = [tp.clone()];
10701 let offsets = || HashMap::from([(tp.clone(), OffsetAndMetadata::new(1).unwrap())]);
10702 let timestamps = || HashMap::from([(tp.clone(), 0_i64)]);
10703 let timeout = Duration::from_secs(1);
10704 let callback = || Arc::new(NoopCallback) as Arc<dyn crate::consumer::OffsetCommitCallback>;
10705 let h = &handle;
10706
10707 assert_closed_error("current_lag", h.current_lag(&tp).await);
10708 assert_closed_error("seek_with_offset", h.seek_with_offset(tp.clone(), 1).await);
10709 assert_closed_error(
10710 "seek_with_offset_and_metadata",
10711 h.seek_with_offset_and_metadata(tp.clone(), OffsetAndMetadata::new(1).unwrap())
10712 .await,
10713 );
10714 assert_closed_error("seek_to_beginning", h.seek_to_beginning(&tps).await);
10715 assert_closed_error("seek_to_end", h.seek_to_end(&tps).await);
10716 assert_closed_error("pause", h.pause(&tps).await);
10717 assert_closed_error("resume", h.resume(&tps).await);
10718 assert_closed_error("position", h.position(&tp).await);
10719 assert_closed_error("position_with_timeout", h.position_with_timeout(&tp, timeout).await);
10720 assert_closed_error("committed", h.committed(&tps).await);
10721 assert_closed_error("committed_with_timeout", h.committed_with_timeout(&tps, timeout).await);
10722 assert_closed_error("partitions_for", h.partitions_for("t").await);
10723 assert_closed_error("partitions_for_with_timeout", h.partitions_for_with_timeout("t", timeout).await);
10724 assert_closed_error("list_topics", h.list_topics().await);
10725 assert_closed_error("list_topics_with_timeout", h.list_topics_with_timeout(timeout).await);
10726 assert_closed_error("beginning_offsets", h.beginning_offsets(&tps).await);
10727 assert_closed_error(
10728 "beginning_offsets_with_timeout",
10729 h.beginning_offsets_with_timeout(&tps, timeout).await,
10730 );
10731 assert_closed_error("end_offsets", h.end_offsets(&tps).await);
10732 assert_closed_error("end_offsets_with_timeout", h.end_offsets_with_timeout(&tps, timeout).await);
10733 assert_closed_error("offsets_for_times", h.offsets_for_times(timestamps()).await);
10734 assert_closed_error(
10735 "offsets_for_times_with_timeout",
10736 h.offsets_for_times_with_timeout(timestamps(), timeout).await,
10737 );
10738 assert_closed_error("commit_sync", h.commit_sync().await);
10739 assert_closed_error("commit_sync_with_timeout", h.commit_sync_with_timeout(timeout).await);
10740 assert_closed_error("commit_sync_with_offsets", h.commit_sync_with_offsets(offsets()).await);
10741 assert_closed_error(
10742 "commit_sync_with_offsets_timeout",
10743 h.commit_sync_with_offsets_timeout(offsets(), timeout).await,
10744 );
10745 assert_closed_error("commit_async", h.commit_async().await);
10746 assert_closed_error("commit_async_with_callback", h.commit_async_with_callback(callback()).await);
10747 assert_closed_error(
10748 "commit_async_with_offsets_callback",
10749 h.commit_async_with_offsets_callback(offsets(), callback()).await,
10750 );
10751
10752 h.wakeup();
10753 }
10754
10755 /// The handle's `assignment()` panics on a closed consumer, as the
10756 /// consumer's own does.
10757 #[tokio::test]
10758 #[should_panic(expected = "This consumer has already been closed.")]
10759 async fn handle_assignment_panics_after_close() {
10760 let (_consumer, handle) = closed_test_consumer_and_handle().await;
10761 let _ = handle.assignment();
10762 }
10763
10764 /// The handle's `paused()` panics on a closed consumer.
10765 #[tokio::test]
10766 #[should_panic(expected = "This consumer has already been closed.")]
10767 async fn handle_paused_panics_after_close() {
10768 let (_consumer, handle) = closed_test_consumer_and_handle().await;
10769 let _ = handle.paused();
10770 }
10771
10772 /// The handle's `group_metadata()` panics on a closed consumer.
10773 #[tokio::test]
10774 #[should_panic(expected = "This consumer has already been closed.")]
10775 async fn handle_group_metadata_panics_after_close() {
10776 let (_consumer, handle) = closed_test_consumer_and_handle().await;
10777 let _ = handle.group_metadata();
10778 }
10779
10780 /// A listener that `close()` runs calls back into the consumer through
10781 /// its handle. Java's `close(CloseOptions)` sets `closed` only in its
10782 /// `finally`, "since the close function itself may trigger rebalance
10783 /// callback that needs the consumer to be open still". The listener's
10784 /// `commit_sync_with_offsets` succeeds and its async commit is sent, both
10785 /// before the leave (`runRebalanceCallbacksOnClose` precedes
10786 /// `leaveGroupOnClose`), and the async commit's callback runs before
10787 /// `close()` returns (`awaitPendingAsyncCommitsAndExecuteCommitCallbacks`).
10788 #[tokio::test]
10789 async fn listener_commits_through_handle_during_close() {
10790 use std::sync::atomic::AtomicUsize;
10791
10792 struct CountingCallback(Arc<AtomicUsize>);
10793 #[async_trait::async_trait]
10794 impl crate::consumer::OffsetCommitCallback for CountingCallback {
10795 async fn on_complete(
10796 &self,
10797 _offsets: &HashMap<TopicPartition, OffsetAndMetadata>,
10798 error: Option<&Error>,
10799 ) -> Result<(), Error> {
10800 assert!(error.is_none(), "the async commit failed: {error:?}");
10801 self.0.fetch_add(1, Ordering::SeqCst);
10802 Ok(())
10803 }
10804 }
10805
10806 struct CommittingListener {
10807 handle: ConsumerHandle,
10808 tp: TopicPartition,
10809 callback: Arc<dyn crate::consumer::OffsetCommitCallback>,
10810 results: Mutex<Vec<Result<(), Error>>>,
10811 }
10812 #[async_trait::async_trait]
10813 impl ConsumerRebalanceListener for CommittingListener {
10814 async fn on_partitions_assigned(&self, _partitions: &[TopicPartition]) -> Result<(), Error> {
10815 Ok(())
10816 }
10817 async fn on_partitions_revoked(&self, _partitions: &[TopicPartition]) -> Result<(), Error> {
10818 let at = |offset| HashMap::from([(self.tp.clone(), OffsetAndMetadata::new(offset).unwrap())]);
10819 let sync = self.handle.commit_sync_with_offsets(at(7)).await;
10820 let async_ = self
10821 .handle
10822 .commit_async_with_offsets_callback(at(8), Arc::clone(&self.callback))
10823 .await;
10824 self.results.lock().unwrap().extend([sync, async_]);
10825 Ok(())
10826 }
10827 }
10828
10829 let tp = TopicPartition::new("t", 0);
10830 let (mut consumer, mut handles) = make_test_consumer_with_channels();
10831 let events = Arc::new(Mutex::new(Vec::<String>::new()));
10832 let seen = Arc::clone(&events);
10833 let seen_tp = tp.clone();
10834 let drainer = tokio::spawn(async move {
10835 let offset_of = |offsets: &Option<HashMap<TopicPartition, OffsetAndMetadata>>| {
10836 offsets.as_ref().and_then(|o| o.get(&seen_tp)).map(OffsetAndMetadata::offset)
10837 };
10838 while let Some(env) = handles.app_event_rx.recv().await {
10839 match env.event {
10840 ApplicationEvent::LeaveGroupOnClose { handle, .. } => {
10841 seen.lock().unwrap().push("leave".to_string());
10842 handle.complete(());
10843 },
10844 ApplicationEvent::CommitSync { handle, offsets_ready, offsets } => {
10845 if let Some(offset) = offset_of(&offsets) {
10846 seen.lock().unwrap().push(format!("commit_sync {offset}"));
10847 }
10848 offsets_ready.complete(());
10849 handle.complete(offsets.unwrap_or_default());
10850 },
10851 ApplicationEvent::CommitAsync { handle, offsets_ready, offsets } => {
10852 if let Some(offset) = offset_of(&offsets) {
10853 seen.lock().unwrap().push(format!("commit_async {offset}"));
10854 }
10855 offsets_ready.complete(());
10856 handle.complete(offsets.unwrap_or_default());
10857 },
10858 _ => {},
10859 }
10860 }
10861 });
10862
10863 let invoked = Arc::new(AtomicUsize::new(0));
10864 let listener = Arc::new(CommittingListener {
10865 handle: consumer.handle(),
10866 tp: tp.clone(),
10867 callback: Arc::new(CountingCallback(Arc::clone(&invoked))),
10868 results: Mutex::new(Vec::new()),
10869 });
10870 *consumer.rebalance_listener.lock().unwrap() =
10871 Some(Arc::clone(&listener) as Arc<dyn ConsumerRebalanceListener>);
10872 // An assigned member with a live epoch, so `close()` runs
10873 // `on_partitions_revoked` (see
10874 // `run_rebalance_callbacks_on_close_invokes_revoked_on_live_epoch`).
10875 let notifier = consumer.state_notifier();
10876 notifier.on_group_assignment_updated(&HashSet::from([tp.clone()]));
10877 notifier.on_member_epoch_updated(Some(5), "member-1");
10878
10879 crate::consumer::Consumer::close(&mut consumer).await.expect("close");
10880
10881 let results = std::mem::take(&mut *listener.results.lock().unwrap());
10882 assert_eq!(results.len(), 2, "on_partitions_revoked ran once and committed twice");
10883 for result in results {
10884 result.expect("a commit through the handle during close() succeeds");
10885 }
10886 assert_eq!(invoked.load(Ordering::SeqCst), 1, "close() ran the async commit's callback");
10887 assert_eq!(
10888 *events.lock().unwrap(),
10889 ["commit_sync 7", "commit_async 8", "leave"],
10890 "the listener's commits precede the leave"
10891 );
10892 drop(drainer);
10893 }
10894
10895 /// Java's `assignment()` runs `acquireAndEnsureOpen()`, which throws
10896 /// `IllegalStateException` on a closed consumer.
10897 #[tokio::test]
10898 #[should_panic(expected = "This consumer has already been closed.")]
10899 async fn assignment_panics_after_close() {
10900 let mut consumer = closed_test_consumer().await;
10901 let _ = crate::consumer::Consumer::assignment(&mut consumer);
10902 }
10903
10904 /// Java's `subscription()` runs `acquireAndEnsureOpen()`.
10905 #[tokio::test]
10906 #[should_panic(expected = "This consumer has already been closed.")]
10907 async fn subscription_panics_after_close() {
10908 let mut consumer = closed_test_consumer().await;
10909 let _ = crate::consumer::Consumer::subscription(&mut consumer);
10910 }
10911
10912 /// Java's `paused()` runs `acquireAndEnsureOpen()`.
10913 #[tokio::test]
10914 #[should_panic(expected = "This consumer has already been closed.")]
10915 async fn paused_panics_after_close() {
10916 let mut consumer = closed_test_consumer().await;
10917 let _ = crate::consumer::Consumer::paused(&mut consumer);
10918 }
10919
10920 /// Java's `groupMetadata()` runs `acquireAndEnsureOpen()`.
10921 #[tokio::test]
10922 #[should_panic(expected = "This consumer has already been closed.")]
10923 async fn group_metadata_panics_after_close() {
10924 let mut consumer = closed_test_consumer().await;
10925 let _ = crate::consumer::Consumer::group_metadata(&mut consumer);
10926 }
10927
10928 /// Java's `groupMetadata()` throws `InvalidGroupIdException` without a
10929 /// `group.id` (`throwIfGroupIdNotDefined`).
10930 #[tokio::test]
10931 #[should_panic(
10932 expected = "To use the group management or offset commit APIs, you must provide a valid group.id in the consumer configuration."
10933 )]
10934 async fn group_metadata_panics_without_group_id() {
10935 let (mut consumer, _handles) = make_test_consumer_without_group_id();
10936 let _ = crate::consumer::Consumer::group_metadata(&mut consumer);
10937 }
10938
10939 /// `ConsumerHandle::group_metadata` runs the consumer's own
10940 /// `throwIfGroupIdNotDefined` check, so without a `group.id` it panics
10941 /// with the same `InvalidGroupIdException` message.
10942 #[tokio::test]
10943 #[should_panic(
10944 expected = "To use the group management or offset commit APIs, you must provide a valid group.id in the consumer configuration."
10945 )]
10946 async fn consumer_handle_group_metadata_panics_without_group_id() {
10947 let (consumer, _handles) = make_test_consumer_without_group_id();
10948 let _ = consumer.handle().group_metadata();
10949 }
10950
10951 /// Java: `testSubscribeGeneratesEvent` —
10952 /// asserts the subscribe call results in the subscription being
10953 /// reflected on `consumer.subscription()` AND the AssignmentChange
10954 /// remains empty. The existing `subscribe_generates_topic_subscription_change_event`
10955 /// (inline above) asserts only event-shape; this test extends
10956 /// coverage to the post-state assertions.
10957 #[tokio::test]
10958 async fn subscribe_reflects_subscription_state_after_event_completes() {
10959 let (mut consumer, handles) = make_test_consumer_with_channels();
10960 let topic = "topic1";
10961 let subs = Arc::clone(&handles.subscriptions);
10962 let completer = {
10963 let topic = topic.to_string();
10964 let mut rx = handles.app_event_rx;
10965 tokio::spawn(async move {
10966 while let Some(env) = rx.recv().await {
10967 if let ApplicationEvent::TopicSubscriptionChange { handle, topics, listener } = env.event {
10968 let mut s = subs.lock().unwrap();
10969 s.subscribe_with_topics(topics, listener).expect("subscribe ok");
10970 drop(s);
10971 handle.complete(());
10972 return Some(topic);
10973 }
10974 }
10975 None
10976 })
10977 };
10978 consumer.subscribe_with_topics(vec![topic.to_string()]).await.expect("ok");
10979 let _ = completer.await.expect("task ok").expect("event received");
10980 let subscription = consumer.subscription();
10981 assert_eq!(subscription.len(), 1);
10982 assert!(subscription.contains(topic));
10983 assert!(consumer.assignment().is_empty());
10984 }
10985
10986 /// Java: `testSubscribeToRe2JPatternThrowsIfNoGroupId`
10987 /// The Re2J pattern subscribe path requires
10988 /// a configured `group.id`; without it, the call errors with the
10989 /// Rust analog of `InvalidGroupIdException` —
10990 /// `Error::invalid_group_id(...)` which surfaces a
10991 /// `KafkaError` variant carrying `Errors::InvalidGroupId` (Issue 16).
10992 #[tokio::test]
10993 async fn subscribe_subscription_pattern_without_group_id_errors() {
10994 use crate::common::protocol::Errors;
10995 let (mut consumer, _handles) = make_test_consumer_with_channels();
10996 consumer.group_id = None;
10997 let err = consumer
10998 .subscribe_with_pattern(SubscriptionPattern::new("t*"))
10999 .await
11000 .expect_err("must err");
11001 assert_eq!(err.error(), Errors::InvalidGroupId, "expected InvalidGroupId, got {err:?}");
11002 }
11003
11004 /// Java: `testUnsubscribeWithoutGroupId` —
11005 /// `unsubscribe()` on a groupless consumer enqueues an
11006 /// `UnsubscribeEvent` (does NOT require `group.id`). Issue 26:
11007 /// uses `make_test_consumer_without_group_id` so the consumer is
11008 /// constructed groupless from the start (mirrors Java's
11009 /// no-group-id ctor) rather than mutating the field post-hoc.
11010 #[tokio::test]
11011 async fn unsubscribe_without_group_id_enqueues_event() {
11012 let (mut consumer, handles) = make_test_consumer_without_group_id();
11013 assert!(consumer.group_id.is_none(), "pre-condition: groupless");
11014 let completer = auto_complete_next_event(handles.app_event_rx);
11015 consumer.unsubscribe().await.expect("ok");
11016 let env = completer.await.expect("task ok").expect("event received");
11017 assert!(matches!(env.event, ApplicationEvent::Unsubscribe { .. }));
11018 }
11019
11020 /// Java: `testGroupRemoteAssignorUnusedIfGroupIdUndefined`
11021 /// With `group.id` undefined,
11022 /// `group.remote.assignor` is unused by the consumer config.
11023 ///
11024 /// Issue 27 — strengthened: constructs a TRUE groupless consumer
11025 /// (via `make_test_consumer_without_group_id`) and asserts the
11026 /// constructor accepts `group_remote_assignor` without producing
11027 /// any observable side effect on the consumer's group-management
11028 /// surface:
11029 /// - `group_id` slot is `None`.
11030 /// - `group_metadata()` falls through to the stub
11031 /// (`group_id == ""`) — no member id was assigned.
11032 /// - `assignment()` is empty pre-subscribe.
11033 ///
11034 /// The Java `config.unused()` set itself is not yet tracked in
11035 /// Rust's `ConsumerConfig`; that piece of the contract is
11036 /// deferred. See `group_id_null_constructs_successfully` for the
11037 /// pure construction-survives assertion.
11038 #[tokio::test]
11039 async fn group_remote_assignor_unused_if_group_id_undefined() {
11040 let (consumer, _handles) = make_test_consumer_without_group_id();
11041 // Pre-condition: the consumer is truly groupless (not just
11042 // post-mutation — the fixture clears both the slot AND the
11043 // underlying config). Issue 27.
11044 assert!(consumer.group_id.is_none());
11045 assert!(consumer.config.group_id.is_none());
11046 // The group-management surface stays inert.
11047 assert!(consumer.assignment().is_empty());
11048 assert!(consumer.subscription().is_empty());
11049 // Java `unused()` set tracking is a `ConsumerConfig`-side
11050 // concern; the Rust analog would assert
11051 // `config.unused().contains(GROUP_REMOTE_ASSIGNOR_CONFIG)`.
11052 // Until that surface lands, the strongest behavioural
11053 // assertion is the one above: groupless consumer constructs
11054 // cleanly without group-management state leaking through.
11055 drop(consumer);
11056 }
11057
11058 /// Java: `testGroupIdNull`. With group.id
11059 /// null, certain config keys (`AUTO_COMMIT_INTERVAL_MS_CONFIG`,
11060 /// `THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED`) flow through to the
11061 /// underlying ConsumerConfig as "used" (Java asserts
11062 /// `!config.unused().contains(...)`).
11063 ///
11064 /// Rust divergence: same as the assignor test above — Rust's
11065 /// `ConsumerConfig` does not track `unused()`. This test stands in
11066 /// for the construction-time assertion that a groupless consumer
11067 /// can be built with these config keys set.
11068 #[tokio::test]
11069 async fn group_id_null_constructs_successfully() {
11070 let (consumer, _handles) = make_test_consumer_with_channels();
11071 // Equivalent of Java's pre-condition: ctor succeeded.
11072 drop(consumer);
11073 }
11074
11075 /// Java: `testGroupIdNotNullAndValid`. With
11076 /// a valid group.id and auto-commit disabled, the consumer
11077 /// constructs successfully and the auto-commit config key is left
11078 /// unused (Java asserts `config.unused().contains(AUTO_COMMIT_INTERVAL_MS_CONFIG)`).
11079 ///
11080 /// Rust divergence: see notes above; this test stands in for the
11081 /// construction-time assertion.
11082 #[tokio::test]
11083 async fn group_id_not_null_constructs_with_auto_commit_disabled() {
11084 let (mut consumer, _handles) = make_test_consumer_with_channels();
11085 // group_id is `Some` in the fixture.
11086 assert!(consumer.group_id.is_some());
11087 // Force auto-commit off and validate the consumer survives.
11088 consumer.auto_commit_enabled = false;
11089 assert!(!consumer.auto_commit_enabled);
11090 drop(consumer);
11091 }
11092
11093 /// Java: `testEnsurePollEventSentOnConsumerPoll`
11094 /// Asserts at least one `AsyncPollEvent` is
11095 /// submitted by `consumer.poll(...)`. Inline test
11096 /// `poll_enqueues_async_poll_event_and_clears_on_completion`
11097 /// already asserts this on a manual-assign consumer; this test
11098 /// stays parallel to the Java assertion shape for clarity.
11099 #[tokio::test]
11100 async fn ensure_poll_event_sent_on_consumer_poll() {
11101 let (mut consumer, handles) = make_test_consumer_with_channels();
11102 let tp = TopicPartition::new("topic".to_string(), 0);
11103 {
11104 let mut subs = consumer.subscriptions.lock().unwrap();
11105 let mut assigned: HashSet<TopicPartition> = HashSet::new();
11106 assigned.insert(tp);
11107 subs.assign_from_user(assigned).unwrap();
11108 }
11109 let mut app_rx = handles.app_event_rx;
11110 // Drain in a task — first AsyncPoll envelope is what we look for.
11111 let saw_async_poll = tokio::spawn(async move {
11112 while let Some(env) = app_rx.recv().await {
11113 if matches!(env.event, ApplicationEvent::AsyncPoll { .. }) {
11114 return true;
11115 }
11116 }
11117 false
11118 });
11119 let _ = consumer.poll(Duration::from_millis(0)).await;
11120 // Give the drainer a moment via its own task.
11121 drop(consumer); // forces all senders to close eventually
11122 let observed = saw_async_poll.await.expect("task ok");
11123 assert!(observed, "poll() must submit at least one AsyncPollEvent");
11124 }
11125
11126 /// Error event is drained and surfaces from
11127 /// `process_background_events`.
11128 #[tokio::test]
11129 async fn process_background_events_surfaces_error_event() {
11130 let (mut consumer, handles) = make_test_consumer_with_channels();
11131 let env =
11132 BackgroundEventEnvelope { event: BackgroundEvent::Error { error: Error::timeout("boom") }, enqueued_ms: 0 };
11133 handles.bg_event_tx.send(env).expect("send ok");
11134 let result = consumer.process_background_events().await;
11135 assert!(matches!(result, Err(Error::Timeout(_))));
11136 }
11137
11138 /// Callback-needed event with no listener registered: succeeds and
11139 /// the ack is sent with `Ok(())` — mirrors Java's
11140 /// `listener.isPresent() == false` no-op branch.
11141 #[tokio::test]
11142 async fn process_background_events_with_no_listener_acks_ok() {
11143 use crate::consumer::internals::ConsumerRebalanceListenerMethodName;
11144 use tokio::sync::oneshot;
11145 let (mut consumer, handles) = make_test_consumer_with_channels();
11146 // AK 4.3.1: the revoke path (`PartitionsRemoved`) exercises the same
11147 // no-listener ack machinery without needing a bg AEP to process an
11148 // ApplyAssignmentEvent (the assign path's `PartitionsAssigned` would).
11149 let (ack_tx, ack_rx) = oneshot::channel::<Result<(), Error>>();
11150 let env = BackgroundEventEnvelope {
11151 event: BackgroundEvent::PartitionsRemoved {
11152 method_name: ConsumerRebalanceListenerMethodName::OnPartitionsRevoked,
11153 partitions: vec![TopicPartition::new("t".to_string(), 0)],
11154 ack: ack_tx,
11155 ack_through_application_events: false,
11156 },
11157 enqueued_ms: 0,
11158 };
11159 handles.bg_event_tx.send(env).expect("send ok");
11160 consumer.process_background_events().await.expect("ok");
11161 // The ack must have been resolved with `Ok(())`.
11162 let ack_result = ack_rx.await.expect("ack received");
11163 assert!(ack_result.is_ok());
11164 }
11165
11166 /// Callback-needed event with a registered listener: the listener's
11167 /// callback is invoked inline on the caller's task, and the ack is sent
11168 /// with the listener's result. (AK 4.3.1: uses the revoke path
11169 /// `PartitionsRemoved`, which invokes the listener directly without a bg
11170 /// AEP; the assign path's `PartitionsAssigned` would require one to
11171 /// process the ApplyAssignmentEvent.)
11172 #[tokio::test]
11173 async fn process_background_events_invokes_registered_listener() {
11174 use crate::consumer::internals::ConsumerRebalanceListenerMethodName;
11175 use async_trait::async_trait;
11176 use std::sync::atomic::AtomicUsize;
11177 use tokio::sync::oneshot;
11178
11179 struct RecordingListener {
11180 count: AtomicUsize,
11181 }
11182 #[async_trait]
11183 impl ConsumerRebalanceListener for RecordingListener {
11184 async fn on_partitions_assigned(&self, _: &[TopicPartition]) -> Result<(), Error> {
11185 Ok(())
11186 }
11187 async fn on_partitions_revoked(&self, _: &[TopicPartition]) -> Result<(), Error> {
11188 self.count.fetch_add(1, Ordering::SeqCst);
11189 Ok(())
11190 }
11191 async fn on_partitions_lost(&self, _: &[TopicPartition]) -> Result<(), Error> {
11192 Ok(())
11193 }
11194 }
11195
11196 let (mut consumer, handles) = make_test_consumer_with_channels();
11197 let listener: Arc<RecordingListener> = Arc::new(RecordingListener { count: AtomicUsize::new(0) });
11198 *consumer.rebalance_listener.lock().unwrap() =
11199 Some(Arc::clone(&listener) as Arc<dyn ConsumerRebalanceListener>);
11200
11201 let (ack_tx, ack_rx) = oneshot::channel::<Result<(), Error>>();
11202 let env = BackgroundEventEnvelope {
11203 event: BackgroundEvent::PartitionsRemoved {
11204 method_name: ConsumerRebalanceListenerMethodName::OnPartitionsRevoked,
11205 partitions: vec![TopicPartition::new("t".to_string(), 0)],
11206 ack: ack_tx,
11207 ack_through_application_events: false,
11208 },
11209 enqueued_ms: 0,
11210 };
11211 handles.bg_event_tx.send(env).expect("send ok");
11212 consumer.process_background_events().await.expect("ok");
11213 assert_eq!(listener.count.load(Ordering::SeqCst), 1);
11214 assert!(ack_rx.await.expect("ack received").is_ok());
11215
11216 // Silence the unused-field warning for `subscriptions` on the
11217 // test handles (used here so the helper struct stays
11218 // forward-compatible with future tests that need to inspect the
11219 // shared state).
11220 drop(handles.subscriptions);
11221 }
11222
11223 /// Phase 41 Issue 3: the ack-send path in `process_background_events`
11224 /// must poke the bg-task wakeup `Notify` so the bg loop observes the ack
11225 /// promptly (rather than waiting out the selector poll timeout). The
11226 /// non-Docker component tests otherwise busy-drive the membership loop and
11227 /// never exercise this poke — removing it after `ack.send(...)` would fail
11228 /// no local test without this one.
11229 ///
11230 /// It must poke the **application-event notify**, and must NOT touch the
11231 /// wakeup token (`network_thread_close.wakeup()` →
11232 /// `WakeupTrigger::wakeup()`). The token is Java's
11233 /// `KafkaConsumer.wakeup()`: cancelling it arms a user-visible
11234 /// `Error::Wakeup` that the next public API call raises (§11). Since
11235 /// every rebalance fires a listener callback, poking the token here made a
11236 /// spurious `Wakeup` the normal outcome of any rebalance — it broke
11237 /// `poll()` for every consumer with a listener registered, across 12
11238 /// integration tests, while the earlier version of this test passed
11239 /// because the fixture's wakeup fn is a flag-setting stub that does not
11240 /// reproduce production's token cancellation.
11241 ///
11242 /// So this asserts both halves: the notify got its permit, AND no
11243 /// user-visible wakeup is pending afterwards.
11244 #[tokio::test]
11245 async fn process_background_events_ack_pokes_bg_notify_not_user_wakeup() {
11246 use crate::consumer::internals::ConsumerRebalanceListenerMethodName;
11247 use async_trait::async_trait;
11248 use tokio::sync::oneshot;
11249
11250 struct NoopListener;
11251 #[async_trait]
11252 impl ConsumerRebalanceListener for NoopListener {
11253 async fn on_partitions_assigned(&self, _: &[TopicPartition]) -> Result<(), Error> {
11254 Ok(())
11255 }
11256 async fn on_partitions_revoked(&self, _: &[TopicPartition]) -> Result<(), Error> {
11257 Ok(())
11258 }
11259 async fn on_partitions_lost(&self, _: &[TopicPartition]) -> Result<(), Error> {
11260 Ok(())
11261 }
11262 }
11263
11264 let (mut consumer, handles) = make_test_consumer_with_channels();
11265 *consumer.rebalance_listener.lock().unwrap() =
11266 Some(Arc::new(NoopListener) as Arc<dyn ConsumerRebalanceListener>);
11267
11268 // Nothing must have fired before the callback is processed.
11269 assert!(
11270 tokio::time::timeout(Duration::from_millis(50), handles.event_notify.notified())
11271 .await
11272 .is_err(),
11273 "the application-event notify must not be poked before the callback ack is sent",
11274 );
11275 assert!(
11276 consumer.wakeup_trigger.maybe_trigger_wakeup().is_ok(),
11277 "no user-visible wakeup may be pending before the callback",
11278 );
11279
11280 let (ack_tx, ack_rx) = oneshot::channel::<Result<(), Error>>();
11281 let env = BackgroundEventEnvelope {
11282 event: BackgroundEvent::PartitionsRemoved {
11283 method_name: ConsumerRebalanceListenerMethodName::OnPartitionsRevoked,
11284 partitions: vec![TopicPartition::new("t".to_string(), 0)],
11285 ack: ack_tx,
11286 ack_through_application_events: false,
11287 },
11288 enqueued_ms: 0,
11289 };
11290 handles.bg_event_tx.send(env).expect("send ok");
11291 consumer.process_background_events().await.expect("ok");
11292 assert!(ack_rx.await.expect("ack received").is_ok());
11293
11294 // The ack-send path must have poked the application-event notify.
11295 // `notify_one()` stores a permit when nobody is parked, so an
11296 // already-satisfied `notified()` resolves immediately.
11297 tokio::time::timeout(Duration::from_secs(1), handles.event_notify.notified())
11298 .await
11299 .expect("process_background_events must poke the bg notify after sending the listener ack");
11300
11301 // ...and must NOT have armed a user-visible wakeup. This is the
11302 // regression: with `network_thread_close.wakeup()` here, the next
11303 // `poll()` / `commit_sync()` / `position()` fails with
11304 // `Wakeup` (Java's message-less `WakeupException`) even though the user never called
11305 // `wakeup()`.
11306 assert!(
11307 !handles.bg_wakeup_called.load(Ordering::Acquire),
11308 "the ack poke must not fire the user-facing wakeup fn",
11309 );
11310 assert!(
11311 consumer.wakeup_trigger.maybe_trigger_wakeup().is_ok(),
11312 "the ack poke must not arm a user-visible Error::Wakeup",
11313 );
11314 drop(handles.subscriptions);
11315 }
11316
11317 /// AK 4.3.1 (KAFKA-20428): when `skip_assignment_events` is set (the
11318 /// unsubscribe path), a pending `PartitionsAssigned` event is NOT
11319 /// applied — its ack is completed EXCEPTIONALLY to unblock the bg
11320 /// reconciliation, with the message Java uses. Translated from
11321 /// `AsyncKafkaConsumerTest#testUnsubscribeWithPendingAssignmentEvent`.
11322 #[tokio::test]
11323 async fn process_background_events_skips_pending_partitions_assigned_when_unsubscribing() {
11324 use tokio::sync::oneshot;
11325 let (mut consumer, handles) = make_test_consumer_with_channels();
11326
11327 let (ack_tx, ack_rx) = oneshot::channel::<Result<(), Error>>();
11328 handles
11329 .bg_event_tx
11330 .send(BackgroundEventEnvelope {
11331 event: BackgroundEvent::PartitionsAssigned {
11332 assigned_partitions: vec![TopicPartition::new("t".to_string(), 0)],
11333 added_partitions: vec![TopicPartition::new("t".to_string(), 0)],
11334 ack: ack_tx,
11335 },
11336 enqueued_ms: 0,
11337 })
11338 .expect("send ok");
11339
11340 // skip_assignment_events = true (unsubscribe path).
11341 consumer
11342 .process_background_events_inner(/* skip_assignment_events = */ true)
11343 .await
11344 .expect("ok — the skipped assignment event is not an app-side error");
11345
11346 // The pending assignment event was completed exceptionally.
11347 let ack_result = ack_rx.await.expect("ack received");
11348 match ack_result {
11349 Err(err) => assert!(
11350 err.to_string()
11351 .contains("Assignment event skipped because consumer is unsubscribing"),
11352 "unexpected skip error: {err}",
11353 ),
11354 Ok(()) => panic!("PartitionsAssigned must be completed with an error when skipping assignment events"),
11355 }
11356 drop(handles.subscriptions);
11357 }
11358
11359 /// AK 4.3.1 (KAFKA-20382): if applying the new assignment
11360 /// (`ApplyAssignmentEvent`) fails, the `PartitionsAssigned` event is
11361 /// completed exceptionally (a background error is surfaced) so the bg
11362 /// reconciliation can complete. Translated from
11363 /// `AsyncKafkaConsumerTest#testPartitionsAssignedEventSendsErrorWhenApplyAssignmentFails`.
11364 #[tokio::test]
11365 async fn partitions_assigned_event_sends_error_when_apply_assignment_fails() {
11366 use tokio::sync::oneshot;
11367 let (mut consumer, mut handles) = make_test_consumer_with_channels();
11368 let bg_event_tx = handles.bg_event_tx.clone();
11369
11370 // Fake bg: fail the ApplyAssignmentEvent that applyNewAssignment sends.
11371 let fake_bg = tokio::spawn(async move {
11372 let env = handles.app_event_rx.recv().await.expect("ApplyAssignmentEvent envelope");
11373 match env.event {
11374 ApplicationEvent::ApplyAssignment { handle, .. } => {
11375 handle.complete_with_error(Error::local_illegal_state("apply failed"));
11376 },
11377 other => panic!("expected ApplyAssignment, got {}", other.type_name()),
11378 }
11379 handles
11380 });
11381
11382 let (ack_tx, ack_rx) = oneshot::channel::<Result<(), Error>>();
11383 bg_event_tx
11384 .send(BackgroundEventEnvelope {
11385 event: BackgroundEvent::PartitionsAssigned {
11386 assigned_partitions: vec![TopicPartition::new("t".to_string(), 0)],
11387 added_partitions: vec![TopicPartition::new("t".to_string(), 0)],
11388 ack: ack_tx,
11389 },
11390 enqueued_ms: 0,
11391 })
11392 .expect("send ok");
11393
11394 // process_background_events records the wrapped apply error as the
11395 // first error and returns it.
11396 let result = consumer.process_background_events().await;
11397 assert!(result.is_err(), "apply-assignment failure must surface as an app-side error");
11398
11399 // The PartitionsAssigned ack was completed with the wrapped error.
11400 let ack_result = ack_rx.await.expect("ack received");
11401 match ack_result {
11402 Err(err) => assert!(
11403 err.to_string().contains("Failed to apply the new assignment"),
11404 "unexpected apply error: {err}",
11405 ),
11406 Ok(()) => panic!("PartitionsAssigned must be completed with an error when apply fails"),
11407 }
11408
11409 let handles = fake_bg.await.expect("fake bg joins");
11410 drop(handles.subscriptions);
11411 }
11412
11413 /// AK 4.3.1 (KAFKA-20106): `collect_fetch` does NOT wait for the
11414 /// reconciliation check when there is no pending reconciliation.
11415 /// Translated from
11416 /// `AsyncKafkaConsumerTest#testPollDoesNotWaitForReconciliationCheckIfNoPendingReconciliation`.
11417 #[tokio::test]
11418 async fn wait_reconciliation_check_returns_true_when_no_pending_reconciliation() {
11419 let (consumer, handles) = make_test_consumer_with_channels();
11420 // has_pending_reconciliation defaults to false.
11421 assert!(consumer.wait_reconciliation_check().await.expect("no wakeup"));
11422 drop(handles.subscriptions);
11423 }
11424
11425 /// The check passes through immediately when it is already complete.
11426 #[tokio::test]
11427 async fn wait_reconciliation_check_returns_true_when_already_complete() {
11428 use crate::consumer::internals::events::AsyncPollState;
11429 let (mut consumer, handles) = make_test_consumer_with_channels();
11430 consumer.has_pending_reconciliation.store(true, Ordering::Release);
11431 let state = Arc::new(AsyncPollState::new());
11432 state.mark_reconciliation_check_complete();
11433 consumer.inflight_poll = Some(InflightPoll { deadline_ms: i64::MAX, state });
11434 assert!(consumer.wait_reconciliation_check().await.expect("no wakeup"));
11435 drop(handles.subscriptions);
11436 }
11437
11438 /// AK 4.3.1 (KAFKA-20106): `collect_fetch` waits until the reconciliation
11439 /// check completes when there is a pending reconciliation. Translated from
11440 /// `AsyncKafkaConsumerTest#testPollWaitsForReconciliationCheckComplete`.
11441 #[tokio::test]
11442 async fn wait_reconciliation_check_waits_then_proceeds_when_completed() {
11443 use crate::consumer::internals::events::AsyncPollState;
11444 let (mut consumer, handles) = make_test_consumer_with_channels();
11445 consumer.has_pending_reconciliation.store(true, Ordering::Release);
11446 let state = Arc::new(AsyncPollState::new());
11447 consumer.inflight_poll = Some(InflightPoll { deadline_ms: i64::MAX, state: Arc::clone(&state) });
11448 // Complete the check from another task shortly after.
11449 let s = Arc::clone(&state);
11450 tokio::spawn(async move {
11451 tokio::time::sleep(Duration::from_millis(20)).await;
11452 s.mark_reconciliation_check_complete();
11453 });
11454 assert!(
11455 consumer.wait_reconciliation_check().await.expect("no wakeup"),
11456 "must proceed once the check completes"
11457 );
11458 drop(handles.subscriptions);
11459 }
11460
11461 /// Java `AsyncKafkaConsumer#collectFetch` — consults the
11462 /// SHARED [`PositionsValidator`], so an error the background side cached
11463 /// through `OffsetFetcherUtils` surfaces on the application task.
11464 ///
11465 /// This is the regression test for the wiring: before
11466 /// `PositionsValidator` was split back out of `OffsetFetcherUtils`, the
11467 /// consumer had no handle on that state at all and
11468 /// `canSkipUpdateFetchPositions` could not be translated.
11469 #[tokio::test]
11470 async fn collect_fetch_propagates_a_pending_validate_positions_error() {
11471 use crate::common::protocol::Errors;
11472 let (consumer, handles) = make_test_consumer_with_channels();
11473 consumer
11474 .positions_validator
11475 .maybe_set_error(Error::new(Errors::UnknownServerError));
11476
11477 let err = consumer
11478 .collect_fetch()
11479 .await
11480 .expect_err("the cached validation error must reach the app task");
11481 assert_eq!(err.error(), Errors::UnknownServerError);
11482 drop(handles.subscriptions);
11483 }
11484
11485 /// Java `AsyncKafkaConsumer#collectFetch` — when the skip check fails, the
11486 /// application task must NOT collect from the fetch buffer until the
11487 /// in-flight poll reports that it has finished validating positions.
11488 /// Java's own comment: this "prevents a race condition where both
11489 /// threads may attempt to update the `SubscriptionState.position()` for
11490 /// a given partition".
11491 ///
11492 /// The buffer staying full is the observable: a collected fetch is
11493 /// drained out of it.
11494 #[tokio::test]
11495 async fn collect_fetch_does_not_drain_the_buffer_until_positions_are_validated() {
11496 use crate::consumer::internals::CompletedFetch;
11497 use crate::consumer::internals::events::AsyncPollState;
11498 use crate::fetch_response_data::PartitionData;
11499
11500 let (mut consumer, handles) = make_test_consumer_with_channels();
11501 // A fresh validator still holds the -1 metadata-version sentinel, so
11502 // `can_skip_update_fetch_positions()` is false and the gate applies.
11503 assert!(
11504 !consumer
11505 .positions_validator
11506 .can_skip_update_fetch_positions()
11507 .expect("no cached error")
11508 );
11509
11510 let state = Arc::new(AsyncPollState::new()); // validate-positions NOT complete
11511 consumer.inflight_poll = Some(InflightPoll { deadline_ms: i64::MAX, state: Arc::clone(&state) });
11512 consumer.fetch_buffer.add(CompletedFetch::new(
11513 TopicPartition::new("t".to_string(), 0),
11514 PartitionData::new(),
11515 ));
11516
11517 let records = consumer.collect_fetch().await.expect("no error");
11518 assert!(records.is_empty(), "gated collect returns no records");
11519 assert!(
11520 !consumer.fetch_buffer.is_empty(),
11521 "the buffer must be left untouched while positions are still being validated"
11522 );
11523
11524 // Once the background task reports the stage complete, the same call
11525 // collects normally and drains the buffer.
11526 state.mark_validate_positions_complete();
11527 let _ = consumer.collect_fetch().await.expect("no error");
11528 assert!(
11529 consumer.fetch_buffer.is_empty(),
11530 "with the gate lifted the buffered fetch is consumed"
11531 );
11532 drop(handles.subscriptions);
11533 }
11534
11535 /// Returns `false` (return empty fetch) when the deadline has already
11536 /// passed and the reconciliation check is not complete.
11537 #[tokio::test]
11538 async fn wait_reconciliation_check_returns_false_on_timeout() {
11539 use crate::consumer::internals::events::AsyncPollState;
11540 let (mut consumer, handles) = make_test_consumer_with_channels();
11541 consumer.has_pending_reconciliation.store(true, Ordering::Release);
11542 let state = Arc::new(AsyncPollState::new());
11543 // deadline already in the past (<= now) -> no time to wait.
11544 consumer.inflight_poll = Some(InflightPoll { deadline_ms: 0, state });
11545 assert!(!consumer.wait_reconciliation_check().await.expect("no wakeup"));
11546 drop(handles.subscriptions);
11547 }
11548
11549 /// AK 4.3.1 (KAFKA-20106): a `wakeup()` interrupts the reconciliation-check
11550 /// wait. Translated from
11551 /// `AsyncKafkaConsumerTest#testWakeupWhileWaitingOnReconciliationCheck`.
11552 #[tokio::test]
11553 async fn wait_reconciliation_check_interrupted_by_wakeup() {
11554 use crate::consumer::internals::events::AsyncPollState;
11555 let (mut consumer, handles) = make_test_consumer_with_channels();
11556 consumer.has_pending_reconciliation.store(true, Ordering::Release);
11557 let state = Arc::new(AsyncPollState::new()); // never completed
11558 consumer.inflight_poll = Some(InflightPoll { deadline_ms: i64::MAX, state });
11559 // Java's `setActiveTask` completes the check future with a
11560 // `WakeupException`, which `getResult` throws out of `collectFetch()`:
11561 // the wait raises Error::Wakeup itself and rotates the token.
11562 consumer.wakeup_trigger.wakeup();
11563 let err = tokio::time::timeout(Duration::from_secs(5), consumer.wait_reconciliation_check())
11564 .await
11565 .expect("wakeup must interrupt the reconciliation-check wait")
11566 .expect_err("the wakeup is raised from the wait");
11567 assert!(matches!(err, Error::Wakeup(_)), "unexpected err: {err:?}");
11568 assert!(
11569 !consumer.wakeup_trigger.current_token().is_cancelled(),
11570 "the token is rotated after raising"
11571 );
11572 drop(handles.subscriptions);
11573 }
11574
11575 /// The close-handle closures must nudge the transport notify, never the
11576 /// [`WakeupTrigger`].
11577 ///
11578 /// `close()` calls `wakeup_trigger.disable()` as its first step, and
11579 /// `WakeupTrigger::wakeup()` is a no-op once disabled — so a close nudge
11580 /// routed through the trigger is dead exactly when it is needed. Both
11581 /// `signal_close()` and `wakeup()` were routed that way, leaving the bg task
11582 /// to discover `running == false` only after its in-flight poll drained.
11583 ///
11584 /// This asserts the wiring directly, on the same function production uses.
11585 /// It cannot regress silently: the trigger is not even a parameter, so
11586 /// nothing about its disabled state can affect the result.
11587 #[tokio::test]
11588 async fn close_handle_fns_nudge_the_transport_notify() {
11589 use std::sync::atomic::AtomicBool;
11590
11591 let running = Arc::new(AtomicBool::new(true));
11592 let notify = Arc::new(tokio::sync::Notify::new());
11593 let close_timeout_ms = Arc::new(AtomicI64::new(-1));
11594 let (signal_close_fn, wakeup_fn) =
11595 build_close_handle_fns(Arc::clone(&running), Arc::clone(&close_timeout_ms), Arc::clone(¬ify));
11596
11597 // `wakeup_fn` alone: nudge only, running flag untouched.
11598 wakeup_fn();
11599 assert!(running.load(Ordering::Acquire), "wakeup must not signal close");
11600 tokio::time::timeout(Duration::from_secs(1), notify.notified())
11601 .await
11602 .expect("wakeup_fn must nudge the transport notify");
11603
11604 // `signal_close_fn`: clears the flag AND nudges, so the bg task wakes
11605 // from its poll and observes the flag on the next loop check.
11606 signal_close_fn(Duration::from_millis(100));
11607 assert!(!running.load(Ordering::Acquire), "signal_close must clear the running flag");
11608 assert_eq!(
11609 100,
11610 close_timeout_ms.load(Ordering::Acquire),
11611 "signal_close must publish the close timeout the bg task's cleanup() reads"
11612 );
11613 tokio::time::timeout(Duration::from_secs(1), notify.notified())
11614 .await
11615 .expect("signal_close_fn must nudge the transport notify");
11616 }
11617
11618 /// Empty bg-events channel: returns immediately with `Ok(())`.
11619 #[tokio::test]
11620 async fn process_background_events_on_empty_channel_is_ok() {
11621 let (mut consumer, _handles) = make_test_consumer_with_channels();
11622 consumer.process_background_events().await.expect("ok");
11623 }
11624
11625 /// Issue 2 regression (Phase M6): the `background-event-queue-size` gauge
11626 /// must snap back to 0 on an *idle* (empty) drain, matching Java's
11627 /// `BackgroundEventHandler.drainEvents` which records
11628 /// `recordBackgroundEventQueueSize(0)` unconditionally. Pre-seed a stale
11629 /// peak (as the bg task's `add` would leave it), then drain an empty
11630 /// channel and assert both the shared `AtomicI64` and the registered
11631 /// metric reset to 0.
11632 #[tokio::test]
11633 async fn idle_drain_resets_background_event_queue_size_to_zero() {
11634 use crate::common::Metric;
11635 let (mut consumer, _handles) = make_test_consumer_with_channels();
11636
11637 // An idle drain with a genuinely empty queue. The counter starts at 0 and
11638 // must stay there.
11639 //
11640 // This test used to store 2 into the counter while leaving the channel
11641 // empty, then assert an empty drain reset it to 0. That premise was
11642 // self-contradictory — it claimed two events were "enqueued, not yet
11643 // drained" while nothing was queued — and it asserted the very
11644 // under-reporting that motivated conserving the counter: the old
11645 // `store(0)` wiped a live count. With the counter conserved (`+1` per send, `-1` per
11646 // dequeue) an empty drain has nothing to subtract, so 0 is reached by
11647 // construction rather than by overwriting.
11648 consumer.process_background_events().await.expect("ok");
11649
11650 // The shared counter must be reset to 0.
11651 assert_eq!(
11652 consumer.background_event_queue_size.load(Ordering::SeqCst),
11653 0,
11654 "an empty drain must leave the conserved counter at 0"
11655 );
11656 // The registered gauge must read 0, not linger at the stale peak.
11657 let mn = consumer.metrics.metric_name(
11658 "background-event-queue-size",
11659 crate::consumer::internals::ConsumerUtils::CONSUMER_METRIC_GROUP,
11660 );
11661 let value = consumer
11662 .metrics
11663 .metric(&mn)
11664 .expect("background-event-queue-size metric present")
11665 .metric_value()
11666 .as_double()
11667 .expect("double-valued gauge");
11668 assert_eq!(
11669 value, 0.0,
11670 "the gauge must be refreshed to 0 on an idle drain, matching Java's unconditional record"
11671 );
11672 }
11673
11674 /// Java `AsyncKafkaConsumerTest.testRecordBackgroundEventQueueSizeAndBackgroundEventQueueTime`
11675 /// Deferred from Phase M6 (needed the public `metrics()`
11676 /// accessor + a mock-clock-injectable consumer); translated here.
11677 ///
11678 /// Java enqueues a `PartitionsRemovedEvent` stamped
11679 /// with `time.milliseconds()`, records `recordBackgroundEventQueueSize(1)`,
11680 /// sleeps the mock clock 10 ms, calls `processBackgroundEvents()`, then
11681 /// asserts via the registry: `background-event-queue-size` == 0,
11682 /// `background-event-queue-time-avg` == 10, `-time-max` == 10.
11683 #[tokio::test]
11684 async fn test_record_background_event_queue_size_and_time() {
11685 use crate::common::Metric;
11686 use crate::common::utils::MockTime;
11687 use crate::consumer::internals::ConsumerRebalanceListenerMethodName;
11688 use tokio::sync::oneshot;
11689
11690 // Mock clock so the recorded queue-time (now - enqueuedMs) is exactly
11691 // 10 ms, deterministically. `self.time` drives both the enqueue stamp
11692 // and the drain-time read in `process_background_events`.
11693 let (mut consumer, handles) = make_test_consumer_with_channels();
11694
11695 // Swap in the mock clock (start at an arbitrary non-zero epoch).
11696 let mock_time = Arc::new(MockTime::with_auto_tick_ms_current_time_ms_current_high_res_time_ns(
11697 0, 1_000, 0,
11698 ));
11699 consumer.time = Arc::clone(&mock_time) as Arc<dyn Time>;
11700
11701 // Java: `event.setEnqueuedMs(time.milliseconds()); backgroundEventQueue.add(event);`
11702 // A no-listener callback-needed event acks Ok(()) — the time recording
11703 // does not depend on the listener result.
11704 let enqueued_ms = mock_time.milliseconds();
11705 let (ack_tx, _ack_rx) = oneshot::channel::<Result<(), Error>>();
11706 let env = BackgroundEventEnvelope {
11707 event: BackgroundEvent::PartitionsRemoved {
11708 method_name: ConsumerRebalanceListenerMethodName::OnPartitionsRevoked,
11709 partitions: Vec::new(),
11710 ack: ack_tx,
11711 ack_through_application_events: false,
11712 },
11713 enqueued_ms,
11714 };
11715 handles.bg_event_tx.send(env).expect("send ok");
11716
11717 // Java: `asyncConsumerMetrics.recordBackgroundEventQueueSize(1);`
11718 consumer.async_consumer_metrics.record_background_event_queue_size(1);
11719
11720 // Java: `time.sleep(10); consumer.processBackgroundEvents();`
11721 mock_time.sleep(10);
11722 consumer.process_background_events().await.expect("drain ok");
11723
11724 // Read the values through the PUBLIC `metrics()` accessor (M7).
11725 let snapshot = consumer.metrics();
11726 let read = |name: &str| -> f64 {
11727 let mn = consumer
11728 .metrics
11729 .metric_name(name, crate::consumer::internals::ConsumerUtils::CONSUMER_METRIC_GROUP);
11730 snapshot
11731 .get(&mn)
11732 .unwrap_or_else(|| panic!("metric {name} present"))
11733 .metric_value()
11734 .as_double()
11735 .expect("double-valued metric")
11736 };
11737
11738 assert_eq!(read("background-event-queue-size"), 0.0);
11739 assert_eq!(read("background-event-queue-time-avg"), 10.0);
11740 assert_eq!(read("background-event-queue-time-max"), 10.0);
11741 }
11742
11743 /// Phase M7: the public `metrics()` accessor returns the registry snapshot
11744 /// that the metrics managers populate. Mirrors the read contract of
11745 /// Java's `AsyncKafkaConsumer.metrics()` (`Collections.unmodifiableMap`).
11746 ///
11747 /// This proves that the three metrics families that register EAGERLY in
11748 /// this fixture — fetch (`create_fetch_metrics_manager`), kafka-consumer
11749 /// (`KafkaConsumerMetrics::new`), and async-consumer
11750 /// (`AsyncConsumerMetrics::new`) — each surface a representative metric in
11751 /// the public `metrics()` snapshot, i.e. the registry plumbing reaches the
11752 /// public accessor and the snapshot is the live registry, not an
11753 /// empty/partial map.
11754 ///
11755 /// The other four families (heartbeat / offset-commit / rebalance /
11756 /// rebalance-callback) register against the shared `Metrics` only through
11757 /// their request managers, which the production ctor builds but this
11758 /// fixture does not (`RequestManagers::new(None × 7)`). Their registration
11759 /// against the shared `Metrics` is covered by their own M4/M5 manager tests
11760 /// (`heartbeat_metrics`, `offset_commit_metrics`, the
11761 /// `ConsumerRebalanceMetricsManager` / `RebalanceCallbackMetricsManager`
11762 /// tests). We deliberately do NOT force-register managers the fixture does
11763 /// not build, so this test stays honest about what it constructs.
11764 #[tokio::test]
11765 async fn metrics_snapshot_includes_eagerly_registered_families() {
11766 use crate::common::Metric;
11767 // Group name built by `FetchMetricsRegistry::new` from the
11768 // `"consumer"` prefix (`create_fetch_metrics_manager`).
11769 const FETCH_MANAGER_METRIC_GROUP: &str = "consumer-fetch-manager-metrics";
11770
11771 let consumer = make_test_consumer();
11772 let snapshot = consumer.metrics();
11773 assert!(!snapshot.is_empty(), "metrics() must not be empty");
11774
11775 let assert_present = |name: &str, group: &str| {
11776 let mn = consumer.metrics.metric_name(name, group);
11777 assert!(
11778 snapshot.contains_key(&mn),
11779 "metrics() snapshot must include `{name}` (group `{group}`)"
11780 );
11781 };
11782
11783 // Fetch family (M3): a client-level fetch metric registered eagerly in
11784 // `FetchMetricsManager::new`.
11785 assert_present("records-consumed-total", FETCH_MANAGER_METRIC_GROUP);
11786 // Kafka-consumer family (M4): registered eagerly in
11787 // `KafkaConsumerMetrics::new` under the consumer-metrics group.
11788 assert_present(
11789 "last-poll-seconds-ago",
11790 crate::consumer::internals::ConsumerUtils::CONSUMER_METRIC_GROUP,
11791 );
11792 // Async-consumer family (M6): registered eagerly in
11793 // `AsyncConsumerMetrics::new` under the consumer-metrics group.
11794 assert_present(
11795 "background-event-queue-size",
11796 crate::consumer::internals::ConsumerUtils::CONSUMER_METRIC_GROUP,
11797 );
11798
11799 // The snapshot is keyed by MetricName and the values impl `Metric`:
11800 // every entry's `metric_name()` matches its key (sanity of the snapshot).
11801 for (name, metric) in snapshot.snapshot().iter() {
11802 assert_eq!(metric.metric_name(), name);
11803 }
11804 }
11805
11806 /// Stands in for the bg task's `CommitSync` handling
11807 /// (`ApplicationEventProcessor#process(SyncCommitEvent)`): on the first `CommitSync`
11808 /// it logs `"snapshot"`, after `before_snapshot` (when given) has run and
11809 /// 200 ms have passed, then marks the offsets ready and keeps the commit future pending, so
11810 /// the commit's later wait is the one a wakeup ends. Every
11811 /// `CommitSync` after it completes at once.
11812 fn spawn_commit_sync_snapshotter(
11813 mut app_event_rx: mpsc::UnboundedReceiver<ApplicationEventEnvelope>,
11814 log: Arc<Mutex<Vec<&'static str>>>,
11815 before_snapshot: Option<Box<dyn FnOnce() + Send>>,
11816 ) -> tokio::task::JoinHandle<()> {
11817 tokio::spawn(async move {
11818 let mut before_snapshot = before_snapshot;
11819 let mut pending = Vec::new();
11820 while let Some(env) = app_event_rx.recv().await {
11821 if let ApplicationEvent::CommitSync { handle, offsets_ready, .. } = env.event {
11822 if pending.is_empty() {
11823 if let Some(before) = before_snapshot.take() {
11824 before();
11825 // The commit is still waiting well after it.
11826 tokio::time::sleep(Duration::from_millis(200)).await;
11827 }
11828 log.lock().unwrap().push("snapshot");
11829 offsets_ready.complete(());
11830 // Held, so the commit future stays pending.
11831 pending.push(handle);
11832 } else {
11833 offsets_ready.complete(());
11834 handle.complete(HashMap::new());
11835 }
11836 }
11837 }
11838 })
11839 }
11840
11841 /// Issue 11 regression: `commit_sync` observes a pending `wakeup()` and
11842 /// returns `Error::Wakeup`, at Java's point: only once the bg task has
11843 /// snapshotted the offsets. Java's offsets-ready wait has no
11844 /// `setActiveTask` (`AsyncKafkaConsumer#commit`); its
11845 /// `wakeupTrigger.setActiveTask(commitFuture)` comes after it (`AsyncKafkaConsumer#commitSync(Optional,Duration)`).
11846 #[tokio::test]
11847 async fn issue_11_commit_sync_observes_wakeup_during_wait() {
11848 let (mut consumer, handles) = make_test_consumer_with_channels();
11849 let ConsumerTestHandles { app_event_rx, bg_event_tx: _bg_event_tx, .. } = handles;
11850 let log = Arc::new(Mutex::new(Vec::new()));
11851 let bg = spawn_commit_sync_snapshotter(app_event_rx, Arc::clone(&log), None);
11852 consumer.wakeup_trigger.wakeup();
11853
11854 let err = consumer
11855 .commit_sync_with_timeout(Duration::from_secs(5))
11856 .await
11857 .expect_err("must wake up before deadline");
11858 assert!(matches!(err, Error::Wakeup(_)), "expected Wakeup, got {err:?}");
11859 log.lock().unwrap().push("wakeup");
11860 assert_eq!(*log.lock().unwrap(), vec!["snapshot", "wakeup"]);
11861 // Token rotated after the wakeup was surfaced.
11862 assert!(!consumer.wakeup_trigger.current_token().is_cancelled());
11863 bg.abort();
11864 }
11865
11866 /// A `wakeup()` while `commit_sync` waits for its offsets does not end
11867 /// that wait: the commit raises `Error::Wakeup` only after the bg task has
11868 /// snapshotted the offsets (`ApplicationEventProcessor#process(SyncCommitEvent)`), so
11869 /// a `poll` after the `WakeupError` cannot move the positions the woken
11870 /// commit commits. Java's `offsetsReady` wait has no `setActiveTask`
11871 /// (`AsyncKafkaConsumer#commit`).
11872 #[tokio::test]
11873 async fn a_wakeup_during_the_offsets_ready_wait_is_raised_after_the_snapshot() {
11874 let (mut consumer, handles) = make_test_consumer_with_channels();
11875 let ConsumerTestHandles { app_event_rx, bg_event_tx: _bg_event_tx, .. } = handles;
11876 let log = Arc::new(Mutex::new(Vec::new()));
11877 let trigger = consumer.wakeup_trigger.clone();
11878 let before_log = Arc::clone(&log);
11879 // The wakeup lands while the commit waits for its offsets.
11880 let before: Box<dyn FnOnce() + Send> = Box::new(move || {
11881 trigger.wakeup();
11882 before_log.lock().unwrap().push("wakeup()");
11883 });
11884 let bg = spawn_commit_sync_snapshotter(app_event_rx, Arc::clone(&log), Some(before));
11885
11886 let err = consumer
11887 .commit_sync_with_timeout(Duration::from_secs(5))
11888 .await
11889 .expect_err("the wakeup is raised by the commit future's wait");
11890 assert!(matches!(err, Error::Wakeup(_)), "expected Wakeup, got {err:?}");
11891 log.lock().unwrap().push("WakeupError");
11892 assert_eq!(*log.lock().unwrap(), vec!["wakeup()", "snapshot", "WakeupError"]);
11893 assert!(!consumer.wakeup_trigger.current_token().is_cancelled());
11894 bg.abort();
11895 }
11896
11897 /// The same through a [`ConsumerHandle`]: its offsets-ready wait is not
11898 /// wakeable either, and the wakeup is raised after the snapshot.
11899 #[tokio::test]
11900 async fn handle_a_wakeup_during_the_offsets_ready_wait_is_raised_after_the_snapshot() {
11901 let (consumer, handles) = make_test_consumer_with_channels();
11902 let ConsumerTestHandles { app_event_rx, bg_event_tx: _bg_event_tx, .. } = handles;
11903 let handle = consumer.handle();
11904 let _in_callback = consumer.enter_callback_scope();
11905 let log = Arc::new(Mutex::new(Vec::new()));
11906 let waker = handle.clone();
11907 let before_log = Arc::clone(&log);
11908 let before: Box<dyn FnOnce() + Send> = Box::new(move || {
11909 waker.wakeup();
11910 before_log.lock().unwrap().push("wakeup()");
11911 });
11912 let bg = spawn_commit_sync_snapshotter(app_event_rx, Arc::clone(&log), Some(before));
11913
11914 let err = handle
11915 .commit_sync()
11916 .await
11917 .expect_err("the wakeup is raised by the commit future's wait");
11918 assert!(matches!(err, Error::Wakeup(_)), "expected Wakeup, got {err:?}");
11919 log.lock().unwrap().push("WakeupError");
11920 assert_eq!(*log.lock().unwrap(), vec!["wakeup()", "snapshot", "WakeupError"]);
11921 bg.abort();
11922 drop(consumer);
11923 }
11924
11925 /// Issue 11 regression: `committed_with_timeout` ALSO observes wakeup
11926 /// (Java's `committed` calls `setActiveTask`).
11927 #[tokio::test]
11928 async fn issue_11_committed_observes_wakeup_during_wait() {
11929 let (mut consumer, _handles) = make_test_consumer_with_channels();
11930 consumer.wakeup_trigger.wakeup();
11931
11932 let tp = TopicPartition::new("t".to_string(), 0);
11933 let err = consumer
11934 .committed_with_timeout(&[tp], Duration::from_secs(5))
11935 .await
11936 .expect_err("must wake up before deadline");
11937 assert!(matches!(err, Error::Wakeup(_)), "expected Wakeup, got {err:?}");
11938 assert!(!consumer.wakeup_trigger.current_token().is_cancelled());
11939 }
11940
11941 /// Java's `WakeupTrigger.setActiveTask` on a future that is already done
11942 /// keeps its result and leaves the wakeup pending
11943 /// (`WakeupTriggerTest.testExceptionTriggeredWhenTaskAsynchronouslyCompletedBeforeSet`).
11944 /// `commitSync(Map.of())` waits on `CompletableFuture.completedFuture(null)`
11945 /// (`AsyncKafkaConsumer#commit`), so after a
11946 /// `wakeup()` it returns normally and the next wakeable call throws.
11947 #[tokio::test]
11948 async fn wakeup_before_commit_sync_with_empty_offsets_is_left_for_the_next_call() {
11949 let (mut consumer, _handles) = make_test_consumer_with_channels();
11950 consumer.wakeup_trigger.wakeup();
11951
11952 consumer
11953 .commit_sync_with_offsets(HashMap::new())
11954 .await
11955 .expect("an already-completed commit keeps its result");
11956 assert!(
11957 consumer.wakeup_trigger.current_token().is_cancelled(),
11958 "the wakeup must stay pending for the next call"
11959 );
11960
11961 let tp = TopicPartition::new("t".to_string(), 0);
11962 let err = consumer
11963 .committed_with_timeout(&[tp], Duration::from_secs(5))
11964 .await
11965 .expect_err("the next wakeable call raises the pending wakeup");
11966 assert!(matches!(err, Error::Wakeup(_)), "expected Wakeup, got {err:?}");
11967 assert!(!consumer.wakeup_trigger.current_token().is_cancelled());
11968 }
11969
11970 /// The same through a [`ConsumerHandle`], whose waits go through
11971 /// `ConsumerHandle::await_completion`.
11972 #[tokio::test]
11973 async fn handle_wakeup_before_commit_sync_with_empty_offsets_is_left_for_the_next_call() {
11974 let (mut consumer, _handles) = make_test_consumer_with_channels();
11975 let handle = consumer.handle();
11976 let _in_callback = consumer.enter_callback_scope();
11977 handle.wakeup();
11978
11979 handle
11980 .commit_sync_with_offsets(HashMap::new())
11981 .await
11982 .expect("an already-completed commit keeps its result");
11983 assert!(
11984 consumer.wakeup_trigger.current_token().is_cancelled(),
11985 "the wakeup must stay pending for the next call"
11986 );
11987
11988 let tp = TopicPartition::new("t".to_string(), 0);
11989 let err = consumer
11990 .committed_with_timeout(&[tp], Duration::from_secs(5))
11991 .await
11992 .expect_err("the next wakeable call raises the pending wakeup");
11993 assert!(matches!(err, Error::Wakeup(_)), "expected Wakeup, got {err:?}");
11994 }
11995
11996 /// `awaitPendingAsyncCommitsAndExecuteCommitCallbacks` completes
11997 /// `futureToAwait` at once through `whenComplete` when the last async
11998 /// commit is already done, before `setActiveTask(futureToAwait)`
11999 /// (`AsyncKafkaConsumer#awaitPendingAsyncCommitsAndExecuteCommitCallbacks`): a pending wakeup does not fail
12000 /// the wait and stays for the next call. An empty-offsets `commitAsync`
12001 /// leaves exactly such a done commit (`AsyncKafkaConsumer#commit`).
12002 #[tokio::test]
12003 async fn wakeup_before_commit_sync_waiting_on_a_done_async_commit_is_left_armed() {
12004 let (consumer, _handles) = make_test_consumer_with_channels();
12005 let handle = consumer.handle();
12006 let _in_callback = consumer.enter_callback_scope();
12007 handle
12008 .commit_async_with_offsets_callback(HashMap::new(), Arc::new(NoopCallback))
12009 .await
12010 .expect("empty-offsets commit_async");
12011 assert!(
12012 consumer
12013 .commit_state
12014 .last_pending_async_commit()
12015 .expect("a pending async commit")
12016 .is_done(),
12017 "precondition: the pending async commit is already done"
12018 );
12019 handle.wakeup();
12020
12021 handle
12022 .commit_sync_with_offsets(HashMap::new())
12023 .await
12024 .expect("a done pending commit and a done commit keep their results");
12025 assert!(
12026 consumer.wakeup_trigger.maybe_trigger_wakeup().is_err(),
12027 "the wakeup is still pending"
12028 );
12029 }
12030
12031 /// The outcomes a task can have in Java's `WakeupTriggerTest`
12032 /// (`complete`, `completeExceptionally`, `cancel`), applied to the
12033 /// sending end of the awaited receiver; a dropped sender is Rust's
12034 /// cancelled task, which the wait reports as an illegal state.
12035 #[derive(Clone, Copy, Debug)]
12036 enum TaskOutcome {
12037 Completed,
12038 Failed,
12039 Cancelled,
12040 }
12041
12042 impl TaskOutcome {
12043 const ALL: [TaskOutcome; 3] = [TaskOutcome::Completed, TaskOutcome::Failed, TaskOutcome::Cancelled];
12044
12045 fn apply(self, sender: tokio::sync::oneshot::Sender<Result<(), Error>>) {
12046 match self {
12047 TaskOutcome::Completed => {
12048 let _ = sender.send(Ok(()));
12049 },
12050 TaskOutcome::Failed => {
12051 let _ = sender.send(Err(Error::local_illegal_state("Simulated error")));
12052 },
12053 TaskOutcome::Cancelled => drop(sender),
12054 }
12055 }
12056
12057 fn assert_result(self, result: Result<(), Error>) {
12058 match (self, result) {
12059 (TaskOutcome::Completed, Ok(())) => {},
12060 (TaskOutcome::Failed, Err(Error::LocalIllegalState(e))) => assert_eq!(e.message(), "Simulated error"),
12061 (TaskOutcome::Cancelled, Err(Error::LocalIllegalState(e))) => assert_eq!(
12062 e.message(),
12063 "Background task dropped the completion sender without completing it"
12064 ),
12065 (outcome, result) => panic!("{outcome:?}: unexpected result {result:?}"),
12066 }
12067 }
12068 }
12069
12070 /// Java `testExceptionTriggeredWhenTaskAsynchronouslyCompletedBeforeSet`,
12071 /// `...FailedBeforeSet` and `...CancelledBeforeSet`: a task done before
12072 /// the wait starts keeps its outcome over an earlier `wakeup()`, which
12073 /// stays pending. Both waits are covered: the owning consumer's
12074 /// `await_completion` and `ConsumerHandle::await_completion`.
12075 #[tokio::test]
12076 async fn wakeup_before_a_task_done_before_the_wait_stays_pending() {
12077 for outcome in TaskOutcome::ALL {
12078 let (consumer, _handles) = make_test_consumer_with_channels();
12079 let (sender, receiver) = tokio::sync::oneshot::channel();
12080 outcome.apply(sender);
12081 consumer.wakeup_trigger.wakeup();
12082 let deadline = consumer.time.milliseconds() + 5000;
12083 let result = consumer.await_completion::<()>(receiver, deadline, "timeout", true).await;
12084 outcome.assert_result(result);
12085 assert!(
12086 consumer.wakeup_trigger.maybe_trigger_wakeup().is_err(),
12087 "{outcome:?}: wakeup still pending"
12088 );
12089
12090 let (sender, receiver) = tokio::sync::oneshot::channel();
12091 outcome.apply(sender);
12092 let consumer_handle = consumer.handle();
12093 let ConsumerHandleInner::Async(state) = &consumer_handle.inner else {
12094 panic!("an AsyncKafkaConsumer hands out an async handle");
12095 };
12096 let result = state.await_completion::<()>(receiver, deadline, "timeout", true).await;
12097 outcome.assert_result(result);
12098 assert!(
12099 consumer.wakeup_trigger.maybe_trigger_wakeup().is_err(),
12100 "{outcome:?}: wakeup still pending"
12101 );
12102 }
12103 }
12104
12105 /// Java `testExceptionTriggeredWhenTaskAsynchronouslyCompleted`,
12106 /// `...Failed` and `...Cancelled`: a task that completes during the wait
12107 /// returns its outcome, and a `wakeup()` after that stays pending for the
12108 /// next call.
12109 #[tokio::test]
12110 async fn wakeup_after_a_task_done_during_the_wait_stays_pending() {
12111 for outcome in TaskOutcome::ALL {
12112 let (consumer, _handles) = make_test_consumer_with_channels();
12113 let (sender, receiver) = tokio::sync::oneshot::channel();
12114 let completer = tokio::spawn(async move {
12115 tokio::time::sleep(Duration::from_millis(20)).await;
12116 outcome.apply(sender);
12117 });
12118 let deadline = consumer.time.milliseconds() + 5000;
12119 let result = consumer.await_completion::<()>(receiver, deadline, "timeout", true).await;
12120 completer.await.expect("completer ok");
12121 outcome.assert_result(result);
12122 consumer.wakeup_trigger.wakeup();
12123 assert!(
12124 consumer.wakeup_trigger.maybe_trigger_wakeup().is_err(),
12125 "{outcome:?}: wakeup pending"
12126 );
12127 }
12128 }
12129
12130 /// Asserts that `call` raised a pending wakeup before submitting its
12131 /// event: `Error::Wakeup`, the token rotated, nothing on the event queue.
12132 fn assert_wakeup_raised_before_submitting<T>(
12133 call: &str,
12134 result: Result<T, Error>,
12135 trigger: &WakeupTrigger,
12136 app_event_rx: &mut mpsc::UnboundedReceiver<ApplicationEventEnvelope>,
12137 ) {
12138 match result {
12139 Err(Error::Wakeup(_)) => {},
12140 Err(err) => panic!("{call}: expected Wakeup, got {err:?}"),
12141 Ok(_) => panic!("{call}: expected Wakeup, got a result"),
12142 }
12143 assert!(!trigger.current_token().is_cancelled(), "{call}: the raised wakeup is rotated");
12144 assert!(app_event_rx.try_recv().is_err(), "{call}: no event is submitted");
12145 }
12146
12147 /// Java's `committed`, `position` (through `updateFetchPositions`),
12148 /// `partitionsFor` and `listTopics` call `wakeupTrigger.setActiveTask`
12149 /// before `applicationEventHandler.addAndGet`
12150 /// (`AsyncKafkaConsumer#committed(Set,Duration)`, `#updateFetchPositions`, `#partitionsFor(String,Duration)`,
12151 /// `#listTopics(Duration)`), so a wakeup pending when the call starts fails it whatever the background
12152 /// thread does with the event, as `testWakeupCommitted` asserts. The same
12153 /// through a [`ConsumerHandle`]'s `position` and `committed`.
12154 #[tokio::test]
12155 async fn a_pending_wakeup_fails_the_wakeable_queries_before_their_event_is_submitted() {
12156 let (mut consumer, mut handles) = make_test_consumer_with_channels();
12157 let tp = TopicPartition::new("t".to_string(), 0);
12158 consumer
12159 .subscriptions
12160 .lock()
12161 .unwrap()
12162 .assign_from_user(HashSet::from([tp.clone()]))
12163 .unwrap();
12164 let trigger = consumer.wakeup_trigger.clone();
12165 let timeout = Duration::from_secs(5);
12166
12167 trigger.wakeup();
12168 let result = consumer.committed_with_timeout(std::slice::from_ref(&tp), timeout).await;
12169 assert_wakeup_raised_before_submitting("committed", result, &trigger, &mut handles.app_event_rx);
12170
12171 trigger.wakeup();
12172 let result = consumer.position_with_timeout(&tp, timeout).await;
12173 assert_wakeup_raised_before_submitting("position", result, &trigger, &mut handles.app_event_rx);
12174
12175 trigger.wakeup();
12176 let result = consumer.partitions_for_with_timeout("t", timeout).await;
12177 assert_wakeup_raised_before_submitting("partitions_for", result, &trigger, &mut handles.app_event_rx);
12178
12179 trigger.wakeup();
12180 let result = consumer.list_topics_with_timeout(timeout).await;
12181 assert_wakeup_raised_before_submitting("list_topics", result, &trigger, &mut handles.app_event_rx);
12182
12183 let handle = consumer.handle();
12184 let _in_callback = consumer.enter_callback_scope();
12185 trigger.wakeup();
12186 let result = handle.position_with_timeout(&tp, timeout).await;
12187 assert_wakeup_raised_before_submitting("handle position", result, &trigger, &mut handles.app_event_rx);
12188
12189 trigger.wakeup();
12190 let result = handle.committed(std::slice::from_ref(&tp)).await;
12191 assert_wakeup_raised_before_submitting("handle committed", result, &trigger, &mut handles.app_event_rx);
12192 }
12193
12194 /// A clock that, at its first reading after a `CheckAndUpdatePositions`
12195 /// event is submitted, takes the event (leaving it pending), calls
12196 /// `wakeup()` and moves past every deadline, so the wait for the update
12197 /// ends by its deadline with the wakeup still pending.
12198 struct WakeAtUpdateTime {
12199 now_ms: AtomicI64,
12200 app_event_rx: std::sync::Mutex<mpsc::UnboundedReceiver<ApplicationEventEnvelope>>,
12201 held: std::sync::Mutex<Vec<ApplicationEventEnvelope>>,
12202 trigger: WakeupTrigger,
12203 }
12204
12205 impl WakeAtUpdateTime {
12206 fn install(
12207 consumer: &mut AsyncKafkaConsumer<Vec<u8>, Vec<u8>>,
12208 app_event_rx: mpsc::UnboundedReceiver<ApplicationEventEnvelope>,
12209 ) -> Arc<Self> {
12210 let time = Arc::new(Self {
12211 now_ms: AtomicI64::new(1_000),
12212 app_event_rx: std::sync::Mutex::new(app_event_rx),
12213 held: std::sync::Mutex::new(Vec::new()),
12214 trigger: consumer.wakeup_trigger.clone(),
12215 });
12216 consumer.time = Arc::clone(&time) as Arc<dyn Time>;
12217 time
12218 }
12219
12220 fn updates_taken(&self) -> usize {
12221 self.held
12222 .lock()
12223 .unwrap()
12224 .iter()
12225 .filter(|envelope| matches!(envelope.event, ApplicationEvent::CheckAndUpdatePositions { .. }))
12226 .count()
12227 }
12228 }
12229
12230 impl Time for WakeAtUpdateTime {
12231 fn milliseconds(&self) -> i64 {
12232 if let Ok(envelope) = self.app_event_rx.lock().unwrap().try_recv() {
12233 if matches!(envelope.event, ApplicationEvent::CheckAndUpdatePositions { .. }) {
12234 self.trigger.wakeup();
12235 self.now_ms.fetch_add(3_600_000, Ordering::SeqCst);
12236 }
12237 self.held.lock().unwrap().push(envelope);
12238 }
12239 self.now_ms.load(Ordering::SeqCst)
12240 }
12241
12242 fn nanoseconds(&self) -> i64 {
12243 self.milliseconds() * 1_000_000
12244 }
12245 }
12246
12247 /// Java's `position` calls `wakeupTrigger.maybeTriggerWakeup()` after
12248 /// every `updateFetchPositions`, before `timer.notExpired()`
12249 /// (`AsyncKafkaConsumer#position(TopicPartition,Duration)`): a wakeup still pending when the
12250 /// update returns fails the call, here ahead of the timeout the expired
12251 /// update would otherwise lead to. Both the consumer's and a
12252 /// [`ConsumerHandle`]'s `position`.
12253 #[tokio::test]
12254 async fn position_raises_a_wakeup_pending_after_the_update_ahead_of_the_timeout() {
12255 let tp = TopicPartition::new("t".to_string(), 0);
12256 for through_handle in [false, true] {
12257 let (mut consumer, handles) = make_test_consumer_with_channels();
12258 consumer
12259 .subscriptions
12260 .lock()
12261 .unwrap()
12262 .assign_from_user(HashSet::from([tp.clone()]))
12263 .unwrap();
12264 let time = WakeAtUpdateTime::install(&mut consumer, handles.app_event_rx);
12265 let result = if through_handle {
12266 let _in_callback = consumer.enter_callback_scope();
12267 consumer.handle().position_with_timeout(&tp, Duration::from_secs(5)).await
12268 } else {
12269 consumer.position_with_timeout(&tp, Duration::from_secs(5)).await
12270 };
12271 let err = result.expect_err("the wakeup fails position");
12272 assert!(
12273 matches!(err, Error::Wakeup(_)),
12274 "handle={through_handle}: expected Wakeup, got {err:?}"
12275 );
12276 assert_eq!(time.updates_taken(), 1, "handle={through_handle}: one update was submitted");
12277 assert!(
12278 !consumer.wakeup_trigger.current_token().is_cancelled(),
12279 "handle={through_handle}: the raised wakeup is rotated"
12280 );
12281 }
12282 }
12283
12284 /// Issue 14 regression: `position_with_timeout` must propagate
12285 /// non-Timeout errors from the underlying `CheckAndUpdatePositions`
12286 /// event (e.g. an authorization error surfaced by the bg task).
12287 /// The previous `.await.ok()` blanket-swallowed all errors so the
12288 /// user observed a generic Timeout instead of the root cause.
12289 /// Java only catches `TimeoutException`
12290 /// (`AsyncKafkaConsumer#updateFetchPositions`); anything else propagates.
12291 #[tokio::test]
12292 async fn issue_14_position_propagates_non_timeout_errors() {
12293 let (mut consumer, mut handles) = make_test_consumer_with_channels();
12294 // Pre-assign a partition so position_with_timeout reaches the
12295 // event-submit path instead of returning IllegalState early.
12296 let tp = TopicPartition::new("t".to_string(), 0);
12297 {
12298 let mut subs = consumer.subscriptions.lock().unwrap();
12299 let mut assigned: HashSet<TopicPartition> = HashSet::new();
12300 assigned.insert(tp.clone());
12301 subs.assign_from_user(assigned).unwrap();
12302 }
12303
12304 // Drainer: complete the CheckAndUpdatePositions handle with
12305 // an explicit non-Timeout error (mirrors a bg-side
12306 // illegal-state failure). Without the Issue 14 fix the
12307 // `.await.ok()` would silently swallow this and the loop
12308 // would spin until the user-supplied timeout — yielding a
12309 // misleading Timeout instead of the root cause.
12310 let drainer = tokio::spawn(async move {
12311 while let Some(env) = handles.app_event_rx.recv().await {
12312 if let ApplicationEvent::CheckAndUpdatePositions { handle } = env.event {
12313 handle
12314 .complete_with_error(Error::local_illegal_state("bg-side test failure (Issue 14 regression)"));
12315 return;
12316 }
12317 }
12318 });
12319
12320 let err = consumer
12321 .position_with_timeout(&tp, Duration::from_secs(5))
12322 .await
12323 .expect_err("must surface the bg-task error, not a generic Timeout");
12324 assert!(
12325 matches!(err, Error::LocalIllegalState(ref m) if m.message().contains("Issue 14 regression")),
12326 "expected IllegalState (bg-task explicit error), got {err:?}"
12327 );
12328 drainer.await.expect("drainer ok");
12329 }
12330
12331 /// Issue 22 regression: `commit_async` must NOT observe wakeup at
12332 /// any phase of the call. Java's `commitAsync`
12333 /// (`AsyncKafkaConsumer#commitAsync(Optional,OffsetCommitCallback)`) is documented as
12334 /// non-blocking and never throws `WakeupException`. Pre-cancelling
12335 /// the wakeup token before calling `commit_async` must not surface
12336 /// `Error::Wakeup`.
12337 #[tokio::test]
12338 async fn issue_22_commit_async_does_not_observe_wakeup() {
12339 let (mut consumer, mut handles) = make_test_consumer_with_channels();
12340 // Pre-cancel the wakeup token: `commit_async` must not surface a
12341 // Wakeup error from the shared `commit_inner`'s offsets-ready wait,
12342 // which is not wakeable (Java's `offsetsReady` wait).
12343 consumer.wakeup_trigger.wakeup();
12344
12345 // Drainer: complete the CommitAsync envelope normally.
12346 let completer = tokio::spawn(async move {
12347 while let Some(env) = handles.app_event_rx.recv().await {
12348 if let ApplicationEvent::CommitAsync { handle, offsets_ready, .. } = env.event {
12349 offsets_ready.complete(());
12350 handle.complete(HashMap::new());
12351 return;
12352 }
12353 }
12354 });
12355
12356 consumer.commit_async().await.expect("commit_async must NOT surface Wakeup");
12357 completer.await.expect("completer ok");
12358 }
12359
12360 /// Issue 11 negative: APIs Java doesn't `setActiveTask` (e.g.
12361 /// `pause`) must NOT observe wakeup. `pause` submits its event with
12362 /// `enable_wakeup=false`, so the wait completes normally.
12363 #[tokio::test]
12364 async fn issue_11_pause_does_not_observe_wakeup() {
12365 let (mut consumer, mut handles) = make_test_consumer_with_channels();
12366 consumer.wakeup_trigger.wakeup();
12367 let tp = TopicPartition::new("t".to_string(), 0);
12368
12369 // Spawn a completer that resolves the PausePartitions event.
12370 let completer = tokio::spawn(async move {
12371 while let Some(env) = handles.app_event_rx.recv().await {
12372 if let ApplicationEvent::PausePartitions { handle, .. } = env.event {
12373 handle.complete(());
12374 return;
12375 }
12376 }
12377 });
12378
12379 // Should succeed without raising Wakeup despite the cancelled token.
12380 consumer.pause(&[tp]).await.expect("pause must NOT observe wakeup");
12381 completer.await.expect("completer ok");
12382 }
12383
12384 // ─── §31 regression pair ─────────────────────────────────────────
12385 //
12386 // consumer-threading.md §31: `ConsumerRebalanceListener` callbacks
12387 // execute on the caller's task. Two contracts:
12388 // - Test A: a listener that calls `commit_sync()` (through its
12389 // `ConsumerHandle`) from INSIDE `on_partitions_revoked`, delivered by
12390 // `poll()`, must succeed (no deadlock).
12391 // - Test B: the rebalance state machine must NOT advance until
12392 // the listener future resolves.
12393 //
12394 // They exercise the cross-task handshake from
12395 // `process_background_events` (app side) ↔ the bg task's listener ack.
12396
12397 /// §31 Test A, Java `AsyncKafkaConsumerTest.testCommitInRebalanceCallback`:
12398 /// `poll()` delivers `on_partitions_revoked`, whose body calls
12399 /// `commitSync(Map.of(tp, new OffsetAndMetadata(0)))` on the consumer it
12400 /// captured, and the commit succeeds.
12401 ///
12402 /// The listener commits through its [`ConsumerHandle`] (the Rust
12403 /// equivalent of Java capturing the `consumer` variable). The fake bg task
12404 /// answers the `CommitSync` event while the listener is still running,
12405 /// which only works because the listener runs on the caller's task inside
12406 /// `poll()` and the commit waits on its own event without draining.
12407 #[tokio::test]
12408 async fn test_commit_in_rebalance_callback() {
12409 use crate::consumer::internals::ConsumerRebalanceListenerMethodName;
12410 use async_trait::async_trait;
12411 use std::sync::atomic::AtomicUsize;
12412 use tokio::sync::oneshot;
12413
12414 struct CommittingListener {
12415 handle: ConsumerHandle,
12416 outcome: std::sync::Mutex<Option<Result<(), Error>>>,
12417 }
12418 #[async_trait]
12419 impl ConsumerRebalanceListener for CommittingListener {
12420 async fn on_partitions_revoked(&self, partitions: &[TopicPartition]) -> Result<(), Error> {
12421 let offsets: HashMap<TopicPartition, OffsetAndMetadata> = partitions
12422 .iter()
12423 .map(|tp| (tp.clone(), OffsetAndMetadata::new(0).expect("offset")))
12424 .collect();
12425 *self.outcome.lock().unwrap() = Some(self.handle.commit_sync_with_offsets(offsets).await);
12426 Ok(())
12427 }
12428 async fn on_partitions_assigned(&self, _: &[TopicPartition]) -> Result<(), Error> {
12429 Ok(())
12430 }
12431 }
12432
12433 let (mut consumer, handles) = make_test_consumer_with_channels();
12434 let ConsumerTestHandles { mut app_event_rx, bg_event_tx, subscriptions, .. } = handles;
12435 let tp = TopicPartition::new("foo".to_string(), 3);
12436 subscriptions
12437 .lock()
12438 .unwrap()
12439 .assign_from_user(HashSet::from([tp.clone()]))
12440 .unwrap();
12441 let listener = Arc::new(CommittingListener { handle: consumer.handle(), outcome: std::sync::Mutex::new(None) });
12442 *consumer.rebalance_listener.lock().unwrap() =
12443 Some(Arc::clone(&listener) as Arc<dyn ConsumerRebalanceListener>);
12444 let commits = Arc::new(AtomicUsize::new(0));
12445 let completer = {
12446 let commits = Arc::clone(&commits);
12447 tokio::spawn(async move {
12448 while let Some(env) = app_event_rx.recv().await {
12449 if let ApplicationEvent::CommitSync { handle, offsets_ready, .. } = env.event {
12450 commits.fetch_add(1, Ordering::SeqCst);
12451 offsets_ready.complete(());
12452 handle.complete(HashMap::new());
12453 }
12454 }
12455 })
12456 };
12457
12458 let (ack_tx, ack_rx) = oneshot::channel::<Result<(), Error>>();
12459 bg_event_tx
12460 .send(BackgroundEventEnvelope {
12461 event: BackgroundEvent::PartitionsRemoved {
12462 method_name: ConsumerRebalanceListenerMethodName::OnPartitionsRevoked,
12463 partitions: vec![tp],
12464 ack: ack_tx,
12465 ack_through_application_events: false,
12466 },
12467 enqueued_ms: 0,
12468 })
12469 .expect("bg event sent");
12470
12471 tokio::time::timeout(Duration::from_secs(5), consumer.poll(Duration::ZERO))
12472 .await
12473 .expect("poll() must not deadlock on the commit inside the listener")
12474 .expect("poll ok");
12475 ack_rx.await.expect("ack").expect("listener returned Ok");
12476 match listener.outcome.lock().unwrap().take() {
12477 Some(Ok(())) => {},
12478 other => panic!("commitSync() inside onPartitionsRevoked must succeed, got {other:?}"),
12479 }
12480 assert_eq!(
12481 commits.load(Ordering::SeqCst),
12482 1,
12483 "the in-callback commitSync sent one CommitSync event"
12484 );
12485 completer.abort();
12486 drop(subscriptions);
12487 }
12488
12489 /// §31 Test B: rebalance state machine does NOT advance until the
12490 /// listener future resolves.
12491 ///
12492 /// Setup: a listener whose `on_partitions_revoked` body blocks on
12493 /// a test-held channel. The test posts a
12494 /// `RebalanceListenerCallbackNeeded` event via the bg-events
12495 /// channel, observes that the `ack` is NOT received before the
12496 /// channel is released, then releases the channel and verifies
12497 /// the ack arrives.
12498 #[tokio::test]
12499 async fn section_31_rebalance_does_not_advance_until_listener_resolves() {
12500 use crate::consumer::internals::ConsumerRebalanceListenerMethodName;
12501 use async_trait::async_trait;
12502 use tokio::sync::oneshot;
12503 use tokio::time::timeout;
12504
12505 struct BlockingListener {
12506 release_rx: tokio::sync::Mutex<Option<oneshot::Receiver<()>>>,
12507 invoked: Arc<std::sync::atomic::AtomicBool>,
12508 }
12509 #[async_trait]
12510 impl ConsumerRebalanceListener for BlockingListener {
12511 async fn on_partitions_revoked(&self, _: &[TopicPartition]) -> Result<(), Error> {
12512 self.invoked.store(true, Ordering::SeqCst);
12513 // Take the receiver out of the mutex and await it —
12514 // the test side holds the matching sender and decides
12515 // when to release.
12516 let rx = self.release_rx.lock().await.take().expect("listener invoked exactly once");
12517 let _ = rx.await;
12518 Ok(())
12519 }
12520 async fn on_partitions_assigned(&self, _: &[TopicPartition]) -> Result<(), Error> {
12521 Ok(())
12522 }
12523 async fn on_partitions_lost(&self, _: &[TopicPartition]) -> Result<(), Error> {
12524 Ok(())
12525 }
12526 }
12527
12528 let (mut consumer, handles) = make_test_consumer_with_channels();
12529 let (release_tx, release_rx) = oneshot::channel::<()>();
12530 let invoked = Arc::new(std::sync::atomic::AtomicBool::new(false));
12531 let listener: Arc<BlockingListener> = Arc::new(BlockingListener {
12532 release_rx: tokio::sync::Mutex::new(Some(release_rx)),
12533 invoked: Arc::clone(&invoked),
12534 });
12535 *consumer.rebalance_listener.lock().unwrap() = Some(listener as Arc<dyn ConsumerRebalanceListener>);
12536
12537 // Post a `RebalanceListenerCallbackNeeded` event mimicking
12538 // the bg task's `enqueue_rebalance_callback`.
12539 let (ack_tx, ack_rx) = oneshot::channel::<Result<(), Error>>();
12540 handles
12541 .bg_event_tx
12542 .send(BackgroundEventEnvelope {
12543 event: BackgroundEvent::PartitionsRemoved {
12544 method_name: ConsumerRebalanceListenerMethodName::OnPartitionsRevoked,
12545 partitions: vec![TopicPartition::new("t".to_string(), 0)],
12546 ack: ack_tx,
12547 ack_through_application_events: false,
12548 },
12549 enqueued_ms: 0,
12550 })
12551 .expect("bg event sent");
12552
12553 // Drive process_background_events on a separate task; it will
12554 // block inside the listener body waiting on `release_rx`.
12555 let drainer = tokio::spawn(async move {
12556 consumer.process_background_events().await.expect("drain ok");
12557 });
12558
12559 // Sanity: the listener IS invoked, but the ack stays pending.
12560 // Give the drainer a tick of runtime.
12561 tokio::task::yield_now().await;
12562 tokio::time::sleep(Duration::from_millis(20)).await;
12563 assert!(invoked.load(Ordering::SeqCst), "listener body must have been entered");
12564
12565 // Ack must NOT have arrived yet — the listener is blocked on
12566 // the release channel.
12567 let ack_pending = timeout(Duration::from_millis(50), async {
12568 // Re-create a fresh receiver borrow via select — we can't
12569 // poll the rx without consuming. Use a select! pattern
12570 // to test "would the recv resolve right now".
12571 let mut rx_pin = std::pin::pin!(&mut { ack_rx });
12572 tokio::select! {
12573 biased;
12574 _ = &mut rx_pin => false, // ack arrived — bad
12575 _ = tokio::time::sleep(Duration::from_millis(25)) => true, // still pending — good
12576 }
12577 })
12578 .await
12579 .expect("timeout outer guard");
12580 assert!(
12581 ack_pending,
12582 "ack must NOT arrive before listener future resolves (§31 contract)"
12583 );
12584
12585 // Now release the listener.
12586 let _ = release_tx.send(());
12587 drainer.await.expect("drainer ok");
12588 // The drainer completing means process_background_events
12589 // sent the ack (Ok(())) and returned — the §31 advancement
12590 // happened only after the listener future resolved.
12591 }
12592
12593 // ─── Poll lifecycle tests (commit 4/N) ───
12594 //
12595 // Stand-ins for Java's `testWakeupBeforeCallingPoll`, `testWakeupAfterEmptyFetch`,
12596 // `testClearWakeupTriggerAfterPoll`, the `checkInflightPoll` arms, and the
12597 // "no subscription / no assignment" early-return arm.
12598 //
12599 // Skipped Java tests for this commit (each carries a one-line rationale):
12600 // - `testRecordBackgroundEventQueueSizeAndBackgroundEventQueueTime` —
12601 // `AsyncConsumerMetrics` is now WIRED (Phase M6) and verified by the
12602 // handler-side smoke tests; the end-to-end value assertion under a
12603 // mock clock awaits the M7 public `metrics()` accessor + MockTime
12604 // fixture (see the matching skip note in the commit-8 batch above).
12605 // - `testReaperInvokedInPoll` — depends on the metrics observers that
12606 // would observe the reaper invocations. The `reap` call itself is
12607 // wired and unit-tested via the bg-events drain test in commit 3.
12608
12609 /// Java: `poll()` throws `IllegalStateException` when there is no
12610 /// subscription or assignment.
12611 #[tokio::test]
12612 async fn poll_returns_illegal_state_without_subscription() {
12613 let (mut consumer, _handles) = make_test_consumer_with_channels();
12614 let err = consumer.poll(Duration::from_millis(0)).await.expect_err("must err");
12615 assert!(
12616 matches!(err, Error::LocalIllegalState(ref msg)
12617 if msg.message() == "Consumer is not subscribed to any topics or assigned any partitions"),
12618 "unexpected err: {err:?}"
12619 );
12620 }
12621
12622 /// Java: `testFailOnClosedConsumer` (the `poll` arm).
12623 #[tokio::test]
12624 async fn poll_on_closed_consumer_errors() {
12625 let (mut consumer, _handles) = make_test_consumer_with_channels();
12626 consumer.closed.store(true, Ordering::Release);
12627 let err = consumer.poll(Duration::from_millis(0)).await.expect_err("must err");
12628 assert!(
12629 matches!(err, Error::LocalIllegalState(ref msg)
12630 if msg.message().contains("already been closed")),
12631 "unexpected err: {err:?}"
12632 );
12633 }
12634
12635 /// Java: `testWakeupBeforeCallingPoll` — `wakeup()` posted before
12636 /// `poll()` must surface as `Error::Wakeup`. After the error is
12637 /// raised, a subsequent `poll()` must observe a fresh token (Java's
12638 /// "clear the volatile flag after throwing WakeupException once").
12639 #[tokio::test]
12640 async fn poll_observes_pending_wakeup_and_rotates_token() {
12641 let (mut consumer, handles) = make_test_consumer_with_channels();
12642 // Pre-populate an assignment so `poll()` does not bail with the
12643 // "not subscribed" error before reaching the wakeup check.
12644 let tp = TopicPartition::new("t".to_string(), 0);
12645 {
12646 let mut subs = consumer.subscriptions.lock().unwrap();
12647 let mut assigned: HashSet<TopicPartition> = HashSet::new();
12648 assigned.insert(tp.clone());
12649 subs.assign_from_user(assigned).unwrap();
12650 }
12651
12652 consumer.wakeup_trigger.wakeup();
12653 let pre_token = consumer.wakeup_trigger.current_token();
12654 assert!(pre_token.is_cancelled(), "pre-condition: wakeup() cancelled the token");
12655
12656 let err = consumer.poll(Duration::from_millis(0)).await.expect_err("wakeup err");
12657 assert!(matches!(err, Error::Wakeup(_)), "unexpected err: {err:?}");
12658
12659 // After raising the wakeup error the consumer must have rotated
12660 // the token so the next poll observes a fresh one (§11).
12661 let post_token = consumer.wakeup_trigger.current_token();
12662 assert!(!post_token.is_cancelled(), "token must be rotated after Wakeup err");
12663
12664 drop(handles);
12665 }
12666
12667 /// `checkInflightPoll` submits a fresh AsyncPollEvent when none is
12668 /// in flight, and clears it on completion. Mirrors the Java unit-test
12669 /// behavior in `testEnsurePollEventSentOnConsumerPoll`.
12670 #[tokio::test]
12671 async fn poll_enqueues_async_poll_event_and_clears_on_completion() {
12672 let (mut consumer, mut handles) = make_test_consumer_with_channels();
12673 // Pre-populate the assignment so `poll()` does not error early.
12674 let tp = TopicPartition::new("t".to_string(), 0);
12675 {
12676 let mut subs = consumer.subscriptions.lock().unwrap();
12677 let mut assigned: HashSet<TopicPartition> = HashSet::new();
12678 assigned.insert(tp.clone());
12679 subs.assign_from_user(assigned).unwrap();
12680 }
12681
12682 // First poll() — submits AsyncPoll, returns empty. We don't yet
12683 // drain the event off the channel, so the inflight state is
12684 // present and incomplete.
12685 let records = consumer.poll(Duration::from_millis(0)).await.expect("poll ok");
12686 assert!(records.is_empty(), "empty fetch buffer should yield empty records");
12687 assert!(consumer.inflight_poll.is_some(), "first poll() submits an AsyncPoll event");
12688
12689 // Drain the envelope off the channel and synchronously mark the
12690 // shared state complete (mirrors the bg task's behavior).
12691 let env = handles.app_event_rx.try_recv().expect("AsyncPoll envelope must be on channel");
12692 let async_poll_state = match env.event {
12693 ApplicationEvent::AsyncPoll { state, .. } => state,
12694 other => panic!("expected AsyncPoll, got {}", other.type_name()),
12695 };
12696 async_poll_state.complete_successfully();
12697
12698 // Second poll() drives `maybe_clear_previous_inflight_poll` over
12699 // the now-completed state and clears the inflight slot. The bg
12700 // task channel is still drained on each invocation; the new
12701 // AsyncPoll for this iteration goes back onto the channel and we
12702 // do not bother completing it.
12703 let _ = consumer.poll(Duration::from_millis(0)).await.expect("second poll ok");
12704 // Both the previously-completed event (cleared because complete)
12705 // and the freshly-submitted one for this iteration are accounted
12706 // for: after the second poll the inflight slot holds the new
12707 // (incomplete) event because `maybe_clear_current_inflight_poll`
12708 // is run with `newly_submitted_event=true`.
12709 assert!(
12710 consumer.inflight_poll.is_some(),
12711 "second poll() submits a fresh AsyncPoll event"
12712 );
12713 }
12714
12715 /// A previous-iteration AsyncPoll event that completed with an error
12716 /// is surfaced from the next `poll()` call (mirrors Java's
12717 /// `maybeClearPreviousInflightPoll` error arm).
12718 #[tokio::test]
12719 async fn poll_surfaces_previous_inflight_poll_error() {
12720 let (mut consumer, _handles) = make_test_consumer_with_channels();
12721 let tp = TopicPartition::new("t".to_string(), 0);
12722 {
12723 let mut subs = consumer.subscriptions.lock().unwrap();
12724 let mut assigned: HashSet<TopicPartition> = HashSet::new();
12725 assigned.insert(tp);
12726 subs.assign_from_user(assigned).unwrap();
12727 }
12728 // Plant a previous inflight that already completed with an error.
12729 let state = Arc::new(AsyncPollState::new());
12730 state.complete_with_error(Error::timeout("prior poll deadline"));
12731 consumer.inflight_poll = Some(InflightPoll { deadline_ms: 0, state });
12732
12733 let err = consumer.poll(Duration::from_millis(0)).await.expect_err("must err");
12734 assert!(
12735 matches!(err, Error::Timeout(ref m) if m.message() == "prior poll deadline"),
12736 "unexpected err: {err:?}"
12737 );
12738 assert!(
12739 consumer.inflight_poll.is_none(),
12740 "previous inflight poll must be cleared after error"
12741 );
12742 }
12743
12744 // ─── Commit tests (commit 5/N) ───
12745 //
12746 // Stand-ins for Java's
12747 // - `testCommitAsyncWithNullCallback`
12748 // - `testCommitAsyncUserSuppliedCallbackNoException`
12749 // - `testCommitAsyncShouldCopyOffsets`
12750 // - `testCommittedExceptionThrown`
12751 // - `testEnsureCommitSyncExecutedCommitAsyncCallbacks`
12752 //
12753 // Skipped Java tests for this commit:
12754 // - `testCommitInRebalanceCallback` — translated as
12755 // `test_commit_in_rebalance_callback` (§31 regression pair).
12756 // - `testCommitAsyncUserSuppliedCallbackWithException` (parameterized) —
12757 // unit tested in `OffsetCommitCallbackInvoker` tests; the consumer
12758 // side just routes the error.
12759 // - Tests that exercise auto-commit-on-close — covered in commit 7/N.
12760
12761 /// Helper: build a HashMap with a single tp -> offset entry.
12762 fn singleton_offsets(tp: TopicPartition, offset: i64) -> HashMap<TopicPartition, OffsetAndMetadata> {
12763 let mut m = HashMap::new();
12764 m.insert(tp, OffsetAndMetadata::new(offset).expect("non-neg"));
12765 m
12766 }
12767
12768 /// `commit_async` without a callback enqueues a `CommitAsync` event
12769 /// against the bg task. Mirrors Java's `testCommitAsyncWithNullCallback`.
12770 #[tokio::test]
12771 async fn commit_async_with_no_callback_enqueues_commit_async_event() {
12772 let (mut consumer, mut handles) = make_test_consumer_with_channels();
12773 // Spawn a completer: pull CommitAsync envelope, complete both
12774 // its `offsets_ready` and `handle`.
12775 let completer = tokio::spawn(async move {
12776 while let Some(env) = handles.app_event_rx.recv().await {
12777 if let ApplicationEvent::CommitAsync { handle, offsets_ready, .. } = env.event {
12778 offsets_ready.complete(());
12779 handle.complete(HashMap::new());
12780 return true;
12781 }
12782 }
12783 false
12784 });
12785 consumer.commit_async().await.expect("ok");
12786 assert!(completer.await.expect("task ok"), "must see CommitAsync envelope");
12787 }
12788
12789 /// `commit_sync` runs the interceptor `on_commit` chain on the
12790 /// committed offsets. Mirrors Java's `testInterceptorOnCommit`.
12791 /// Issue 17 — registers a real tracking interceptor on the consumer
12792 /// and asserts it was invoked with the committed offsets after
12793 /// `commit_sync_with_offsets` returns.
12794 #[tokio::test]
12795 async fn commit_sync_invokes_interceptor_chain() {
12796 use crate::consumer::ConsumerInterceptor;
12797 use crate::consumer::internals::ConsumerInterceptors;
12798
12799 // Tracking interceptor — records every `on_commit` call.
12800 struct TrackingInterceptor {
12801 recorded: Arc<std::sync::Mutex<Vec<HashMap<TopicPartition, OffsetAndMetadata>>>>,
12802 }
12803 impl ConsumerInterceptor<Vec<u8>, Vec<u8>> for TrackingInterceptor {
12804 fn on_consume(&self, _records: &mut ConsumerRecords<Vec<u8>, Vec<u8>>) {}
12805 fn on_commit(&self, offsets: &HashMap<TopicPartition, OffsetAndMetadata>) {
12806 self.recorded.lock().unwrap().push(offsets.clone());
12807 }
12808 }
12809
12810 let (mut consumer, mut handles) = make_test_consumer_with_channels();
12811
12812 // Register the tracking interceptor on the consumer.
12813 let recorded: Arc<std::sync::Mutex<Vec<HashMap<TopicPartition, OffsetAndMetadata>>>> =
12814 Arc::new(std::sync::Mutex::new(Vec::new()));
12815 let tracker: Box<dyn ConsumerInterceptor<Vec<u8>, Vec<u8>>> =
12816 Box::new(TrackingInterceptor { recorded: Arc::clone(&recorded) });
12817 let new_chain = ConsumerInterceptors::<Vec<u8>, Vec<u8>>::new(vec![tracker]);
12818 *consumer.interceptors.lock().unwrap() = new_chain;
12819
12820 let tp = TopicPartition::new("t".to_string(), 0);
12821 let offsets = singleton_offsets(tp.clone(), 42);
12822
12823 // Spawn a completer to complete the CommitSync envelope with the
12824 // committed offsets.
12825 let offsets_for_completer = offsets.clone();
12826 let completer = tokio::spawn(async move {
12827 while let Some(env) = handles.app_event_rx.recv().await {
12828 if let ApplicationEvent::CommitSync { handle, offsets_ready, .. } = env.event {
12829 offsets_ready.complete(());
12830 handle.complete(offsets_for_completer.clone());
12831 return true;
12832 }
12833 }
12834 false
12835 });
12836
12837 consumer.commit_sync_with_offsets(offsets.clone()).await.expect("ok");
12838 assert!(completer.await.expect("task ok"));
12839
12840 // Verify the tracking interceptor was invoked exactly once with
12841 // the committed offsets (Java parity: `testInterceptorOnCommit`).
12842 let recorded_snapshot = recorded.lock().unwrap().clone();
12843 assert_eq!(recorded_snapshot.len(), 1, "on_commit must be invoked exactly once");
12844 assert_eq!(recorded_snapshot[0].get(&tp).map(|v| v.offset()), Some(42));
12845 }
12846
12847 /// `commit_async` with an empty offsets map short-circuits without
12848 /// enqueuing a CommitAsync event (Java's `if (offsets.isPresent() &&
12849 /// offsets.get().isEmpty()) return completedFuture(null)`).
12850 #[tokio::test]
12851 async fn commit_async_with_empty_offsets_short_circuits() {
12852 let (mut consumer, mut handles) = make_test_consumer_with_channels();
12853 consumer
12854 .commit_async_with_offsets_callback(HashMap::new(), Arc::new(NoopCallback))
12855 .await
12856 .expect("ok");
12857 // No envelope should be on the channel.
12858 let env = handles.app_event_rx.try_recv();
12859 assert!(env.is_err(), "expected no envelope, got {env:?}");
12860 }
12861
12862 /// `commit_sync` on a groupless consumer errors with the Rust
12863 /// analog of Java's `InvalidGroupIdException` —
12864 /// `Error::invalid_group_id(...)` (Issue 16). Mirrors Java's
12865 /// `testCommitSyncWithoutGroupId`.
12866 #[tokio::test]
12867 async fn commit_sync_without_group_id_errors() {
12868 use crate::common::protocol::Errors;
12869 let (mut consumer, _handles) = make_test_consumer_with_channels();
12870 consumer.group_id = None;
12871 let err = consumer.commit_sync().await.expect_err("must err");
12872 assert_eq!(err.error(), Errors::InvalidGroupId, "expected InvalidGroupId, got {err:?}");
12873 assert!(
12874 err.message().contains("group.id"),
12875 "message must reference group.id, got: {}",
12876 err.message()
12877 );
12878 }
12879
12880 /// `commit_sync` after a pending async commit drains the async first
12881 /// (mirrors Java's `testEnsureCommitSyncExecutedCommitAsyncCallbacks`
12882 /// — the assertion is observable side: `last_pending_async_commit`
12883 /// is consumed by the sync path).
12884 #[tokio::test]
12885 async fn commit_sync_drains_pending_async_commit() {
12886 let (mut consumer, mut handles) = make_test_consumer_with_channels();
12887 // Spawn a single completer that races for both events.
12888 let async_completer = tokio::spawn(async move {
12889 while let Some(env) = handles.app_event_rx.recv().await {
12890 match env.event {
12891 ApplicationEvent::CommitAsync { handle, offsets_ready, .. } => {
12892 offsets_ready.complete(());
12893 handle.complete(HashMap::new());
12894 },
12895 ApplicationEvent::CommitSync { handle, offsets_ready, .. } => {
12896 offsets_ready.complete(());
12897 handle.complete(HashMap::new());
12898 return true;
12899 },
12900 _ => {},
12901 }
12902 }
12903 false
12904 });
12905 consumer.commit_async().await.expect("async ok");
12906 assert!(
12907 consumer.commit_state.has_last_pending_async_commit(),
12908 "async commit must register pending"
12909 );
12910 consumer.commit_sync().await.expect("sync ok");
12911 assert!(
12912 !consumer.commit_state.has_last_pending_async_commit(),
12913 "sync must consume the pending async commit"
12914 );
12915 assert!(async_completer.await.expect("task ok"));
12916 }
12917
12918 /// Test-only callback that records no state. Used to keep the
12919 /// `commit_async_with_offsets_callback` arg slot non-null in tests
12920 /// that don't observe the callback firing.
12921 struct NoopCallback;
12922 #[async_trait::async_trait]
12923 impl crate::consumer::OffsetCommitCallback for NoopCallback {
12924 async fn on_complete(
12925 &self,
12926 _offsets: &HashMap<TopicPartition, OffsetAndMetadata>,
12927 _error: Option<&Error>,
12928 ) -> Result<(), Error> {
12929 Ok(())
12930 }
12931 }
12932
12933 // ─── Seek / position / committed / lag tests (commit 6/N) ───
12934
12935 /// `seek` with a negative offset rejects with `LocalIllegalArgument`.
12936 /// Java: `seek` throws `IllegalArgumentException("seek offset must not
12937 /// be a negative number")`.
12938 #[tokio::test]
12939 async fn seek_rejects_negative_offset() {
12940 let (mut consumer, _handles) = make_test_consumer_with_channels();
12941 let tp = TopicPartition::new("t".to_string(), 0);
12942 let err = consumer.seek_with_offset(tp, -1).await.expect_err("must err");
12943 assert!(matches!(err, Error::LocalIllegalArgument(_)), "unexpected err: {err:?}");
12944 }
12945
12946 /// `seek` enqueues a `SeekUnvalidated` event. Mirrors Java's
12947 /// `testSeek` event-shape assertion.
12948 #[tokio::test]
12949 async fn seek_enqueues_seek_unvalidated_event() {
12950 let (mut consumer, handles) = make_test_consumer_with_channels();
12951 let completer = auto_complete_next_event(handles.app_event_rx);
12952 let tp = TopicPartition::new("t".to_string(), 0);
12953 consumer.seek_with_offset(tp.clone(), 42).await.expect("ok");
12954 let env = completer.await.expect("task ok").expect("event received");
12955 assert!(matches!(env.event, ApplicationEvent::SeekUnvalidated { partition, offset, .. }
12956 if partition == tp && offset == 42));
12957 }
12958
12959 /// `position` on an unassigned partition returns `LocalIllegalState`.
12960 #[tokio::test]
12961 async fn position_on_unassigned_partition_errors() {
12962 let (mut consumer, _handles) = make_test_consumer_with_channels();
12963 let tp = TopicPartition::new("t".to_string(), 0);
12964 let err = consumer
12965 .position_with_timeout(&tp, Duration::from_millis(0))
12966 .await
12967 .expect_err("must err");
12968 assert!(matches!(err, Error::LocalIllegalState(_)), "unexpected err: {err:?}");
12969 }
12970
12971 /// `committed` on an empty partition set returns an empty map without
12972 /// enqueuing an event. Java: `if (partitions.isEmpty()) return
12973 /// Collections.emptyMap();`.
12974 #[tokio::test]
12975 async fn committed_with_empty_partitions_returns_empty_map() {
12976 let (mut consumer, mut handles) = make_test_consumer_with_channels();
12977 let map = consumer
12978 .committed_with_timeout(&[], Duration::from_millis(100))
12979 .await
12980 .expect("ok");
12981 assert!(map.is_empty());
12982 assert!(handles.app_event_rx.try_recv().is_err(), "no event enqueued");
12983 }
12984
12985 /// `committed` without group_id errors with the Rust analog of
12986 /// Java's `InvalidGroupIdException` —
12987 /// `Error::invalid_group_id(...)` (Issue 16).
12988 #[tokio::test]
12989 async fn committed_without_group_id_errors() {
12990 use crate::common::protocol::Errors;
12991 let (mut consumer, _handles) = make_test_consumer_with_channels();
12992 consumer.group_id = None;
12993 let tp = TopicPartition::new("t".to_string(), 0);
12994 let err = consumer
12995 .committed_with_timeout(std::slice::from_ref(&tp), Duration::from_millis(0))
12996 .await
12997 .expect_err("must err");
12998 assert_eq!(err.error(), Errors::InvalidGroupId, "expected InvalidGroupId, got {err:?}");
12999 }
13000
13001 /// `pause` with empty input is a no-op (matches Java's
13002 /// `if (!partitions.isEmpty()) addAndGet(...)` short-circuit).
13003 #[tokio::test]
13004 async fn pause_with_empty_set_is_noop() {
13005 let (mut consumer, mut handles) = make_test_consumer_with_channels();
13006 consumer.pause(&[]).await.expect("ok");
13007 assert!(handles.app_event_rx.try_recv().is_err(), "no event enqueued");
13008 }
13009
13010 /// `resume` with empty input is a no-op (matches Java's
13011 /// `if (!partitions.isEmpty()) addAndGet(...)` short-circuit).
13012 #[tokio::test]
13013 async fn resume_with_empty_set_is_noop() {
13014 let (mut consumer, mut handles) = make_test_consumer_with_channels();
13015 consumer.resume(&[]).await.expect("ok");
13016 assert!(handles.app_event_rx.try_recv().is_err(), "no event enqueued");
13017 }
13018
13019 /// `enforce_rebalance` is a documented no-op under KIP-848 (Java
13020 /// `log.warn("Operation not supported in new consumer group protocol")`).
13021 #[tokio::test]
13022 async fn enforce_rebalance_is_noop() {
13023 let (mut consumer, _handles) = make_test_consumer_with_channels();
13024 consumer.enforce_rebalance().await.expect("ok");
13025 consumer.enforce_rebalance_with_reason("test reason").await.expect("ok");
13026 }
13027
13028 /// `offsets_for_times` rejects negative timestamps.
13029 #[tokio::test]
13030 async fn offsets_for_times_rejects_negative_timestamp() {
13031 let (mut consumer, _handles) = make_test_consumer_with_channels();
13032 let mut ts = HashMap::new();
13033 ts.insert(TopicPartition::new("t".to_string(), 0), -5);
13034 let err = consumer
13035 .offsets_for_times_with_timeout(ts, Duration::from_millis(0))
13036 .await
13037 .expect_err("must err");
13038 assert!(
13039 matches!(err, Error::LocalIllegalArgument(ref msg) if msg.message().contains("negative")),
13040 "unexpected err: {err:?}"
13041 );
13042 }
13043
13044 /// `offsets_for_times` with empty map returns empty map.
13045 #[tokio::test]
13046 async fn offsets_for_times_with_empty_map_returns_empty() {
13047 let (mut consumer, _handles) = make_test_consumer_with_channels();
13048 let map = consumer
13049 .offsets_for_times_with_timeout(HashMap::new(), Duration::from_millis(0))
13050 .await
13051 .expect("ok");
13052 assert!(map.is_empty());
13053 }
13054
13055 /// `beginning_offsets` with empty input returns empty map.
13056 #[tokio::test]
13057 async fn beginning_offsets_with_empty_input_returns_empty() {
13058 let (mut consumer, _handles) = make_test_consumer_with_channels();
13059 let map = consumer
13060 .beginning_offsets_with_timeout(&[], Duration::from_millis(0))
13061 .await
13062 .expect("ok");
13063 assert!(map.is_empty());
13064 }
13065
13066 /// `partitions_for` with zero timeout and empty metadata cache
13067 /// errors with `Timeout`. Java: `if (timeout.toMillis() == 0L) throw
13068 /// new TimeoutException()`.
13069 #[tokio::test]
13070 async fn partitions_for_with_zero_timeout_and_empty_metadata_errors() {
13071 let (mut consumer, _handles) = make_test_consumer_with_channels();
13072 let err = consumer
13073 .partitions_for_with_timeout("t", Duration::from_millis(0))
13074 .await
13075 .expect_err("must err");
13076 assert!(matches!(err, Error::Timeout(_)), "unexpected err: {err:?}");
13077 }
13078
13079 /// `list_topics` with zero timeout errors with `Timeout`.
13080 #[tokio::test]
13081 async fn list_topics_with_zero_timeout_errors() {
13082 let (mut consumer, _handles) = make_test_consumer_with_channels();
13083 let err = consumer
13084 .list_topics_with_timeout(Duration::from_millis(0))
13085 .await
13086 .expect_err("must err");
13087 assert!(matches!(err, Error::Timeout(_)), "unexpected err: {err:?}");
13088 }
13089
13090 // ─── Phase 11 commit (9/N) Java test translations: poll / commit / wakeup ───
13091 //
13092 // Translates Java's `AsyncKafkaConsumerTest` poll-loop, commit-flow,
13093 // and wakeup tests. Many already have an inline analog (commits 4-5);
13094 // each Java test is either translated here or carries an explicit
13095 // `// SKIP: covered by <existing_test>` rationale.
13096 //
13097 // SKIPs (commit 9 batch):
13098 // - testWakeupCommitted — covered by inline
13099 // `issue_11_committed_observes_wakeup_during_wait`.
13100 // - testEnsureCommitSyncExecutedCommitAsyncCallbacks — callback-fire
13101 // unit-tested in `OffsetCommitCallbackInvoker` + the inline
13102 // `commit_sync_drains_pending_async_commit` covers the drain path.
13103 // - testEnsureCallbackExecutedByApplicationThread — Rust's
13104 // `&mut self` API guarantees callbacks run on the caller's task;
13105 // no separate thread-identity assertion translates.
13106 // - testEnsurePollExecutedCommitAsyncCallbacks /
13107 // testEnsureShutdownExecutedCommitAsyncCallbacks — callback-fire
13108 // wire-up exercised via `OffsetCommitCallbackInvoker` unit tests
13109 // and via the close drainer that completes CommitAsync envelopes.
13110 // - testCommitAsyncWithNullCallback — covered by the inline
13111 // `commit_async_with_no_callback_enqueues_commit_async_event`.
13112 // - testInterceptorCommitSync / testInterceptorCommitAsync /
13113 // testInterceptorAutoCommitOnClose / testNoInterceptorCommitSyncFailed /
13114 // testNoInterceptorCommitAsyncFailed — interceptor-tracking path
13115 // deferred to Issue 17 (commit 12 batch).
13116 // - testWakeupAfterEmptyFetch / testWakeupAfterNonEmptyFetch —
13117 // require a fully-wired FetchCollector observable that signals
13118 // "wakeup mid-fetch". Without a `MockClient`-backed bg task this
13119 // would only exercise the wakeup_trigger plumbing already tested
13120 // by `wakeup_before_poll_throws_once_then_succeeds`. Requires
13121 // Fetch response routing (deferred to Phase 12.5 per
13122 // `Phase-12/RESPONSE-ROUTING-AUDIT.md`).
13123 // - testCommitted — full happy-path commit fetch — requires a
13124 // completer that returns offsets through the FetchCommittedOffsets
13125 // handle; the new `committed_propagates_event_error` exercises
13126 // the same path with an error variant. Phase 12.5 integration
13127 // tests cover the happy path against a real broker.
13128 // - testPollThrowsInterruptExceptionIfInterrupted — Java's
13129 // `Thread.currentThread().interrupt()` is unrepresentable in
13130 // Rust (no thread-level interrupt flag). The equivalent is
13131 // cancellation via the consumer's `wakeup_trigger`, covered by
13132 // `wakeup_before_poll_throws_once_then_succeeds`.
13133 // - testListenerCallbacksInvoke (parameterized) — exercises the
13134 // §31 callback-invocation pipeline. The §31 regression pair in
13135 // commit 11/N covers the same surface with stricter scenarios.
13136
13137 /// Java: `testWakeupBeforeCallingPoll`. The
13138 /// inline `poll_observes_pending_wakeup_and_rotates_token` already
13139 /// asserts the first half (wakeup-before-poll surfaces Wakeup). The
13140 /// new piece is the second half: a subsequent `poll()` does NOT
13141 /// throw (token was rotated, fresh poll succeeds).
13142 #[tokio::test]
13143 async fn wakeup_before_poll_throws_once_then_succeeds() {
13144 let (mut consumer, mut handles) = make_test_consumer_with_channels();
13145 let tp = TopicPartition::new("foo".to_string(), 3);
13146 {
13147 let mut subs = consumer.subscriptions.lock().unwrap();
13148 let mut assigned: HashSet<TopicPartition> = HashSet::new();
13149 assigned.insert(tp.clone());
13150 subs.assign_from_user(assigned).unwrap();
13151 }
13152
13153 // Drainer: complete every AsyncPoll envelope so successful
13154 // polls can land.
13155 let drainer = tokio::spawn(async move {
13156 while let Some(env) = handles.app_event_rx.recv().await {
13157 if let ApplicationEvent::AsyncPoll { state, .. } = env.event {
13158 state.complete_successfully();
13159 }
13160 }
13161 });
13162
13163 consumer.wakeup_trigger.wakeup();
13164 let err = consumer.poll(Duration::from_millis(0)).await.expect_err("wakeup");
13165 assert!(matches!(err, Error::Wakeup(_)));
13166
13167 // Second poll: must NOT raise (Java's `assertDoesNotThrow`).
13168 let _ = consumer.poll(Duration::from_millis(0)).await.expect("ok");
13169 drop(drainer);
13170 }
13171
13172 /// Java: `testClearWakeupTriggerAfterPoll`. After
13173 /// a successful `poll()`, the wakeup trigger is cleared so the next
13174 /// `poll()` does not see a stale wakeup signal.
13175 #[tokio::test]
13176 async fn clear_wakeup_trigger_after_poll() {
13177 let (mut consumer, mut handles) = make_test_consumer_with_channels();
13178 let tp = TopicPartition::new("foo".to_string(), 3);
13179 {
13180 let mut subs = consumer.subscriptions.lock().unwrap();
13181 let mut assigned: HashSet<TopicPartition> = HashSet::new();
13182 assigned.insert(tp.clone());
13183 subs.assign_from_user(assigned).unwrap();
13184 }
13185
13186 let drainer = tokio::spawn(async move {
13187 while let Some(env) = handles.app_event_rx.recv().await {
13188 if let ApplicationEvent::AsyncPoll { state, .. } = env.event {
13189 state.complete_successfully();
13190 }
13191 }
13192 });
13193
13194 // First poll completes successfully.
13195 let _ = consumer.poll(Duration::from_millis(0)).await.expect("ok");
13196 // Second poll must not raise.
13197 let _ = consumer.poll(Duration::from_millis(0)).await.expect("ok");
13198 // Wakeup trigger has no pending task.
13199 assert!(!consumer.wakeup_trigger.current_token().is_cancelled());
13200 drop(drainer);
13201 }
13202
13203 /// Java: `testCommittedExceptionThrown`. When
13204 /// the `FetchCommittedOffsetsEvent` is completed exceptionally, the
13205 /// error propagates out of `committed_with_timeout`.
13206 #[tokio::test]
13207 async fn committed_propagates_event_error() {
13208 let (mut consumer, mut handles) = make_test_consumer_with_channels();
13209 // Drainer completes the FetchCommittedOffsets handle with an error.
13210 let drainer = tokio::spawn(async move {
13211 while let Some(env) = handles.app_event_rx.recv().await {
13212 if let ApplicationEvent::FetchCommittedOffsets { handle, .. } = env.event {
13213 handle.complete_with_error(Error::local_illegal_state("Test error"));
13214 return;
13215 }
13216 }
13217 });
13218
13219 let tp = TopicPartition::new("t0".to_string(), 2);
13220 let err = consumer
13221 .committed_with_timeout(std::slice::from_ref(&tp), Duration::from_secs(1))
13222 .await
13223 .expect_err("must err");
13224 // Java surfaces this as `KafkaException`; Rust surfaces the
13225 // underlying Error.
13226 assert!(!matches!(err, Error::Timeout(_)), "non-timeout err propagates, got {err:?}");
13227 drainer.await.expect("drainer ok");
13228 }
13229
13230 /// Java: `testCommitAsyncShouldCopyOffsets` —
13231 /// commit_async must capture a copy of the user-supplied offsets;
13232 /// post-call modifications to the user's map must not affect the
13233 /// enqueued event.
13234 ///
13235 /// Rust translation note: Rust's ownership semantics make this
13236 /// inherent — the consumer takes the `HashMap` by value
13237 /// (`commit_async_with_offsets_callback(offsets: HashMap<...>, ...)`)
13238 /// — so the test asserts that the event's snapshot is the same as
13239 /// the input.
13240 #[tokio::test]
13241 async fn commit_async_captures_offsets() {
13242 let (mut consumer, mut handles) = make_test_consumer_with_channels();
13243 let tp = TopicPartition::new("t0".to_string(), 2);
13244 let offsets = singleton_offsets(tp.clone(), 10);
13245
13246 let offsets_for_assert = offsets.clone();
13247 let completer = tokio::spawn(async move {
13248 while let Some(env) = handles.app_event_rx.recv().await {
13249 if let ApplicationEvent::CommitAsync { handle, offsets_ready, offsets: ev_offsets } = env.event {
13250 // The event's offsets must contain the same entries.
13251 let captured = ev_offsets.expect("CommitAsync.offsets is Some");
13252 assert_eq!(captured.get(&tp).map(|v| v.offset()), Some(10));
13253 assert_eq!(captured.len(), offsets_for_assert.len());
13254 offsets_ready.complete(());
13255 handle.complete(HashMap::new());
13256 return true;
13257 }
13258 }
13259 false
13260 });
13261
13262 consumer
13263 .commit_async_with_offsets_callback(offsets, Arc::new(NoopCallback))
13264 .await
13265 .expect("ok");
13266 assert!(completer.await.expect("task ok"));
13267 }
13268
13269 /// Java: `testCommitSyncShouldCopyOffsets` — the
13270 /// symmetric test for `commit_sync_with_offsets`.
13271 #[tokio::test]
13272 async fn commit_sync_captures_offsets() {
13273 let (mut consumer, mut handles) = make_test_consumer_with_channels();
13274 let tp = TopicPartition::new("t0".to_string(), 2);
13275 let offsets = singleton_offsets(tp.clone(), 10);
13276
13277 let completer = tokio::spawn(async move {
13278 while let Some(env) = handles.app_event_rx.recv().await {
13279 if let ApplicationEvent::CommitSync { handle, offsets_ready, offsets: ev_offsets } = env.event {
13280 let captured = ev_offsets.expect("CommitSync.offsets is Some");
13281 assert_eq!(captured.get(&tp).map(|v| v.offset()), Some(10));
13282 offsets_ready.complete(());
13283 handle.complete(HashMap::new());
13284 return true;
13285 }
13286 }
13287 false
13288 });
13289
13290 consumer.commit_sync_with_offsets(offsets).await.expect("ok");
13291 assert!(completer.await.expect("task ok"));
13292 }
13293
13294 /// Java: `testBackgroundError`. An
13295 /// `ErrorEvent` posted by the bg task surfaces from `poll()`.
13296 #[tokio::test]
13297 async fn poll_surfaces_single_background_error() {
13298 let (mut consumer, handles) = make_test_consumer_with_channels();
13299 let tp = TopicPartition::new("t".to_string(), 0);
13300 {
13301 let mut subs = consumer.subscriptions.lock().unwrap();
13302 let mut assigned: HashSet<TopicPartition> = HashSet::new();
13303 assigned.insert(tp);
13304 subs.assign_from_user(assigned).unwrap();
13305 }
13306 // Post an error to the bg event channel BEFORE calling poll —
13307 // poll's process_background_events drain must surface it.
13308 handles
13309 .bg_event_tx
13310 .send(BackgroundEventEnvelope {
13311 event: BackgroundEvent::Error {
13312 error: Error::local_illegal_state("Nobody expects the Spanish Inquisition"),
13313 },
13314 enqueued_ms: 0,
13315 })
13316 .expect("send ok");
13317
13318 let err = consumer.poll(Duration::from_millis(0)).await.expect_err("error must surface");
13319 let msg = format!("{err}");
13320 assert!(
13321 msg.contains("Nobody expects the Spanish Inquisition"),
13322 "expected the bg error message, got: {msg}"
13323 );
13324 // Java wraps each background-event failure through
13325 // `ConsumerUtils.maybeWrapAsKafkaException(t)`
13326 // in `org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumer#processBackgroundEvents(boolean)`, so what
13327 // reaches the application is always a `KafkaException` — even though the bg task raised a generic
13328 // `IllegalStateException`.
13329 assert!(
13330 err.is_kafka_error(),
13331 "the background error must be wrapped into the Kafka error hierarchy: {err:?}"
13332 );
13333 assert!(
13334 !matches!(err, Error::LocalIllegalState(_)),
13335 "must not surface as a raw IllegalState: {err:?}"
13336 );
13337 // The original is still reachable as the cause (Java's `getCause()`).
13338 let source = std::error::Error::source(&err).expect("the original error must be the cause");
13339 assert!(
13340 source.to_string().contains("Nobody expects the Spanish Inquisition"),
13341 "cause must be the original bg error, got: {source}"
13342 );
13343 }
13344
13345 /// Java: `testMultipleBackgroundErrors`. When
13346 /// multiple errors are queued, only the FIRST is surfaced; the rest
13347 /// remain on the queue or are silently consumed. Java asserts the
13348 /// queue is empty after.
13349 #[tokio::test]
13350 async fn poll_surfaces_first_background_error_only() {
13351 let (mut consumer, handles) = make_test_consumer_with_channels();
13352 let tp = TopicPartition::new("t".to_string(), 0);
13353 {
13354 let mut subs = consumer.subscriptions.lock().unwrap();
13355 let mut assigned: HashSet<TopicPartition> = HashSet::new();
13356 assigned.insert(tp);
13357 subs.assign_from_user(assigned).unwrap();
13358 }
13359 // Post TWO errors. Java's loop returns on the first.
13360 handles
13361 .bg_event_tx
13362 .send(BackgroundEventEnvelope {
13363 event: BackgroundEvent::Error {
13364 error: Error::local_illegal_state("Nobody expects the Spanish Inquisition"),
13365 },
13366 enqueued_ms: 0,
13367 })
13368 .expect("send ok");
13369 handles
13370 .bg_event_tx
13371 .send(BackgroundEventEnvelope {
13372 event: BackgroundEvent::Error { error: Error::local_illegal_state("Spam, Spam, Spam") },
13373 enqueued_ms: 0,
13374 })
13375 .expect("send ok");
13376
13377 let err = consumer.poll(Duration::from_millis(0)).await.expect_err("first error surfaces");
13378 let msg = format!("{err}");
13379 // Java's assertion: the FIRST error message is observed.
13380 assert!(msg.contains("Spanish Inquisition"), "got: {msg}");
13381 }
13382
13383 /// Java: `testCommitSyncAwaitsCommitAsyncButDoesNotFail`
13384 /// An async-commit that fails does NOT
13385 /// propagate to the subsequent sync commit (the callback consumes
13386 /// the error).
13387 #[tokio::test]
13388 async fn commit_sync_does_not_fail_when_pending_async_failed() {
13389 let (mut consumer, mut handles) = make_test_consumer_with_channels();
13390 // Drainer: complete the async commit EXCEPTIONALLY, then the
13391 // sync commit normally.
13392 let completer = tokio::spawn(async move {
13393 let mut saw_sync = false;
13394 while let Some(env) = handles.app_event_rx.recv().await {
13395 match env.event {
13396 ApplicationEvent::CommitAsync { handle, offsets_ready, .. } => {
13397 offsets_ready.complete(());
13398 handle.complete_with_error(Error::local_illegal_state("Test error"));
13399 },
13400 ApplicationEvent::CommitSync { handle, offsets_ready, .. } => {
13401 offsets_ready.complete(());
13402 handle.complete(HashMap::new());
13403 saw_sync = true;
13404 return saw_sync;
13405 },
13406 _ => {},
13407 }
13408 }
13409 saw_sync
13410 });
13411
13412 // Async fail
13413 consumer.commit_async().await.expect("async ok");
13414 // Sync must NOT raise (Java's `assertDoesNotThrow`)
13415 consumer.commit_sync().await.expect("sync must not fail");
13416 assert!(completer.await.expect("task ok"));
13417 }
13418
13419 /// Java: `testCommitAsyncUserSuppliedCallbackWithException` (a
13420 /// `@ParameterizedTest`) — the parameter is the exception type, supplied by
13421 /// `commitExceptionSupplier` as `new KafkaException(..)` and
13422 /// `new GroupAuthorizationException(..)`. The Rust analog is two test
13423 /// methods, one per variant.
13424 ///
13425 /// This one must be a *bare* `KafkaException` — [`Error::kafka`], the
13426 /// [`Error::KafkaError`] variant — not a `LocalIllegalState`, which is
13427 /// outside the `KafkaException` hierarchy entirely. Java's assertion is
13428 /// `assertSame(exception.getClass(), callback.exception.getClass())`: the
13429 /// point of the case is that a `KafkaException` reaches the user callback
13430 /// *unwrapped*, which a class that `is_kafka_error()` rejects cannot
13431 /// exercise (the sibling below was corrected for the same reason).
13432 /// The message drops Java's "exception" wording per CLAUDE.md §2.
13433 #[tokio::test]
13434 async fn commit_async_user_supplied_callback_with_error_kafka() {
13435 commit_async_callback_with_error(Error::kafka_message("Test error")).await;
13436 }
13437
13438 #[tokio::test]
13439 async fn commit_async_user_supplied_callback_with_error_group_authz() {
13440 // Issue 23: must use `Error::GroupAuthorization`, not a string-shaped
13441 // `LocalIllegalArgument`. Java's `@ParameterizedTest` second parameter is
13442 // `GroupAuthorizationException`
13443 // (`org.apache.kafka.clients.consumer.internals.AsyncKafkaConsumerTest#commitExceptionSupplier`).
13444 commit_async_callback_with_error(Error::group_authorization("test-group")).await;
13445 }
13446
13447 async fn commit_async_callback_with_error(injected: Error) {
13448 use std::sync::atomic::AtomicUsize;
13449 struct RecordingCallback {
13450 saw_error: Arc<std::sync::Mutex<Option<String>>>,
13451 invoked: Arc<AtomicUsize>,
13452 }
13453 #[async_trait::async_trait]
13454 impl crate::consumer::OffsetCommitCallback for RecordingCallback {
13455 async fn on_complete(
13456 &self,
13457 _offsets: &HashMap<TopicPartition, OffsetAndMetadata>,
13458 error: Option<&Error>,
13459 ) -> Result<(), Error> {
13460 if let Some(e) = error {
13461 *self.saw_error.lock().unwrap() = Some(format!("{e}"));
13462 }
13463 self.invoked.fetch_add(1, Ordering::SeqCst);
13464 Ok(())
13465 }
13466 }
13467
13468 let (mut consumer, mut handles) = make_test_consumer_with_channels();
13469 let tp = TopicPartition::new("my-topic".to_string(), 1);
13470 let offsets = singleton_offsets(tp, 200);
13471
13472 let injected_clone = injected.clone();
13473 let completer = tokio::spawn(async move {
13474 while let Some(env) = handles.app_event_rx.recv().await {
13475 if let ApplicationEvent::CommitAsync { handle, offsets_ready, .. } = env.event {
13476 offsets_ready.complete(());
13477 handle.complete_with_error(injected_clone.clone());
13478 return true;
13479 }
13480 }
13481 false
13482 });
13483
13484 let saw_error = Arc::new(std::sync::Mutex::new(None::<String>));
13485 let invoked = Arc::new(AtomicUsize::new(0));
13486 let cb: Arc<RecordingCallback> =
13487 Arc::new(RecordingCallback { saw_error: Arc::clone(&saw_error), invoked: Arc::clone(&invoked) });
13488 consumer.commit_async_with_offsets_callback(offsets, cb).await.expect("ok");
13489
13490 // Wait for the spawned continuation to enqueue the callback.
13491 if let Some(mut rx) = consumer.commit_state.last_pending_async_commit() {
13492 rx.wait().await;
13493 }
13494
13495 // Drain the callback so it fires (Java's `forceCommitCallbackInvocation`).
13496 consumer
13497 .offset_commit_callback_invoker
13498 .invoke_pending_callbacks()
13499 .await
13500 .unwrap();
13501
13502 assert!(completer.await.expect("task ok"));
13503 assert_eq!(invoked.load(Ordering::SeqCst), 1, "callback must be invoked once");
13504 let saw_msg = saw_error.lock().unwrap().clone().expect("callback observed error");
13505 let injected_msg = format!("{injected}");
13506 assert!(saw_msg == injected_msg, "callback error mismatch: {saw_msg} != {injected_msg}");
13507 }
13508
13509 /// Verifies ordering when two async commits complete in reverse order.
13510 /// The last pending commit must not resolve until the earlier commit's
13511 /// callback has been enqueued, and callbacks must be invoked in commit
13512 /// order. `close()` and `commit_sync()` depend on this invariant because
13513 /// they await only the most recent pending async commit (Java's
13514 /// `lastPendingAsyncCommit`).
13515 #[tokio::test(flavor = "multi_thread")]
13516 async fn commit_async_last_pending_waits_for_earlier_callbacks() {
13517 struct OrderCallback {
13518 order: Arc<std::sync::Mutex<Vec<i64>>>,
13519 }
13520 #[async_trait::async_trait]
13521 impl crate::consumer::OffsetCommitCallback for OrderCallback {
13522 async fn on_complete(
13523 &self,
13524 offsets: &HashMap<TopicPartition, OffsetAndMetadata>,
13525 error: Option<&Error>,
13526 ) -> Result<(), Error> {
13527 assert!(error.is_none(), "unexpected commit error: {error:?}");
13528 let offset = offsets.values().next().expect("committed offsets").offset();
13529 self.order.lock().unwrap().push(offset);
13530 Ok(())
13531 }
13532 }
13533
13534 let (mut consumer, mut handles) = make_test_consumer_with_channels();
13535 let tp = TopicPartition::new("my-topic".to_string(), 0);
13536 let (release_first_tx, release_first_rx) = tokio::sync::oneshot::channel::<()>();
13537
13538 // Complete the second commit immediately and defer the first until the
13539 // test releases it, reversing the submission order.
13540 let completer = tokio::spawn(async move {
13541 let mut first = None;
13542 while let Some(env) = handles.app_event_rx.recv().await {
13543 if let ApplicationEvent::CommitAsync { handle, offsets_ready, offsets, .. } = env.event {
13544 offsets_ready.complete(());
13545 let offsets = offsets.expect("explicit offsets");
13546 match first.take() {
13547 None => first = Some((handle, offsets)),
13548 Some((first_handle, first_offsets)) => {
13549 handle.complete(offsets);
13550 let _ = release_first_rx.await;
13551 first_handle.complete(first_offsets);
13552 return true;
13553 },
13554 }
13555 }
13556 }
13557 false
13558 });
13559
13560 let order = Arc::new(std::sync::Mutex::new(Vec::new()));
13561 let cb: Arc<dyn crate::consumer::OffsetCommitCallback> = Arc::new(OrderCallback { order: Arc::clone(&order) });
13562 consumer
13563 .commit_async_with_offsets_callback(singleton_offsets(tp.clone(), 1), Arc::clone(&cb))
13564 .await
13565 .expect("commit 1");
13566 consumer
13567 .commit_async_with_offsets_callback(singleton_offsets(tp, 2), cb)
13568 .await
13569 .expect("commit 2");
13570
13571 let mut last = consumer.commit_state.last_pending_async_commit().expect("pending async commit");
13572 assert!(
13573 tokio::time::timeout(Duration::from_millis(200), last.wait()).await.is_err(),
13574 "last pending commit must not resolve while an earlier commit is outstanding"
13575 );
13576
13577 release_first_tx.send(()).expect("completer alive");
13578 last.wait().await;
13579 consumer
13580 .offset_commit_callback_invoker
13581 .invoke_pending_callbacks()
13582 .await
13583 .unwrap();
13584
13585 assert!(completer.await.expect("task ok"));
13586 assert_eq!(*order.lock().unwrap(), vec![1, 2], "callbacks must fire in commit order");
13587 }
13588
13589 /// An empty-offsets commit skips ahead of an in-flight commit (as in Java),
13590 /// but a non-empty commit issued after it must still chain behind that
13591 /// in-flight commit. With commits 1, empty, 3 and commit 3 completing before
13592 /// commit 1, the last pending commit must not resolve while commit 1 is
13593 /// outstanding, and callbacks must be invoked as empty, 1, 3.
13594 #[tokio::test(flavor = "multi_thread")]
13595 async fn commit_async_after_empty_offsets_commit_still_waits_for_earlier_commit() {
13596 struct LabelCallback {
13597 order: Arc<std::sync::Mutex<Vec<String>>>,
13598 }
13599 #[async_trait::async_trait]
13600 impl crate::consumer::OffsetCommitCallback for LabelCallback {
13601 async fn on_complete(
13602 &self,
13603 offsets: &HashMap<TopicPartition, OffsetAndMetadata>,
13604 error: Option<&Error>,
13605 ) -> Result<(), Error> {
13606 assert!(error.is_none(), "unexpected commit error: {error:?}");
13607 let label = offsets
13608 .values()
13609 .next()
13610 .map_or_else(|| "empty".to_string(), |o| o.offset().to_string());
13611 self.order.lock().unwrap().push(label);
13612 Ok(())
13613 }
13614 }
13615
13616 let (mut consumer, mut handles) = make_test_consumer_with_channels();
13617 let tp = TopicPartition::new("my-topic".to_string(), 0);
13618 let (release_first_tx, release_first_rx) = tokio::sync::oneshot::channel::<()>();
13619
13620 // The empty commit enqueues no event, so commit 3 is the second
13621 // CommitAsync event. Complete it immediately and defer commit 1 until
13622 // the test releases it, reversing the submission order.
13623 let completer = tokio::spawn(async move {
13624 let mut first = None;
13625 while let Some(env) = handles.app_event_rx.recv().await {
13626 if let ApplicationEvent::CommitAsync { handle, offsets_ready, offsets, .. } = env.event {
13627 offsets_ready.complete(());
13628 let offsets = offsets.expect("explicit offsets");
13629 match first.take() {
13630 None => first = Some((handle, offsets)),
13631 Some((first_handle, first_offsets)) => {
13632 handle.complete(offsets);
13633 let _ = release_first_rx.await;
13634 first_handle.complete(first_offsets);
13635 return true;
13636 },
13637 }
13638 }
13639 }
13640 false
13641 });
13642
13643 let order = Arc::new(std::sync::Mutex::new(Vec::new()));
13644 let cb: Arc<dyn crate::consumer::OffsetCommitCallback> = Arc::new(LabelCallback { order: Arc::clone(&order) });
13645 consumer
13646 .commit_async_with_offsets_callback(singleton_offsets(tp.clone(), 1), Arc::clone(&cb))
13647 .await
13648 .expect("commit 1");
13649 consumer
13650 .commit_async_with_offsets_callback(HashMap::new(), Arc::clone(&cb))
13651 .await
13652 .expect("empty commit");
13653 consumer
13654 .commit_async_with_offsets_callback(singleton_offsets(tp, 3), cb)
13655 .await
13656 .expect("commit 3");
13657
13658 let mut last = consumer.commit_state.last_pending_async_commit().expect("pending async commit");
13659 assert!(
13660 tokio::time::timeout(Duration::from_millis(200), last.wait()).await.is_err(),
13661 "last pending commit must not resolve while an earlier commit is outstanding"
13662 );
13663
13664 release_first_tx.send(()).expect("completer alive");
13665 last.wait().await;
13666 consumer
13667 .offset_commit_callback_invoker
13668 .invoke_pending_callbacks()
13669 .await
13670 .unwrap();
13671
13672 assert!(completer.await.expect("task ok"));
13673 assert_eq!(
13674 *order.lock().unwrap(),
13675 vec!["empty", "1", "3"],
13676 "only the empty commit may skip ahead; the others fire in commit order"
13677 );
13678 }
13679
13680 /// Consumer with an in-flight async commit, built by
13681 /// [`set_up_consumer_with_incomplete_async_commit`].
13682 struct IncompleteAsyncCommit {
13683 consumer: AsyncKafkaConsumer<Vec<u8>, Vec<u8>>,
13684 /// Handle of the pending `CommitAsync` event (Java's
13685 /// `asyncCommitFuture`).
13686 async_commit:
13687 crate::consumer::internals::events::CompletableEventHandle<HashMap<TopicPartition, OffsetAndMetadata>>,
13688 /// Keeps the background-event channel open for the test's duration.
13689 _bg_event_tx: mpsc::UnboundedSender<BackgroundEventEnvelope>,
13690 }
13691
13692 /// Java: `setUpConsumerWithIncompleteAsyncCommit`.
13693 /// Assigns `tp`, seeks it to 20 and issues a `commit_async` whose
13694 /// offsets are marked ready but whose commit is left incomplete. The
13695 /// spawned task stands in for the background task: it completes
13696 /// assignment / seek events and every `CommitSync` successfully (Java's
13697 /// `completeCommitSyncApplicationEventSuccessfully`). The translated Java
13698 /// tests pass `callback = None`; the Rust-specific test passes one to
13699 /// observe callback order.
13700 async fn set_up_consumer_with_incomplete_async_commit(
13701 tp: &TopicPartition,
13702 callback: Option<Arc<dyn crate::consumer::OffsetCommitCallback>>,
13703 ) -> IncompleteAsyncCommit {
13704 set_up_consumer_with_incomplete_async_commit_via(tp, callback, false).await
13705 }
13706
13707 /// [`set_up_consumer_with_incomplete_async_commit`], issuing the
13708 /// `commit_async` through a [`ConsumerHandle`] when `via_handle`, as a
13709 /// Java callback calling back into the consumer does.
13710 async fn set_up_consumer_with_incomplete_async_commit_via(
13711 tp: &TopicPartition,
13712 callback: Option<Arc<dyn crate::consumer::OffsetCommitCallback>>,
13713 via_handle: bool,
13714 ) -> IncompleteAsyncCommit {
13715 let (mut consumer, handles) = make_test_consumer_with_channels();
13716 let ConsumerTestHandles { mut app_event_rx, bg_event_tx, .. } = handles;
13717 let (async_commit_tx, async_commit_rx) = tokio::sync::oneshot::channel();
13718 tokio::spawn(async move {
13719 let mut async_commit_tx = Some(async_commit_tx);
13720 while let Some(env) = app_event_rx.recv().await {
13721 match env.event {
13722 ApplicationEvent::AssignmentChange { handle, .. }
13723 | ApplicationEvent::SeekUnvalidated { handle, .. } => {
13724 handle.complete(());
13725 },
13726 ApplicationEvent::CommitAsync { handle, offsets_ready, .. } => {
13727 // Java: `markOffsetsReadyForCommitEvent()`.
13728 offsets_ready.complete(());
13729 if let Some(tx) = async_commit_tx.take() {
13730 let _ = tx.send(handle);
13731 }
13732 },
13733 ApplicationEvent::CommitSync { handle, offsets_ready, offsets } => {
13734 offsets_ready.complete(());
13735 handle.complete(offsets.unwrap_or_default());
13736 },
13737 _ => {},
13738 }
13739 }
13740 });
13741
13742 consumer.assign(vec![tp.clone()]).await.expect("assign");
13743 consumer.seek_with_offset(tp.clone(), 20).await.expect("seek");
13744 if via_handle {
13745 let handle = consumer.handle();
13746 let _in_callback = consumer.enter_callback_scope();
13747 match callback {
13748 Some(cb) => handle.commit_async_with_callback(cb).await.expect("commit_async"),
13749 None => handle.commit_async().await.expect("commit_async"),
13750 }
13751 } else {
13752 match callback {
13753 Some(cb) => consumer.commit_async_with_callback(cb).await.expect("commit_async"),
13754 None => consumer.commit_async().await.expect("commit_async"),
13755 }
13756 }
13757 let async_commit = async_commit_rx.await.expect("CommitAsync event enqueued");
13758 IncompleteAsyncCommit { consumer, async_commit, _bg_event_tx: bg_event_tx }
13759 }
13760
13761 fn assert_pending_async_commit_timeout(err: &Error) {
13762 assert!(matches!(err, Error::Timeout(_)), "expected Timeout, got {err:?}");
13763 assert_eq!(err.message(), "Timed out waiting for last pending async commit to complete");
13764 }
13765
13766 /// Java's `getResult(future, timer)` returns a completed future's result
13767 /// with an expired timer (`CompletableFuture.get(0, ...)`): a completed
13768 /// last pending async commit does not time out a zero-timeout wait, while
13769 /// an incomplete one still does.
13770 #[tokio::test]
13771 async fn await_pending_async_commit_with_an_expired_timer_returns_when_it_completed() {
13772 let tp = TopicPartition::new("foo".to_string(), 0);
13773 let IncompleteAsyncCommit { mut consumer, async_commit, _bg_event_tx } =
13774 set_up_consumer_with_incomplete_async_commit(&tp, None).await;
13775 let now = consumer.time.milliseconds();
13776 let err = consumer
13777 .await_pending_async_commits_and_execute_commit_callbacks(now, false)
13778 .await
13779 .expect_err("an incomplete commit times out");
13780 assert_pending_async_commit_timeout(&err);
13781
13782 async_commit.complete(HashMap::new());
13783 wait_last_pending_async_commit(&consumer).await;
13784 let now = consumer.time.milliseconds();
13785 consumer
13786 .await_pending_async_commits_and_execute_commit_callbacks(now, false)
13787 .await
13788 .expect("a completed commit returns with an expired timer");
13789 assert!(!consumer.commit_state.has_last_pending_async_commit(), "and is cleared");
13790 }
13791
13792 /// Java: `testCommitSyncAwaitsCommitAsyncCompletionWithEmptyOffsets`
13793 #[tokio::test]
13794 async fn commit_sync_awaits_commit_async_completion_with_empty_offsets() {
13795 let tp = TopicPartition::new("foo".to_string(), 0);
13796 let IncompleteAsyncCommit { mut consumer, async_commit, _bg_event_tx } =
13797 set_up_consumer_with_incomplete_async_commit(&tp, None).await;
13798
13799 // Commit async is not completed yet, so commit sync should wait for it
13800 // to complete (time out).
13801 let err = consumer
13802 .commit_sync_with_offsets_timeout(HashMap::new(), Duration::from_millis(100))
13803 .await
13804 .expect_err("commit sync must time out");
13805 assert_pending_async_commit_timeout(&err);
13806
13807 // Complete the async commit with an error.
13808 async_commit.complete_with_error(Error::kafka_message("Test error"));
13809
13810 // Commit async is completed, so commit sync completes immediately
13811 // (since offsets are empty).
13812 consumer
13813 .commit_sync_with_offsets_timeout(HashMap::new(), Duration::from_millis(100))
13814 .await
13815 .expect("commit sync must succeed");
13816 }
13817
13818 /// Java: `testCommitSyncAwaitsCommitAsyncCompletionWithNonEmptyOffsets`
13819 #[tokio::test]
13820 async fn commit_sync_awaits_commit_async_completion_with_non_empty_offsets() {
13821 let tp = TopicPartition::new("foo".to_string(), 0);
13822 let IncompleteAsyncCommit { mut consumer, async_commit, _bg_event_tx } =
13823 set_up_consumer_with_incomplete_async_commit(&tp, None).await;
13824
13825 // Commit async is not completed yet, so commit sync should wait for it
13826 // to complete (time out).
13827 let err = consumer
13828 .commit_sync_with_offsets_timeout(singleton_offsets(tp.clone(), 20), Duration::from_millis(100))
13829 .await
13830 .expect_err("commit sync must time out");
13831 assert_pending_async_commit_timeout(&err);
13832
13833 // Complete the async commit.
13834 async_commit.complete(HashMap::new());
13835
13836 // Commit async is completed, so commit sync does not need to wait
13837 // before committing its offsets.
13838 consumer
13839 .commit_sync_with_offsets_timeout(singleton_offsets(tp, 20), Duration::from_millis(100))
13840 .await
13841 .expect("commit sync must succeed");
13842 }
13843
13844 /// Rust-specific companion to the two tests above, covering an empty-offsets
13845 /// `commit_async` issued while an earlier async commit is in flight. As in
13846 /// Java (`AsyncKafkaConsumer#commitAsync(Optional,OffsetCommitCallback)`, `AsyncKafkaConsumer#commit`), the empty
13847 /// commit completes immediately and replaces `lastPendingAsyncCommit`, so it is
13848 /// not chained behind the earlier commit: its callback is enqueued before
13849 /// `commit_async` returns, the next `commit_sync` does not wait for the
13850 /// earlier commit, and the empty commit's callback runs first.
13851 #[tokio::test]
13852 async fn commit_sync_does_not_await_commit_async_before_later_empty_offsets_commit_async() {
13853 struct LabelCallback {
13854 label: &'static str,
13855 order: Arc<std::sync::Mutex<Vec<&'static str>>>,
13856 }
13857 #[async_trait::async_trait]
13858 impl crate::consumer::OffsetCommitCallback for LabelCallback {
13859 async fn on_complete(
13860 &self,
13861 _offsets: &HashMap<TopicPartition, OffsetAndMetadata>,
13862 _error: Option<&Error>,
13863 ) -> Result<(), Error> {
13864 self.order.lock().unwrap().push(self.label);
13865 Ok(())
13866 }
13867 }
13868
13869 let tp = TopicPartition::new("foo".to_string(), 0);
13870 let order = Arc::new(std::sync::Mutex::new(Vec::new()));
13871 let first = Arc::new(LabelCallback { label: "first", order: Arc::clone(&order) });
13872 let IncompleteAsyncCommit { mut consumer, async_commit, _bg_event_tx } =
13873 set_up_consumer_with_incomplete_async_commit(&tp, Some(first)).await;
13874
13875 let second = Arc::new(LabelCallback { label: "second", order: Arc::clone(&order) });
13876 consumer
13877 .commit_async_with_offsets_callback(HashMap::new(), second)
13878 .await
13879 .expect("empty commit_async");
13880
13881 // As in Java, the empty commit's callback is enqueued before
13882 // `commit_async` returns, so it runs on the next callback invocation.
13883 consumer
13884 .offset_commit_callback_invoker
13885 .invoke_pending_callbacks()
13886 .await
13887 .unwrap();
13888 assert_eq!(
13889 *order.lock().unwrap(),
13890 vec!["second"],
13891 "the empty commit's callback must be enqueued before commit_async returns"
13892 );
13893
13894 // The last pending async commit is the already-completed empty one, so
13895 // commit sync does not wait for the earlier, still incomplete commit.
13896 consumer
13897 .commit_sync_with_timeout(Duration::from_millis(100))
13898 .await
13899 .expect("commit sync must not wait for the earlier async commit");
13900 assert_eq!(
13901 *order.lock().unwrap(),
13902 vec!["second"],
13903 "the earlier commit's callback must not run before it completes"
13904 );
13905
13906 // The earlier commit still enqueues its callback once it completes.
13907 async_commit.complete(singleton_offsets(tp, 20));
13908 tokio::time::timeout(Duration::from_secs(5), async {
13909 while order.lock().unwrap().len() < 2 {
13910 tokio::task::yield_now().await;
13911 consumer
13912 .offset_commit_callback_invoker
13913 .invoke_pending_callbacks()
13914 .await
13915 .unwrap();
13916 }
13917 })
13918 .await
13919 .expect("the earlier commit's callback must run after it completes");
13920 assert_eq!(*order.lock().unwrap(), vec!["second", "first"]);
13921 }
13922
13923 // ─── Close / lifecycle tests (commit 7/N) ───
13924 //
13925 // Stand-ins for Java's `testCloseShouldBeIdempotent`,
13926 // `testWakeupShouldThrowAfterClose`, `testLeaveGroupOnClose`,
13927 // `testRunRebalanceCallbacksOnClose`. The fully end-to-end versions
13928 // (`MockClient`-backed bg task observing the actual close events)
13929 // land in commit (10/N) — these unit-test the close-path control
13930 // flow.
13931 //
13932 // Skipped Java tests:
13933 // - `testCloseInvokesStreamsRebalanceListenerOn*` — Streams out
13934 // of scope per consumer-threading.md §20.
13935 // - `testCloseWrapsStreamsRebalanceListenerException` — Streams.
13936
13937 /// `close` is idempotent: calling close twice does NOT panic or
13938 /// surface an error. Mirrors Java's `testCloseShouldBeIdempotent`.
13939 #[tokio::test]
13940 async fn close_is_idempotent() {
13941 let (mut consumer, mut handles) = make_test_consumer_with_channels();
13942 // Drain LeaveGroupOnClose events on a background task, completing
13943 // each handle with Ok(()) so close_internal does not block.
13944 // Drainer that handles every close-path event (LeaveGroupOnClose,
13945 // CommitSync (auto-commit), CommitOnClose, StopFindCoordinatorOnClose).
13946 let drainer = tokio::spawn(async move {
13947 while let Some(env) = handles.app_event_rx.recv().await {
13948 match env.event {
13949 ApplicationEvent::LeaveGroupOnClose { handle, .. } => {
13950 handle.complete(());
13951 },
13952 ApplicationEvent::CommitSync { handle, offsets_ready, .. } => {
13953 offsets_ready.complete(());
13954 handle.complete(HashMap::new());
13955 },
13956 ApplicationEvent::CommitAsync { handle, offsets_ready, .. } => {
13957 offsets_ready.complete(());
13958 handle.complete(HashMap::new());
13959 },
13960 _ => {},
13961 }
13962 }
13963 });
13964 consumer.close().await.expect("first close ok");
13965 assert!(consumer.is_closed(), "close marks consumer closed");
13966 // Second close: no-op, no error.
13967 consumer.close().await.expect("idempotent close ok");
13968 drop(drainer);
13969 }
13970
13971 /// After `close()`, the wakeup trigger is disabled so subsequent
13972 /// `wakeup()` calls are no-ops (Java
13973 /// `wakeupTrigger.disableWakeups()`).
13974 #[tokio::test]
13975 async fn close_disables_wakeups() {
13976 let (mut consumer, mut handles) = make_test_consumer_with_channels();
13977 // Drainer that handles every close-path event (LeaveGroupOnClose,
13978 // CommitSync (auto-commit), CommitOnClose, StopFindCoordinatorOnClose).
13979 let drainer = tokio::spawn(async move {
13980 while let Some(env) = handles.app_event_rx.recv().await {
13981 match env.event {
13982 ApplicationEvent::LeaveGroupOnClose { handle, .. } => {
13983 handle.complete(());
13984 },
13985 ApplicationEvent::CommitSync { handle, offsets_ready, .. } => {
13986 offsets_ready.complete(());
13987 handle.complete(HashMap::new());
13988 },
13989 ApplicationEvent::CommitAsync { handle, offsets_ready, .. } => {
13990 offsets_ready.complete(());
13991 handle.complete(HashMap::new());
13992 },
13993 _ => {},
13994 }
13995 }
13996 });
13997 consumer.close().await.expect("ok");
13998 // wakeup() should be a no-op now — the wakeup trigger is disabled.
13999 // Verify by calling maybe_trigger_wakeup; it should return Ok.
14000 consumer.wakeup_trigger.wakeup();
14001 assert!(consumer.wakeup_trigger.maybe_trigger_wakeup().is_ok());
14002 drop(drainer);
14003 }
14004
14005 /// `close_with_options(timeout=0)` short-cuts the deadline math but
14006 /// still completes successfully.
14007 #[tokio::test]
14008 async fn close_with_options_zero_timeout_completes() {
14009 use crate::consumer::CloseOptions;
14010 let (mut consumer, mut handles) = make_test_consumer_with_channels();
14011 // Drainer that handles every close-path event (LeaveGroupOnClose,
14012 // CommitSync (auto-commit), CommitOnClose, StopFindCoordinatorOnClose).
14013 let drainer = tokio::spawn(async move {
14014 while let Some(env) = handles.app_event_rx.recv().await {
14015 match env.event {
14016 ApplicationEvent::LeaveGroupOnClose { handle, .. } => {
14017 handle.complete(());
14018 },
14019 ApplicationEvent::CommitSync { handle, offsets_ready, .. } => {
14020 offsets_ready.complete(());
14021 handle.complete(HashMap::new());
14022 },
14023 ApplicationEvent::CommitAsync { handle, offsets_ready, .. } => {
14024 offsets_ready.complete(());
14025 handle.complete(HashMap::new());
14026 },
14027 _ => {},
14028 }
14029 }
14030 });
14031 consumer
14032 .close_with_options(CloseOptions::new_with_timeout(Duration::from_millis(0)))
14033 .await
14034 .expect("ok");
14035 assert!(consumer.is_closed());
14036 drop(drainer);
14037 }
14038
14039 /// `close(CloseOptions.timeout(t))` must bound the bg task's `cleanup()`
14040 /// by what is left of the close timer: Java hands
14041 /// `Duration.ofMillis(closeTimer.remainingMs())` to
14042 /// `applicationEventHandler.close(..)` (`AsyncKafkaConsumer#close(Duration,GroupMembershipOperation,boolean)`),
14043 /// which assigns it to `ConsumerNetworkThread.closeTimeout`
14044 /// (`ConsumerNetworkThread#closeInternal`). Before the fix the Rust close path
14045 /// signalled shutdown without a timeout, so `cleanup()` kept its 30 s
14046 /// default and a 100 ms close waited out any in-flight fetch.
14047 #[tokio::test]
14048 async fn close_with_options_bounds_network_thread_cleanup_by_remaining_close_timer() {
14049 use crate::consumer::CloseOptions;
14050
14051 let (mut consumer, mut handles) = make_test_consumer_with_channels();
14052 let drainer = tokio::spawn(async move {
14053 while let Some(env) = handles.app_event_rx.recv().await {
14054 match env.event {
14055 ApplicationEvent::LeaveGroupOnClose { handle, .. } => {
14056 handle.complete(());
14057 },
14058 ApplicationEvent::CommitSync { handle, offsets_ready, .. } => {
14059 offsets_ready.complete(());
14060 handle.complete(HashMap::new());
14061 },
14062 _ => {},
14063 }
14064 }
14065 });
14066 consumer
14067 .close_with_options(CloseOptions::new_with_timeout(Duration::from_millis(100)))
14068 .await
14069 .expect("close ok");
14070 drainer.abort();
14071
14072 let signalled = handles.signal_close_timeout_ms.load(Ordering::Acquire);
14073 assert!(
14074 (0..=100).contains(&signalled),
14075 "close must hand the bg task the remaining close timer (<= 100 ms), got {signalled} ms"
14076 );
14077 }
14078
14079 /// `close_with_options(CloseOptions::new_with_timeout(..))` is the replacement
14080 /// for Java's deprecated `close(Duration timeout)`, whose body is exactly
14081 /// `close(CloseOptions.timeout(timeout))`
14082 /// (`AsyncKafkaConsumer#close(Duration)`) and which is not translated
14083 /// (CLAUDE.md §3). The user timeout must reach the close path.
14084 ///
14085 /// Asserting `is_closed()` alone would not catch a dropped timeout, so
14086 /// this checks the *deadline* carried on the `LeaveGroupOnClose` event —
14087 /// the only place the timeout is observable.
14088 #[tokio::test]
14089 async fn close_with_options_carries_user_timeout() {
14090 use crate::consumer::CloseOptions;
14091
14092 // Well under the 30s `request.timeout.ms` cap, so the deadline
14093 // reflects the user timeout rather than the cap.
14094 let user_timeout = Duration::from_secs(7);
14095
14096 async fn deadline_delta_for(
14097 close: impl AsyncFnOnce(&mut AsyncKafkaConsumer<Vec<u8>, Vec<u8>>) -> Result<(), Error>,
14098 ) -> i64 {
14099 let (mut consumer, mut handles) = make_test_consumer_with_channels();
14100 let captured = Arc::new(Mutex::new(None::<i64>));
14101 let captured_clone = Arc::clone(&captured);
14102 let drainer = tokio::spawn(async move {
14103 while let Some(env) = handles.app_event_rx.recv().await {
14104 match env.event {
14105 ApplicationEvent::LeaveGroupOnClose { handle, .. } => {
14106 *captured_clone.lock().unwrap() = Some(handle.deadline_ms());
14107 handle.complete(());
14108 },
14109 ApplicationEvent::CommitSync { handle, offsets_ready, .. } => {
14110 offsets_ready.complete(());
14111 handle.complete(HashMap::new());
14112 },
14113 ApplicationEvent::CommitAsync { handle, offsets_ready, .. } => {
14114 offsets_ready.complete(());
14115 handle.complete(HashMap::new());
14116 },
14117 _ => {},
14118 }
14119 }
14120 });
14121 let now_before = consumer.time.milliseconds();
14122 close(&mut consumer).await.expect("close ok");
14123 assert!(consumer.is_closed());
14124 drop(drainer);
14125 let deadline = captured.lock().unwrap().expect("LeaveGroupOnClose seen");
14126 deadline - now_before
14127 }
14128
14129 let via_options =
14130 deadline_delta_for(async |c| c.close_with_options(CloseOptions::new_with_timeout(user_timeout)).await)
14131 .await;
14132 assert!(
14133 (via_options - 7_000).abs() <= 100,
14134 "close_with_options must carry the user timeout (delta={via_options})"
14135 );
14136 }
14137
14138 /// Issue 15 regression: `close_internal` must cap the
14139 /// user-supplied timeout at `request.timeout.ms` per Java's
14140 /// `createTimerForCloseRequests(timeout)`
14141 /// (`AsyncKafkaConsumer#createTimerForCloseRequests`). We assert directly on
14142 /// the deadline carried by the close-path event — Java's
14143 /// `LeaveGroupOnCloseEvent(deadline)` is built with the capped
14144 /// timer; the Rust analog flows through
14145 /// `ApplicationEvent::LeaveGroupOnClose { handle, .. }` and the
14146 /// handle's deadline reflects the capped value.
14147 #[tokio::test]
14148 async fn close_caps_timeout_at_request_timeout_ms() {
14149 use crate::consumer::CloseOptions;
14150 let (mut consumer, mut handles) = make_test_consumer_with_channels();
14151
14152 // Default config: request_timeout_ms = 30s. A 5-minute user
14153 // timeout MUST be capped at 30s before computing the deadline.
14154 let request_timeout_ms = consumer.config.request_timeout_ms() as i64;
14155 assert_eq!(request_timeout_ms, 30_000, "default request_timeout_ms");
14156
14157 // Capture the deadline observed on the `LeaveGroupOnClose`
14158 // event handle — that's the post-cap value Java would build
14159 // via `calculateDeadlineMs(closeTimer)`.
14160 let captured_deadline = Arc::new(Mutex::new(None::<i64>));
14161 let captured_clone = Arc::clone(&captured_deadline);
14162 let drainer = tokio::spawn(async move {
14163 while let Some(env) = handles.app_event_rx.recv().await {
14164 match env.event {
14165 ApplicationEvent::LeaveGroupOnClose { handle, .. } => {
14166 *captured_clone.lock().unwrap() = Some(handle.deadline_ms());
14167 handle.complete(());
14168 },
14169 ApplicationEvent::CommitSync { handle, offsets_ready, .. } => {
14170 offsets_ready.complete(());
14171 handle.complete(HashMap::new());
14172 },
14173 ApplicationEvent::CommitAsync { handle, offsets_ready, .. } => {
14174 offsets_ready.complete(());
14175 handle.complete(HashMap::new());
14176 },
14177 _ => {},
14178 }
14179 }
14180 });
14181
14182 let now_before = consumer.time.milliseconds();
14183 consumer
14184 .close_with_options(CloseOptions::new_with_timeout(Duration::from_secs(300)))
14185 .await
14186 .expect("close ok");
14187 drop(drainer);
14188
14189 let captured_deadline_val = captured_deadline.lock().unwrap().expect("LeaveGroupOnClose seen");
14190 // The user passed 5min = 300_000ms; the cap reduces this to
14191 // 30_000ms. The observed deadline must be at most
14192 // now_before + 30_000ms (with a small fudge for clock drift
14193 // since `now_before` is captured before the cap logic runs).
14194 let user_uncapped = now_before + 300_000;
14195 let expected_capped = now_before + request_timeout_ms;
14196 assert!(
14197 captured_deadline_val < user_uncapped - 1_000,
14198 "deadline must NOT use the raw 5min user timeout \
14199 (captured={captured_deadline_val}, user_uncapped={user_uncapped})"
14200 );
14201 assert!(
14202 captured_deadline_val <= expected_capped + 100,
14203 "deadline must be at most now+request_timeout_ms \
14204 (captured={captured_deadline_val}, expected_capped={expected_capped})"
14205 );
14206 }
14207
14208 /// Issue 12 regression: `run_rebalance_callbacks_on_close` must
14209 /// read from `group_assignment_snapshot` (populated by the
14210 /// `MemberStateListener`) — NOT from
14211 /// `SubscriptionState::assigned_partitions()`. With only a manual
14212 /// `assign(...)` (so the snapshot stays empty), close must NOT
14213 /// invoke any rebalance callback (Java's `runRebalanceCallbacksOnClose`).
14214 #[tokio::test]
14215 async fn run_rebalance_callbacks_on_close_skips_when_snapshot_empty() {
14216 use async_trait::async_trait;
14217 use std::sync::atomic::AtomicUsize;
14218
14219 struct CountingListener {
14220 revoked: AtomicUsize,
14221 lost: AtomicUsize,
14222 }
14223 #[async_trait]
14224 impl ConsumerRebalanceListener for CountingListener {
14225 async fn on_partitions_assigned(&self, _: &[TopicPartition]) -> Result<(), Error> {
14226 Ok(())
14227 }
14228 async fn on_partitions_revoked(&self, _: &[TopicPartition]) -> Result<(), Error> {
14229 self.revoked.fetch_add(1, Ordering::SeqCst);
14230 Ok(())
14231 }
14232 async fn on_partitions_lost(&self, _: &[TopicPartition]) -> Result<(), Error> {
14233 self.lost.fetch_add(1, Ordering::SeqCst);
14234 Ok(())
14235 }
14236 }
14237
14238 let (mut consumer, _handles) = make_test_consumer_with_channels();
14239 let listener: Arc<CountingListener> =
14240 Arc::new(CountingListener { revoked: AtomicUsize::new(0), lost: AtomicUsize::new(0) });
14241 *consumer.rebalance_listener.lock().unwrap() =
14242 Some(Arc::clone(&listener) as Arc<dyn ConsumerRebalanceListener>);
14243
14244 // Populate SubscriptionState as if the user called assign(...).
14245 let tp = TopicPartition::new("t".to_string(), 0);
14246 {
14247 let mut subs = consumer.subscriptions.lock().unwrap();
14248 let mut assigned: HashSet<TopicPartition> = HashSet::new();
14249 assigned.insert(tp);
14250 subs.assign_from_user(assigned).unwrap();
14251 }
14252 // The snapshot stays empty — no reconciliation has run.
14253 assert!(consumer.group_assignment_snapshot.lock().unwrap().is_empty());
14254
14255 consumer.run_rebalance_callbacks_on_close().await.expect("ok");
14256 assert_eq!(
14257 listener.revoked.load(Ordering::SeqCst),
14258 0,
14259 "no listener call when snapshot is empty (manual-assign consumer)"
14260 );
14261 assert_eq!(
14262 listener.lost.load(Ordering::SeqCst),
14263 0,
14264 "no listener call when snapshot is empty"
14265 );
14266 }
14267
14268 /// Issue 12/13 regression: `run_rebalance_callbacks_on_close`
14269 /// invokes `on_partitions_revoked` when the snapshot is non-empty
14270 /// AND `member_epoch > 0` (populated via the state notifier).
14271 /// Without Issue 13's `MemberStateListener` wire-up the epoch would
14272 /// always be -1 and the callback would always be `on_partitions_lost`.
14273 #[tokio::test]
14274 async fn run_rebalance_callbacks_on_close_invokes_revoked_on_live_epoch() {
14275 use async_trait::async_trait;
14276 use std::sync::atomic::AtomicUsize;
14277
14278 struct CountingListener {
14279 revoked: AtomicUsize,
14280 lost: AtomicUsize,
14281 }
14282 #[async_trait]
14283 impl ConsumerRebalanceListener for CountingListener {
14284 async fn on_partitions_assigned(&self, _: &[TopicPartition]) -> Result<(), Error> {
14285 Ok(())
14286 }
14287 async fn on_partitions_revoked(&self, _: &[TopicPartition]) -> Result<(), Error> {
14288 self.revoked.fetch_add(1, Ordering::SeqCst);
14289 Ok(())
14290 }
14291 async fn on_partitions_lost(&self, _: &[TopicPartition]) -> Result<(), Error> {
14292 self.lost.fetch_add(1, Ordering::SeqCst);
14293 Ok(())
14294 }
14295 }
14296
14297 let (mut consumer, _handles) = make_test_consumer_with_channels();
14298 let listener: Arc<CountingListener> =
14299 Arc::new(CountingListener { revoked: AtomicUsize::new(0), lost: AtomicUsize::new(0) });
14300 *consumer.rebalance_listener.lock().unwrap() =
14301 Some(Arc::clone(&listener) as Arc<dyn ConsumerRebalanceListener>);
14302
14303 // The state notifier captures both the snapshot and the epoch
14304 // — mirrors what the bg-task reconciliation step would do.
14305 let notifier = consumer.state_notifier();
14306 let tp = TopicPartition::new("t".to_string(), 0);
14307 let mut snap = HashSet::new();
14308 snap.insert(tp.clone());
14309 notifier.on_group_assignment_updated(&snap);
14310 notifier.on_member_epoch_updated(Some(5), "member-1");
14311
14312 consumer.run_rebalance_callbacks_on_close().await.expect("ok");
14313 assert_eq!(
14314 listener.revoked.load(Ordering::SeqCst),
14315 1,
14316 "live epoch + non-empty snapshot must invoke on_partitions_revoked"
14317 );
14318 assert_eq!(listener.lost.load(Ordering::SeqCst), 0);
14319 }
14320
14321 /// Issue 12/13 regression: when the snapshot is non-empty but the
14322 /// epoch is unknown (member fenced / never received heartbeat
14323 /// response), Java falls through to `on_partitions_lost`.
14324 #[tokio::test]
14325 async fn run_rebalance_callbacks_on_close_invokes_lost_on_unknown_epoch() {
14326 use async_trait::async_trait;
14327 use std::sync::atomic::AtomicUsize;
14328
14329 struct CountingListener {
14330 revoked: AtomicUsize,
14331 lost: AtomicUsize,
14332 }
14333 #[async_trait]
14334 impl ConsumerRebalanceListener for CountingListener {
14335 async fn on_partitions_assigned(&self, _: &[TopicPartition]) -> Result<(), Error> {
14336 Ok(())
14337 }
14338 async fn on_partitions_revoked(&self, _: &[TopicPartition]) -> Result<(), Error> {
14339 self.revoked.fetch_add(1, Ordering::SeqCst);
14340 Ok(())
14341 }
14342 async fn on_partitions_lost(&self, _: &[TopicPartition]) -> Result<(), Error> {
14343 self.lost.fetch_add(1, Ordering::SeqCst);
14344 Ok(())
14345 }
14346 }
14347
14348 let (mut consumer, _handles) = make_test_consumer_with_channels();
14349 let listener: Arc<CountingListener> =
14350 Arc::new(CountingListener { revoked: AtomicUsize::new(0), lost: AtomicUsize::new(0) });
14351 *consumer.rebalance_listener.lock().unwrap() =
14352 Some(Arc::clone(&listener) as Arc<dyn ConsumerRebalanceListener>);
14353
14354 // Populate snapshot but do NOT update epoch — generation_id
14355 // stays at -1, so `on_partitions_lost` is invoked.
14356 let notifier = consumer.state_notifier();
14357 let tp = TopicPartition::new("t".to_string(), 0);
14358 let mut snap = HashSet::new();
14359 snap.insert(tp.clone());
14360 notifier.on_group_assignment_updated(&snap);
14361
14362 consumer.run_rebalance_callbacks_on_close().await.expect("ok");
14363 assert_eq!(listener.revoked.load(Ordering::SeqCst), 0);
14364 assert_eq!(
14365 listener.lost.load(Ordering::SeqCst),
14366 1,
14367 "unknown epoch + non-empty snapshot must invoke on_partitions_lost"
14368 );
14369 }
14370
14371 /// `close` on a groupless consumer skips the leave-group event
14372 /// entirely. Mirrors Java's `if (groupMetadata.get().isEmpty())
14373 /// return;` guard.
14374 #[tokio::test]
14375 async fn close_without_group_id_skips_leave_group() {
14376 let (mut consumer, mut handles) = make_test_consumer_with_channels();
14377 consumer.group_id = None;
14378 consumer.close().await.expect("ok");
14379 // No LeaveGroupOnClose event should have been enqueued.
14380 let mut saw_leave_group = false;
14381 while let Ok(env) = handles.app_event_rx.try_recv() {
14382 if matches!(env.event, ApplicationEvent::LeaveGroupOnClose { .. }) {
14383 saw_leave_group = true;
14384 }
14385 }
14386 assert!(!saw_leave_group, "groupless consumer must not enqueue LeaveGroupOnClose");
14387 }
14388
14389 /// After `close`, every async public API errors with
14390 /// `LocalIllegalState` because `ensure_open()` short-circuits. Mirrors
14391 /// Java's `testShouldThrowAfterClose` (every public method
14392 /// asserted to throw post-close). Issue 20 expanded: covers every
14393 /// blocking-style API on `AsyncKafkaConsumer`, not just two.
14394 #[tokio::test]
14395 async fn close_then_apis_error_with_already_closed() {
14396 use crate::consumer::CloseOptions;
14397 use crate::consumer::SubscriptionPattern;
14398
14399 let (mut consumer, mut handles) = make_test_consumer_with_channels();
14400 // Drainer that handles every close-path event (LeaveGroupOnClose,
14401 // CommitSync (auto-commit), CommitOnClose, StopFindCoordinatorOnClose).
14402 let drainer = tokio::spawn(async move {
14403 while let Some(env) = handles.app_event_rx.recv().await {
14404 match env.event {
14405 ApplicationEvent::LeaveGroupOnClose { handle, .. } => {
14406 handle.complete(());
14407 },
14408 ApplicationEvent::CommitSync { handle, offsets_ready, .. } => {
14409 offsets_ready.complete(());
14410 handle.complete(HashMap::new());
14411 },
14412 ApplicationEvent::CommitAsync { handle, offsets_ready, .. } => {
14413 offsets_ready.complete(());
14414 handle.complete(HashMap::new());
14415 },
14416 _ => {},
14417 }
14418 }
14419 });
14420 consumer.close().await.expect("ok");
14421
14422 let tp = TopicPartition::new("t".to_string(), 0);
14423 let tp_slice = std::slice::from_ref(&tp);
14424
14425 // Macro: assert a Result yields IllegalState (Java's
14426 // `IllegalStateException`).
14427 macro_rules! assert_closed {
14428 ($name:expr, $expr:expr) => {{
14429 let err = $expr.expect_err(concat!($name, ": must err"));
14430 assert!(matches!(err, Error::LocalIllegalState(_)), "{}: {err:?}", $name);
14431 }};
14432 }
14433
14434 // ── subscribe family ────────────────────────────────────────
14435 assert_closed!(
14436 "subscribe_with_topics",
14437 consumer.subscribe_with_topics(vec!["t".to_string()]).await
14438 );
14439 assert_closed!(
14440 "subscribe_with_pattern",
14441 consumer.subscribe_with_pattern(SubscriptionPattern::new("t.*")).await
14442 );
14443 assert_closed!("unsubscribe", consumer.unsubscribe().await);
14444 assert_closed!("assign", consumer.assign(vec![tp.clone()]).await);
14445
14446 // ── poll ────────────────────────────────────────────────────
14447 assert_closed!("poll", consumer.poll(Duration::from_millis(0)).await);
14448
14449 // ── commit family ───────────────────────────────────────────
14450 assert_closed!("commit_sync", consumer.commit_sync().await);
14451 assert_closed!(
14452 "commit_sync_with_timeout",
14453 consumer.commit_sync_with_timeout(Duration::from_millis(0)).await
14454 );
14455 assert_closed!(
14456 "commit_sync_with_offsets",
14457 consumer.commit_sync_with_offsets(HashMap::new()).await
14458 );
14459 assert_closed!(
14460 "commit_sync_with_offsets_timeout",
14461 consumer
14462 .commit_sync_with_offsets_timeout(HashMap::new(), Duration::from_millis(0))
14463 .await
14464 );
14465 assert_closed!("commit_async", consumer.commit_async().await);
14466 assert_closed!(
14467 "commit_async_with_callback",
14468 consumer.commit_async_with_callback(Arc::new(NoopCallback)).await
14469 );
14470 assert_closed!(
14471 "commit_async_with_offsets_callback",
14472 consumer
14473 .commit_async_with_offsets_callback(HashMap::new(), Arc::new(NoopCallback))
14474 .await
14475 );
14476
14477 // ── seek family ─────────────────────────────────────────────
14478 assert_closed!("seek_with_offset", consumer.seek_with_offset(tp.clone(), 0).await);
14479 assert_closed!(
14480 "seek_with_offset_and_metadata",
14481 consumer
14482 .seek_with_offset_and_metadata(tp.clone(), OffsetAndMetadata::new(0).expect("ok"))
14483 .await
14484 );
14485 assert_closed!("seek_to_beginning", consumer.seek_to_beginning(tp_slice).await);
14486 assert_closed!("seek_to_end", consumer.seek_to_end(tp_slice).await);
14487
14488 // ── position / committed / lag ──────────────────────────────
14489 assert_closed!("position", consumer.position(&tp).await);
14490 assert_closed!(
14491 "position_with_timeout",
14492 consumer.position_with_timeout(&tp, Duration::from_millis(0)).await
14493 );
14494 assert_closed!("committed", consumer.committed(tp_slice).await);
14495 assert_closed!(
14496 "committed_with_timeout",
14497 consumer.committed_with_timeout(tp_slice, Duration::from_millis(0)).await
14498 );
14499 assert_closed!("current_lag", consumer.current_lag(&tp).await);
14500
14501 // ── beginning / end / offsetsForTimes ───────────────────────
14502 assert_closed!("beginning_offsets", consumer.beginning_offsets(tp_slice).await);
14503 assert_closed!(
14504 "beginning_offsets_with_timeout",
14505 consumer
14506 .beginning_offsets_with_timeout(tp_slice, Duration::from_millis(0))
14507 .await
14508 );
14509 assert_closed!("end_offsets", consumer.end_offsets(tp_slice).await);
14510 assert_closed!(
14511 "end_offsets_with_timeout",
14512 consumer.end_offsets_with_timeout(tp_slice, Duration::from_millis(0)).await
14513 );
14514 assert_closed!("offsets_for_times", consumer.offsets_for_times(HashMap::new()).await);
14515 assert_closed!(
14516 "offsets_for_times_with_timeout",
14517 consumer
14518 .offsets_for_times_with_timeout(HashMap::new(), Duration::from_millis(0))
14519 .await
14520 );
14521
14522 // ── topic metadata ──────────────────────────────────────────
14523 assert_closed!("partitions_for", consumer.partitions_for("t").await);
14524 assert_closed!(
14525 "partitions_for_with_timeout",
14526 consumer.partitions_for_with_timeout("t", Duration::from_millis(0)).await
14527 );
14528 assert_closed!("list_topics", consumer.list_topics().await);
14529 assert_closed!(
14530 "list_topics_with_timeout",
14531 consumer.list_topics_with_timeout(Duration::from_millis(0)).await
14532 );
14533
14534 // ── pause / resume ──────────────────────────────────────────
14535 assert_closed!("pause", consumer.pause(tp_slice).await);
14536 assert_closed!("resume", consumer.resume(tp_slice).await);
14537
14538 // ── enforce_rebalance ───────────────────────────────────────
14539 // KIP-848 noop (Java's `AsyncKafkaConsumer.enforceRebalance`):
14540 // returns Ok always — neither pre- nor post-close. Documented in
14541 // method rustdoc above.
14542
14543 // ── close-with-options is idempotent (not blocked) ──────────
14544 // close / close_with_options ARE idempotent per Java contract
14545 // — they short-circuit on `is_closed()` and return Ok. Verify
14546 // this matches the assertion above by exercising both
14547 // variants.
14548 consumer.close().await.expect("idempotent close");
14549 consumer
14550 .close_with_options(CloseOptions::new_with_timeout(Duration::from_millis(0)))
14551 .await
14552 .expect("idempotent close_with_options");
14553
14554 drop(drainer);
14555 }
14556
14557 // ─── Phase 11 commit (10/N) Java test translations: close / metadata / lifecycle ───
14558 //
14559 // Final batch: close-path branches, metadata APIs, seek-to-end /
14560 // seek-to-beginning, list-offsets, partition metadata, processBackgroundEvents
14561 // timing.
14562 //
14563 // SKIPs (commit 10 batch):
14564 // - testFailConstructor — PLAN deferral #5 (Supplier-style ctor
14565 // failure paths don't translate; bad-config path observed via
14566 // `Error` returned from `KafkaConsumer::new`).
14567 // - testCloseInvokesStreamsRebalanceListener* /
14568 // testCloseWrapsStreamsRebalanceListenerException — PLAN
14569 // deferral #2 (Streams out of scope per §20).
14570 // - testInterceptorAutoCommitOnClose — deferred to Issue 17 fix
14571 // (commit 12 batch).
14572 // - testReaperInvokedInClose / testReaperInvokedInUnsubscribe /
14573 // testReaperInvokedInPoll — require Mockito-style spy on the
14574 // reaper. The reap calls are wired (close_internal:2675, etc);
14575 // `ConsumerNetworkThreadTest` (Phase 10 commit 8/N) exercises
14576 // the bg-task reap path with a real reaper.
14577 // - testSubscribePatternAgainstBrokerNotSupportingRegex — needs
14578 // MockClient or a wired bg task. Deferred to Phase 12.5
14579 // (response-routing) per `Phase-12/RESPONSE-ROUTING-AUDIT.md`.
14580 // - testGroupMetadataIsResetAfterUnsubscribe — see commit 8 skip
14581 // section.
14582 // - testLongPollWaitIsLimited — requires a full FetchCollector
14583 // wired into the bg task; observable only via integration tests.
14584 // - testNoWakeupInCloseCommit — covered by close-path inline
14585 // tests (the close drainer completes CommitSync envelopes
14586 // normally, demonstrating no wakeup interference).
14587 // - testCommitSyncAllConsumed / testAutoCommitSyncDisabled — require
14588 // a fully-wired SubscriptionState `commit_sync_all_consumed`
14589 // helper that is consumer-internal (Java: package-private). The
14590 // close-path inline tests demonstrate the SyncCommitEvent
14591 // enqueue/no-enqueue behaviour for the auto_commit_enabled flag.
14592 //
14593 // Issue 24 additions (commit 11/N batch):
14594 // - testCloseAwaitPendingAsyncCommitIncomplete — requires the
14595 // `lastPendingAsyncCommit` future to be held in an incomplete
14596 // state past the close-timeout. The Rust analog
14597 // (`AsyncCommitState`'s `PendingAsyncCommit`)
14598 // is exercised by `close_awaits_pending_async_commit_complete`
14599 // for the happy path; the timeout-cause-of-incomplete path
14600 // requires injecting a never-completing handle that is
14601 // specifically held by the test through close's
14602 // `await_pending_async_commits` step. Requires response routing
14603 // for the bg task to drive the incomplete-future timing
14604 // naturally — deferred to Phase 12.5 per
14605 // `Phase-12/RESPONSE-ROUTING-AUDIT.md`.
14606 // - testCloseLeavesGroupDespiteOnPartitionsLostError — Mockito's
14607 // `spy(newConsumer(...))` + `setGroupAssignmentSnapshot` API
14608 // surface does not have an inline Rust analog. The
14609 // `run_rebalance_callbacks_on_close` code path IS covered by
14610 // `close_runs_partitions_lost_on_unknown_epoch` /
14611 // `close_runs_partitions_revoked_on_live_epoch`; the additional
14612 // "leave group fires DESPITE listener throwing" assertion is
14613 // a Mockito-spy fixture cost not worth replicating here. The
14614 // Rust close path's `first_error` tracking + `LeaveGroupOnClose`
14615 // enqueue ordering is unconditional (close_internal:2680+) —
14616 // listener errors do not gate the leave step.
14617 // - testCloseLeavesGroupDespiteInterrupt — Java's
14618 // `InterruptException` has no Rust analog (no thread-interrupt
14619 // primitive). The `wakeup()` path (which is the Rust analog) is
14620 // covered by `close_caps_timeout_at_request_timeout_ms`-class
14621 // tests; the "InterruptException thrown by addAndGet" injection
14622 // is Mockito-only.
14623 // - testGroupRemoteAssignorUsedInConsumerProtocol — depends on
14624 // `ConsumerConfig::unused()` tracking, same blocker as Issue 27
14625 // (no inline Rust surface yet). The construction-side
14626 // assertion is partially covered by
14627 // `group_remote_assignor_unused_if_group_id_undefined` /
14628 // `group_id_null_constructs_successfully`.
14629
14630 /// Java: `testSuccessfulStartupShutdown`. A
14631 /// freshly-constructed consumer can be closed without throwing.
14632 /// The inline `close_is_idempotent` is the stricter version; this
14633 /// stays parallel to Java's name + body for clarity.
14634 #[tokio::test]
14635 async fn successful_startup_shutdown() {
14636 let (mut consumer, mut handles) = make_test_consumer_with_channels();
14637 let drainer = tokio::spawn(async move {
14638 while let Some(env) = handles.app_event_rx.recv().await {
14639 match env.event {
14640 ApplicationEvent::LeaveGroupOnClose { handle, .. } => {
14641 handle.complete(());
14642 },
14643 ApplicationEvent::CommitSync { handle, offsets_ready, .. } => {
14644 offsets_ready.complete(());
14645 handle.complete(HashMap::new());
14646 },
14647 _ => {},
14648 }
14649 }
14650 });
14651 consumer.close().await.expect("close ok");
14652 drop(drainer);
14653 }
14654
14655 /// Java: `testCloseAwaitPendingAsyncCommitComplete`.
14656 /// On close, a pending async commit's callback fires.
14657 #[tokio::test]
14658 async fn close_awaits_pending_async_commit_complete() {
14659 use std::sync::atomic::AtomicUsize;
14660 struct ClosingCallback {
14661 invoked: Arc<AtomicUsize>,
14662 }
14663 #[async_trait::async_trait]
14664 impl crate::consumer::OffsetCommitCallback for ClosingCallback {
14665 async fn on_complete(
14666 &self,
14667 _offsets: &HashMap<TopicPartition, OffsetAndMetadata>,
14668 _error: Option<&Error>,
14669 ) -> Result<(), Error> {
14670 self.invoked.fetch_add(1, Ordering::SeqCst);
14671 Ok(())
14672 }
14673 }
14674
14675 let (mut consumer, mut handles) = make_test_consumer_with_channels();
14676
14677 let drainer = tokio::spawn(async move {
14678 while let Some(env) = handles.app_event_rx.recv().await {
14679 match env.event {
14680 ApplicationEvent::CommitAsync { handle, offsets_ready, .. } => {
14681 offsets_ready.complete(());
14682 handle.complete(HashMap::new());
14683 },
14684 ApplicationEvent::CommitSync { handle, offsets_ready, .. } => {
14685 offsets_ready.complete(());
14686 handle.complete(HashMap::new());
14687 },
14688 ApplicationEvent::LeaveGroupOnClose { handle, .. } => {
14689 handle.complete(());
14690 },
14691 _ => {},
14692 }
14693 }
14694 });
14695
14696 let invoked = Arc::new(AtomicUsize::new(0));
14697 let cb = Arc::new(ClosingCallback { invoked: Arc::clone(&invoked) });
14698 consumer
14699 .commit_async_with_offsets_callback(HashMap::new(), cb)
14700 .await
14701 .expect("ok");
14702 consumer.close().await.expect("close ok");
14703 // The callback must have fired (close drains pending async commits).
14704 assert_eq!(
14705 invoked.load(Ordering::SeqCst),
14706 1,
14707 "pending async-commit callback must fire on close"
14708 );
14709 drop(drainer);
14710 }
14711
14712 /// Java: `testCloseLeavesGroup(0 || DEFAULT_CLOSE_TIMEOUT_MS)`
14713 /// (a `@ParameterizedTest`). The KIP-848 close
14714 /// path always submits a `LeaveGroupOnClose` event before
14715 /// shutting down. Translated as two test methods (timeout=0 and
14716 /// timeout=DEFAULT_CLOSE_TIMEOUT_MS) per the @ParameterizedTest →
14717 /// loop rule.
14718 async fn close_leaves_group_for_timeout_inner(timeout_ms: u64) {
14719 let (mut consumer, mut handles) = make_test_consumer_with_channels();
14720 let saw_leave = Arc::new(AtomicBool::new(false));
14721 let saw_leave_clone = Arc::clone(&saw_leave);
14722 let drainer = tokio::spawn(async move {
14723 while let Some(env) = handles.app_event_rx.recv().await {
14724 match env.event {
14725 ApplicationEvent::LeaveGroupOnClose { handle, .. } => {
14726 saw_leave_clone.store(true, Ordering::SeqCst);
14727 handle.complete(());
14728 },
14729 ApplicationEvent::CommitSync { handle, offsets_ready, .. } => {
14730 offsets_ready.complete(());
14731 handle.complete(HashMap::new());
14732 },
14733 ApplicationEvent::CommitAsync { handle, offsets_ready, .. } => {
14734 offsets_ready.complete(());
14735 handle.complete(HashMap::new());
14736 },
14737 _ => {},
14738 }
14739 }
14740 });
14741
14742 consumer
14743 .close_with_options(crate::consumer::CloseOptions::new_with_timeout(Duration::from_millis(
14744 timeout_ms,
14745 )))
14746 .await
14747 .expect("close ok");
14748
14749 // close()'s last step drops `application_event_handler` and the
14750 // network_thread_close handler. The drainer's `app_event_rx.recv()`
14751 // returns None once all senders are dropped. Wait for the drainer
14752 // to fully drain.
14753 drop(consumer);
14754 let _ = drainer.await;
14755
14756 assert!(
14757 saw_leave.load(Ordering::SeqCst),
14758 "LeaveGroupOnClose must be enqueued (timeout={timeout_ms})"
14759 );
14760 }
14761
14762 #[tokio::test]
14763 async fn close_leaves_group_timeout_zero() {
14764 close_leaves_group_for_timeout_inner(0).await;
14765 }
14766
14767 #[tokio::test]
14768 async fn close_leaves_group_timeout_default() {
14769 // Java: `ConsumerUtils.DEFAULT_CLOSE_TIMEOUT_MS = 30_000`.
14770 use crate::consumer::internals::ConsumerUtils;
14771 close_leaves_group_for_timeout_inner(ConsumerUtils::DEFAULT_CLOSE_TIMEOUT_MS as u64).await;
14772 }
14773
14774 /// A fake bg task serving the close-path events: commits complete at
14775 /// once, and each `LeaveGroupOnClose` handle is passed to `on_leave`,
14776 /// which decides when (and whether) the "network thread" completes it.
14777 fn spawn_close_path_drainer<F, Fut>(
14778 mut app_event_rx: mpsc::UnboundedReceiver<ApplicationEventEnvelope>,
14779 on_leave: F,
14780 ) -> tokio::task::JoinHandle<()>
14781 where
14782 F: Fn(crate::consumer::internals::events::CompletableEventHandle<()>) -> Fut + Send + 'static,
14783 Fut: std::future::Future<Output = ()> + Send + 'static,
14784 {
14785 tokio::spawn(async move {
14786 while let Some(env) = app_event_rx.recv().await {
14787 match env.event {
14788 ApplicationEvent::LeaveGroupOnClose { handle, .. } => on_leave(handle).await,
14789 ApplicationEvent::CommitSync { handle, offsets_ready, .. } => {
14790 offsets_ready.complete(());
14791 handle.complete(HashMap::new());
14792 },
14793 ApplicationEvent::CommitAsync { handle, offsets_ready, .. } => {
14794 offsets_ready.complete(());
14795 handle.complete(HashMap::new());
14796 },
14797 _ => {},
14798 }
14799 }
14800 })
14801 }
14802
14803 /// Audit finding C1-17: `close(Duration::ZERO)` must not shut the
14804 /// background task down before it has processed `LeaveGroupOnClose`.
14805 ///
14806 /// Java's `leaveGroupOnClose` calls `applicationEventHandler.addAndGet(..)`,
14807 /// which blocks on an unbounded `ConsumerUtils.getResult(future)`: the
14808 /// future is completed only by the network thread, which processes the
14809 /// event (member -> LEAVING) before its reaper expires it at the (already
14810 /// passed) deadline. So the member is LEAVING before
14811 /// `applicationEventHandler.close(..)` stops the network thread, and
14812 /// `cleanup()` sends the leave heartbeat even with a 0 ms budget.
14813 ///
14814 /// The fake bg task here processes the event one ~20 ms "iteration" after
14815 /// it is enqueued and then expires it, as the reaper does.
14816 #[tokio::test]
14817 async fn close_with_zero_timeout_waits_for_bg_to_process_leave_group_on_close() {
14818 let (mut consumer, handles) = make_test_consumer_with_channels();
14819 let signal_close_timeout_ms = Arc::clone(&handles.signal_close_timeout_ms);
14820 // `Some(signalled?)` captured when the fake bg task processes the event.
14821 let signalled_at_processing: Arc<Mutex<Option<bool>>> = Arc::new(Mutex::new(None));
14822 let at_processing = Arc::clone(&signalled_at_processing);
14823 let drainer = spawn_close_path_drainer(handles.app_event_rx, move |handle| {
14824 let signal_close_timeout_ms = Arc::clone(&signal_close_timeout_ms);
14825 let at_processing = Arc::clone(&at_processing);
14826 async move {
14827 tokio::time::sleep(Duration::from_millis(20)).await;
14828 at_processing
14829 .lock()
14830 .unwrap()
14831 .get_or_insert(signal_close_timeout_ms.load(Ordering::Acquire) >= 0);
14832 handle.complete_with_error(Error::timeout("LeaveGroupOnClose was 20 ms past its expiration"));
14833 }
14834 });
14835
14836 consumer
14837 .close_with_options(crate::consumer::CloseOptions::new_with_timeout(Duration::ZERO))
14838 .await
14839 .expect("close ok: an expired leave only logs a warning, as in Java");
14840 drainer.abort();
14841
14842 let signalled = signalled_at_processing
14843 .lock()
14844 .unwrap()
14845 .expect("the bg task must have processed LeaveGroupOnClose");
14846 assert!(
14847 !signalled,
14848 "Java: close(Duration.ZERO) blocks in addAndGet(LeaveGroupOnCloseEvent) until the network \
14849 thread has processed (or reaped) the event, so the member is LEAVING before the network \
14850 thread is closed; Rust signalled the bg task to shut down before LeaveGroupOnClose was processed"
14851 );
14852 assert!(
14853 handles.signal_close_timeout_ms.load(Ordering::Acquire) >= 0,
14854 "close must still signal the bg task to shut down"
14855 );
14856 }
14857
14858 /// With a close timeout, `close()` waits for the leave heartbeat's
14859 /// response (the event's completion), then proceeds — Java's
14860 /// `addAndGet` returning and `log.info("Completed leaving the group")`.
14861 #[tokio::test]
14862 async fn close_with_timeout_waits_for_leave_group_on_close_completion() {
14863 let (mut consumer, handles) = make_test_consumer_with_channels();
14864 let signal_close_timeout_ms = Arc::clone(&handles.signal_close_timeout_ms);
14865 let signalled_at_completion: Arc<Mutex<Option<bool>>> = Arc::new(Mutex::new(None));
14866 let at_completion = Arc::clone(&signalled_at_completion);
14867 let drainer = spawn_close_path_drainer(handles.app_event_rx, move |handle| {
14868 let signal_close_timeout_ms = Arc::clone(&signal_close_timeout_ms);
14869 let at_completion = Arc::clone(&at_completion);
14870 async move {
14871 tokio::time::sleep(Duration::from_millis(50)).await;
14872 at_completion
14873 .lock()
14874 .unwrap()
14875 .get_or_insert(signal_close_timeout_ms.load(Ordering::Acquire) >= 0);
14876 handle.complete(());
14877 }
14878 });
14879
14880 let started = std::time::Instant::now();
14881 consumer
14882 .close_with_options(crate::consumer::CloseOptions::new_with_timeout(Duration::from_secs(30)))
14883 .await
14884 .expect("close ok");
14885 let elapsed = started.elapsed();
14886 drainer.abort();
14887
14888 assert_eq!(
14889 *signalled_at_completion.lock().unwrap(),
14890 Some(false),
14891 "close must wait for the LeaveGroupOnClose completion before shutting the bg task down"
14892 );
14893 assert!(
14894 elapsed < Duration::from_secs(10),
14895 "close must return once the leave completes, not wait out the close timeout; took {elapsed:?}"
14896 );
14897 }
14898
14899 /// Deliberate deviation from Java (whose wait is unbounded): if the bg
14900 /// task is alive but never completes `LeaveGroupOnClose` — its reaper
14901 /// would otherwise expire it at the event deadline — `close()` gives up at
14902 /// the event deadline (here already passed, so the submission time) +
14903 /// `request.timeout.ms` and proceeds, instead of hanging.
14904 #[tokio::test]
14905 async fn close_leave_group_wait_gives_up_at_deadline_plus_request_timeout() {
14906 let (mut consumer, handles) = make_test_consumer_with_channels();
14907 consumer.config.request_timeout_ms = 300;
14908 // Keep every handle alive and uncompleted: a stuck bg iteration.
14909 let held: Arc<Mutex<Vec<crate::consumer::internals::events::CompletableEventHandle<()>>>> =
14910 Arc::new(Mutex::new(Vec::new()));
14911 let held_by_bg = Arc::clone(&held);
14912 let drainer = spawn_close_path_drainer(handles.app_event_rx, move |handle| {
14913 held_by_bg.lock().unwrap().push(handle);
14914 async {}
14915 });
14916
14917 let started = std::time::Instant::now();
14918 consumer
14919 .close_with_options(crate::consumer::CloseOptions::new_with_timeout(Duration::ZERO))
14920 .await
14921 .expect("close ok: the backstop only logs the leave warning");
14922 let elapsed = started.elapsed();
14923 drainer.abort();
14924
14925 assert_eq!(held.lock().unwrap().len(), 1, "LeaveGroupOnClose must have been submitted");
14926 assert!(
14927 elapsed >= Duration::from_millis(250),
14928 "close must keep waiting for the bg task past the (already expired) event deadline \
14929 until deadline + request.timeout.ms (300 ms); returned after {elapsed:?}"
14930 );
14931 assert!(
14932 elapsed < Duration::from_secs(10),
14933 "the backstop must bound the wait at about deadline + request.timeout.ms; took {elapsed:?}"
14934 );
14935 assert!(handles.signal_close_timeout_ms.load(Ordering::Acquire) >= 0);
14936 }
14937
14938 /// Runs `close(Duration::ZERO)` against a fake bg task that processes
14939 /// `LeaveGroupOnClose` one ~20 ms "iteration" after it is enqueued, and
14940 /// returns `Some(signalled?)` as seen when it processed the event (`None`
14941 /// when it never processed it).
14942 async fn close_with_zero_timeout_and_slow_bg(
14943 mut consumer: AsyncKafkaConsumer<Vec<u8>, Vec<u8>>,
14944 handles: ConsumerTestHandles,
14945 ) -> Option<bool> {
14946 let signal_close_timeout_ms = Arc::clone(&handles.signal_close_timeout_ms);
14947 let signalled_at_processing: Arc<Mutex<Option<bool>>> = Arc::new(Mutex::new(None));
14948 let at_processing = Arc::clone(&signalled_at_processing);
14949 let drainer = spawn_close_path_drainer(handles.app_event_rx, move |handle| {
14950 let signal_close_timeout_ms = Arc::clone(&signal_close_timeout_ms);
14951 let at_processing = Arc::clone(&at_processing);
14952 async move {
14953 tokio::time::sleep(Duration::from_millis(20)).await;
14954 at_processing
14955 .lock()
14956 .unwrap()
14957 .get_or_insert(signal_close_timeout_ms.load(Ordering::Acquire) >= 0);
14958 handle.complete_with_error(Error::timeout("LeaveGroupOnClose was 20 ms past its expiration"));
14959 }
14960 });
14961 consumer
14962 .close_with_options(crate::consumer::CloseOptions::new_with_timeout(Duration::ZERO))
14963 .await
14964 .expect("close ok: an expired leave only logs a warning, as in Java");
14965 drainer.abort();
14966 assert!(
14967 handles.signal_close_timeout_ms.load(Ordering::Acquire) >= 0,
14968 "close must still signal the bg task to shut down"
14969 );
14970 *signalled_at_processing.lock().unwrap()
14971 }
14972
14973 /// Review fix (PR #263): the close backstop is measured from the moment
14974 /// `LeaveGroupOnClose` is submitted, so a revoked listener that ran past
14975 /// close timeout + `request.timeout.ms` cannot leave it already expired.
14976 /// Here `close(Duration::ZERO)` with `request.timeout.ms=300` first runs a
14977 /// 400 ms `on_partitions_revoked`. Java: `leaveGroupOnClose`'s `addAndGet`
14978 /// waits on an unbounded `ConsumerUtils.getResult(future)`, so the network
14979 /// thread always processes the event (member -> LEAVING) before
14980 /// `applicationEventHandler.close(..)` stops it.
14981 #[tokio::test]
14982 async fn close_waits_for_leave_group_on_close_after_a_listener_outlasting_the_backstop() {
14983 use async_trait::async_trait;
14984
14985 struct SlowRevokeListener;
14986 #[async_trait]
14987 impl ConsumerRebalanceListener for SlowRevokeListener {
14988 async fn on_partitions_assigned(&self, _: &[TopicPartition]) -> Result<(), Error> {
14989 Ok(())
14990 }
14991 async fn on_partitions_revoked(&self, _: &[TopicPartition]) -> Result<(), Error> {
14992 tokio::time::sleep(Duration::from_millis(400)).await;
14993 Ok(())
14994 }
14995 }
14996
14997 let (mut consumer, handles) = make_test_consumer_with_channels();
14998 consumer.config.request_timeout_ms = 300;
14999 *consumer.rebalance_listener.lock().unwrap() =
15000 Some(Arc::new(SlowRevokeListener) as Arc<dyn ConsumerRebalanceListener>);
15001 let notifier = consumer.state_notifier();
15002 notifier.on_group_assignment_updated(&HashSet::from([TopicPartition::new("t".to_string(), 0)]));
15003 notifier.on_member_epoch_updated(Some(5), "member-1");
15004
15005 let started = std::time::Instant::now();
15006 let signalled = close_with_zero_timeout_and_slow_bg(consumer, handles).await;
15007 let elapsed = started.elapsed();
15008 assert!(
15009 elapsed >= Duration::from_millis(400),
15010 "the revoked listener must have run before the leave; took {elapsed:?}"
15011 );
15012 assert_eq!(
15013 signalled,
15014 Some(false),
15015 "Java: close blocks in addAndGet(LeaveGroupOnCloseEvent) until the network thread has \
15016 processed the event, however long the revoked listener ran; Rust must not give up on a \
15017 backstop that expired before the event was submitted (Some(true) = shut down first, \
15018 None = never processed)"
15019 );
15020 }
15021
15022 /// Review fix (PR #263): `request.timeout.ms=0` (accepted, `atLeast(0)`)
15023 /// with `close(Duration::ZERO)` still waits for the bg task to process
15024 /// `LeaveGroupOnClose`: the backstop's grace period is positive even when
15025 /// `request.timeout.ms` is zero. Java: `addAndGet` waits on an unbounded
15026 /// `ConsumerUtils.getResult(future)`.
15027 #[tokio::test]
15028 async fn close_waits_for_leave_group_on_close_with_zero_request_timeout() {
15029 let (mut consumer, handles) = make_test_consumer_with_channels();
15030 consumer.config.request_timeout_ms = 0;
15031 let signalled = close_with_zero_timeout_and_slow_bg(consumer, handles).await;
15032 assert_eq!(
15033 signalled,
15034 Some(false),
15035 "Java: close blocks in addAndGet(LeaveGroupOnCloseEvent) until the network thread has \
15036 processed the event, even with request.timeout.ms=0; Rust must not give up on a \
15037 zero-length backstop (Some(true) = shut down first, None = never processed)"
15038 );
15039 }
15040
15041 /// The bg task drops `LeaveGroupOnClose`'s completion sender without
15042 /// completing it (it shut down): `close()` proceeds without an error, as
15043 /// Java's close does when the network thread's shutdown fails the event
15044 /// with a `TimeoutException`.
15045 #[tokio::test]
15046 async fn close_proceeds_when_bg_task_drops_leave_group_on_close() {
15047 let (mut consumer, handles) = make_test_consumer_with_channels();
15048 let drainer = spawn_close_path_drainer(handles.app_event_rx, |handle| {
15049 drop(handle);
15050 async {}
15051 });
15052
15053 let started = std::time::Instant::now();
15054 consumer
15055 .close_with_options(crate::consumer::CloseOptions::new_with_timeout(Duration::from_secs(30)))
15056 .await
15057 .expect("close ok: a bg task that ended without completing the leave does not fail close");
15058 let elapsed = started.elapsed();
15059 drainer.abort();
15060 assert!(
15061 elapsed < Duration::from_secs(10),
15062 "close must not wait for a gone bg task; took {elapsed:?}"
15063 );
15064 }
15065
15066 /// The bg task is already gone when `close()` starts: there is nothing to
15067 /// wait for, so close returns at once. Java's `add` throws
15068 /// `KafkaException` from `ensureNetworkThreadAlive()` and close rethrows
15069 /// the first error as "Failed to close kafka consumer".
15070 #[tokio::test]
15071 async fn close_returns_at_once_when_bg_task_is_already_gone() {
15072 let (mut consumer, handles) = make_test_consumer_with_channels();
15073 drop(handles.app_event_rx);
15074
15075 let started = std::time::Instant::now();
15076 let err = consumer
15077 .close_with_options(crate::consumer::CloseOptions::new_with_timeout(Duration::from_secs(30)))
15078 .await
15079 .expect_err("Java: close() rethrows the dead-thread KafkaException");
15080 let elapsed = started.elapsed();
15081 assert!(
15082 elapsed < Duration::from_secs(5),
15083 "close must not wait for a gone bg task; took {elapsed:?}"
15084 );
15085 assert_eq!(err.to_string(), "Failed to close kafka consumer");
15086 }
15087
15088 /// With the real background task, `close(Duration::ZERO)` returns once
15089 /// the bg reaper expires `LeaveGroupOnClose` at its (already passed)
15090 /// deadline, within about one bg iteration, not after the
15091 /// `request.timeout.ms` (30 s) backstop. The member joins through a fake
15092 /// group coordinator that never answers the leave heartbeat, so the leave
15093 /// cannot complete by a response, and since the coordinator is known the
15094 /// heartbeat is sent rather than skipped (`onHeartbeatRequestSkipped`
15095 /// would complete the leave at once). The reaper's timeout is therefore
15096 /// the only way the event completes. Java: `ConsumerNetworkThread.runOnce`
15097 /// processes the event (building the leave heartbeat) before it reaps it,
15098 /// and `leaveGroupOnClose` then logs "couldn't complete it within" and
15099 /// proceeds.
15100 #[test]
15101 fn close_with_zero_timeout_returns_when_the_bg_reaper_expires_the_leave() {
15102 use std::collections::HashMap;
15103
15104 use crate::common::serialization::StringDeserializer;
15105 use crate::consumer::{CloseOptions, Consumer, KafkaConsumer};
15106 use crate::test_fake_broker::FakeBroker;
15107
15108 const TOPIC: &str = "topic";
15109 let runtime = tokio::runtime::Builder::new_multi_thread().enable_all().build().unwrap();
15110 let (elapsed, epochs_at_close, epochs_after_close) = runtime.block_on(async move {
15111 let broker = FakeBroker::start_with_group_coordinator("fake-cluster", TOPIC).await;
15112 let props = HashMap::from([
15113 (ConsumerConfig::BOOTSTRAP_SERVERS_CONFIG.to_string(), broker.bootstrap()),
15114 (ConsumerConfig::GROUP_PROTOCOL_CONFIG.to_string(), "consumer".to_string()),
15115 (ConsumerConfig::GROUP_ID_CONFIG.to_string(), "test-group".to_string()),
15116 ]);
15117 let mut consumer = KafkaConsumer::<String, String>::new(
15118 ConsumerConfig::new(&props).unwrap(),
15119 StringDeserializer,
15120 StringDeserializer,
15121 )
15122 .expect("consumer");
15123 consumer
15124 .subscribe_with_topics(vec![TOPIC.to_string()])
15125 .await
15126 .expect("subscribe");
15127 let deadline = tokio::time::Instant::now() + Duration::from_secs(30);
15128 while consumer.group_metadata().generation_id() != 1 && tokio::time::Instant::now() < deadline {
15129 consumer.poll(Duration::from_millis(50)).await.expect("poll");
15130 }
15131 assert_eq!(
15132 1,
15133 consumer.group_metadata().generation_id(),
15134 "the member must join the fake coordinator first; heartbeats: {:?}",
15135 broker.heartbeat_epochs()
15136 );
15137 let epochs_at_close = broker.heartbeat_epochs();
15138 let started = std::time::Instant::now();
15139 consumer
15140 .close_with_options(CloseOptions::new_with_timeout(Duration::ZERO))
15141 .await
15142 .expect("close");
15143 let elapsed = started.elapsed();
15144 let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
15145 while !broker.heartbeat_epochs().contains(&-1) && tokio::time::Instant::now() < deadline {
15146 tokio::time::sleep(Duration::from_millis(10)).await;
15147 }
15148 (elapsed, epochs_at_close, broker.heartbeat_epochs())
15149 });
15150 assert!(
15151 !epochs_at_close.contains(&-1),
15152 "no leave heartbeat before close: {epochs_at_close:?}"
15153 );
15154 assert!(
15155 epochs_after_close.contains(&-1),
15156 "Java: the network thread processes LeaveGroupOnClose before reaping it, so the leave \
15157 heartbeat (epoch -1) is sent even with a zero timeout; heartbeats: {epochs_after_close:?}"
15158 );
15159 assert!(
15160 elapsed < Duration::from_secs(10),
15161 "close(Duration.ZERO) must return when the reaper expires LeaveGroupOnClose, not after the \
15162 request.timeout.ms backstop; took {elapsed:?}"
15163 );
15164 }
15165
15166 /// Java: `AsyncKafkaConsumerTest#testVerifyApplicationEventOnShutdown`.
15167 /// On close, `CommitOnCloseEvent` is enqueued (Java verifies via
15168 /// `verify(applicationEventHandler).add(any(CommitOnCloseEvent.class))`).
15169 #[tokio::test]
15170 async fn close_enqueues_commit_on_close_event() {
15171 let (mut consumer, mut handles) = make_test_consumer_with_channels();
15172 let saw_commit_on_close = Arc::new(AtomicBool::new(false));
15173 let saw_clone = Arc::clone(&saw_commit_on_close);
15174 let drainer = tokio::spawn(async move {
15175 while let Some(env) = handles.app_event_rx.recv().await {
15176 match env.event {
15177 ApplicationEvent::CommitOnClose => {
15178 saw_clone.store(true, Ordering::SeqCst);
15179 },
15180 ApplicationEvent::LeaveGroupOnClose { handle, .. } => {
15181 handle.complete(());
15182 },
15183 ApplicationEvent::CommitSync { handle, offsets_ready, .. } => {
15184 offsets_ready.complete(());
15185 handle.complete(HashMap::new());
15186 },
15187 ApplicationEvent::CommitAsync { handle, offsets_ready, .. } => {
15188 offsets_ready.complete(());
15189 handle.complete(HashMap::new());
15190 },
15191 _ => {},
15192 }
15193 }
15194 });
15195 consumer.close().await.expect("close ok");
15196 assert!(
15197 saw_commit_on_close.load(Ordering::SeqCst),
15198 "CommitOnClose event must be enqueued"
15199 );
15200 drop(drainer);
15201 }
15202
15203 /// Java: `AsyncKafkaConsumerTest#testSeekToBeginning`. `seek_to_beginning`
15204 /// enqueues a `ResetOffset` event with the EARLIEST strategy.
15205 #[tokio::test]
15206 async fn seek_to_beginning_enqueues_reset_offset_event() {
15207 let (mut consumer, mut handles) = make_test_consumer_with_channels();
15208 let tp = TopicPartition::new("test".to_string(), 0);
15209 let topics = vec![tp.clone()];
15210
15211 let captured_strategy = Arc::new(std::sync::Mutex::new(None::<AutoOffsetResetStrategy>));
15212 let captured_partitions = Arc::new(std::sync::Mutex::new(Vec::<TopicPartition>::new()));
15213 let cap_strategy = Arc::clone(&captured_strategy);
15214 let cap_partitions = Arc::clone(&captured_partitions);
15215 let drainer = tokio::spawn(async move {
15216 while let Some(env) = handles.app_event_rx.recv().await {
15217 if let ApplicationEvent::ResetOffset { handle, partitions, offset_reset_strategy } = env.event {
15218 *cap_strategy.lock().unwrap() = Some(offset_reset_strategy);
15219 *cap_partitions.lock().unwrap() = partitions.into_iter().collect();
15220 handle.complete(());
15221 return;
15222 }
15223 }
15224 });
15225
15226 consumer.seek_to_beginning(&topics).await.expect("ok");
15227 drainer.await.expect("task ok");
15228
15229 let strat = captured_strategy.lock().unwrap().clone().expect("ResetOffset event seen");
15230 assert_eq!(strat, AutoOffsetResetStrategy::EARLIEST);
15231 let parts = captured_partitions.lock().unwrap().clone();
15232 assert!(parts.contains(&tp));
15233 }
15234
15235 /// Java: `testSeekToEnd`. Symmetric to
15236 /// seek_to_beginning but with LATEST.
15237 #[tokio::test]
15238 async fn seek_to_end_enqueues_reset_offset_event() {
15239 let (mut consumer, mut handles) = make_test_consumer_with_channels();
15240 let tp = TopicPartition::new("test".to_string(), 0);
15241 let topics = vec![tp.clone()];
15242
15243 let captured_strategy = Arc::new(std::sync::Mutex::new(None::<AutoOffsetResetStrategy>));
15244 let cap_strategy = Arc::clone(&captured_strategy);
15245 let drainer = tokio::spawn(async move {
15246 while let Some(env) = handles.app_event_rx.recv().await {
15247 if let ApplicationEvent::ResetOffset { handle, offset_reset_strategy, .. } = env.event {
15248 *cap_strategy.lock().unwrap() = Some(offset_reset_strategy);
15249 handle.complete(());
15250 return;
15251 }
15252 }
15253 });
15254
15255 consumer.seek_to_end(&topics).await.expect("ok");
15256 drainer.await.expect("task ok");
15257
15258 let strat = captured_strategy.lock().unwrap().clone().expect("ResetOffset event seen");
15259 assert_eq!(strat, AutoOffsetResetStrategy::LATEST);
15260 }
15261
15262 /// Java: `testSeekToBeginningWithException`.
15263 /// When the `ResetOffsetEvent` is completed exceptionally with a
15264 /// timeout, `seek_to_beginning` surfaces the error.
15265 #[tokio::test]
15266 async fn seek_to_beginning_propagates_event_error() {
15267 let (mut consumer, mut handles) = make_test_consumer_with_channels();
15268 let tp = TopicPartition::new("test".to_string(), 0);
15269
15270 let drainer = tokio::spawn(async move {
15271 while let Some(env) = handles.app_event_rx.recv().await {
15272 if let ApplicationEvent::ResetOffset { handle, .. } = env.event {
15273 handle.complete_with_error(Error::timeout("test timeout"));
15274 return;
15275 }
15276 }
15277 });
15278
15279 let err = consumer.seek_to_beginning(&[tp]).await.expect_err("must err");
15280 assert!(matches!(err, Error::Timeout(_)), "unexpected err: {err:?}");
15281 drainer.await.expect("task ok");
15282 }
15283
15284 /// Java: `testSeekToEndWithException`. Symmetric.
15285 #[tokio::test]
15286 async fn seek_to_end_propagates_event_error() {
15287 let (mut consumer, mut handles) = make_test_consumer_with_channels();
15288 let tp = TopicPartition::new("test".to_string(), 0);
15289
15290 let drainer = tokio::spawn(async move {
15291 while let Some(env) = handles.app_event_rx.recv().await {
15292 if let ApplicationEvent::ResetOffset { handle, .. } = env.event {
15293 handle.complete_with_error(Error::timeout("test timeout"));
15294 return;
15295 }
15296 }
15297 });
15298
15299 let err = consumer.seek_to_end(&[tp]).await.expect_err("must err");
15300 assert!(matches!(err, Error::Timeout(_)), "unexpected err: {err:?}");
15301 drainer.await.expect("task ok");
15302 }
15303
15304 /// Java: `testBeginningOffsets`. With a positive
15305 /// timeout the `beginning_offsets_with_timeout` waits for the
15306 /// `ListOffsets` event to complete and returns the per-partition
15307 /// offsets map.
15308 #[tokio::test]
15309 async fn beginning_offsets_returns_event_result() {
15310 let (mut consumer, mut handles) = make_test_consumer_with_channels();
15311 let t0 = TopicPartition::new("t0".to_string(), 2);
15312 let t1 = TopicPartition::new("t0".to_string(), 3);
15313
15314 let drainer = tokio::spawn(async move {
15315 while let Some(env) = handles.app_event_rx.recv().await {
15316 if let ApplicationEvent::ListOffsets { handle, .. } = env.event {
15317 let mut result: HashMap<TopicPartition, Option<OffsetAndTimestampInternal>> = HashMap::new();
15318 result.insert(
15319 TopicPartition::new("t0".to_string(), 2),
15320 Some(OffsetAndTimestampInternal::new(5, 1, None)),
15321 );
15322 result.insert(
15323 TopicPartition::new("t0".to_string(), 3),
15324 Some(OffsetAndTimestampInternal::new(6, 3, None)),
15325 );
15326 handle.complete(result);
15327 return;
15328 }
15329 }
15330 });
15331
15332 let offsets = consumer
15333 .beginning_offsets_with_timeout(&[t0.clone(), t1.clone()], Duration::from_millis(100))
15334 .await
15335 .expect("ok");
15336 assert_eq!(offsets.get(&t0), Some(&5));
15337 assert_eq!(offsets.get(&t1), Some(&6));
15338 drainer.await.expect("task ok");
15339 }
15340
15341 /// Java: `OffsetFetcherTest.testBeginningOffsetsDuplicateTopicPartition`
15342 /// (`beginningOffsets(asList(tp0, tp0))`). The duplicate partition in
15343 /// the input slice collapses to a single entry: the
15344 /// `timestamps_to_search` map built in `beginning_or_end_offsets`
15345 /// (slice → `HashMap`) de-duplicates, producing exactly ONE wire
15346 /// request / ONE result entry.
15347 #[tokio::test]
15348 async fn beginning_offsets_duplicate_topic_partition_collapses() {
15349 let (mut consumer, mut handles) = make_test_consumer_with_channels();
15350 let tp = TopicPartition::new("t0".to_string(), 0);
15351
15352 let drainer = tokio::spawn(async move {
15353 while let Some(env) = handles.app_event_rx.recv().await {
15354 if let ApplicationEvent::ListOffsets { handle, timestamps_to_search, .. } = env.event {
15355 // The duplicate tp must have collapsed to one entry.
15356 assert_eq!(
15357 timestamps_to_search.len(),
15358 1,
15359 "duplicate topic-partition must collapse to a single search entry"
15360 );
15361 let mut result: HashMap<TopicPartition, Option<OffsetAndTimestampInternal>> = HashMap::new();
15362 result.insert(
15363 TopicPartition::new("t0".to_string(), 0),
15364 Some(OffsetAndTimestampInternal::new(2, -1, None)),
15365 );
15366 handle.complete(result);
15367 return;
15368 }
15369 }
15370 });
15371
15372 let offsets = consumer
15373 .beginning_offsets_with_timeout(&[tp.clone(), tp.clone()], Duration::from_millis(100))
15374 .await
15375 .expect("ok");
15376 assert_eq!(offsets.len(), 1, "duplicate partition collapses to one result entry");
15377 assert_eq!(offsets.get(&tp), Some(&2));
15378 drainer.await.expect("task ok");
15379 }
15380
15381 /// Java: `OffsetFetcherTest.testEndOffsetsDuplicateTopicPartition`
15382 /// (`endOffsets(asList(tp0, tp0))`). Same slice→map collapse as the
15383 /// beginning-offsets variant.
15384 #[tokio::test]
15385 async fn end_offsets_duplicate_topic_partition_collapses() {
15386 let (mut consumer, mut handles) = make_test_consumer_with_channels();
15387 let tp = TopicPartition::new("t0".to_string(), 0);
15388
15389 let drainer = tokio::spawn(async move {
15390 while let Some(env) = handles.app_event_rx.recv().await {
15391 if let ApplicationEvent::ListOffsets { handle, timestamps_to_search, .. } = env.event {
15392 assert_eq!(
15393 timestamps_to_search.len(),
15394 1,
15395 "duplicate topic-partition must collapse to a single search entry"
15396 );
15397 let mut result: HashMap<TopicPartition, Option<OffsetAndTimestampInternal>> = HashMap::new();
15398 result.insert(
15399 TopicPartition::new("t0".to_string(), 0),
15400 Some(OffsetAndTimestampInternal::new(5, -1, None)),
15401 );
15402 handle.complete(result);
15403 return;
15404 }
15405 }
15406 });
15407
15408 let offsets = consumer
15409 .end_offsets_with_timeout(&[tp.clone(), tp.clone()], Duration::from_millis(100))
15410 .await
15411 .expect("ok");
15412 assert_eq!(offsets.len(), 1, "duplicate partition collapses to one result entry");
15413 assert_eq!(offsets.get(&tp), Some(&5));
15414 drainer.await.expect("task ok");
15415 }
15416
15417 /// Java: `testBeginningOffsetsThrowsKafkaExceptionForUnderlyingExecutionFailure`
15418 /// The `ListOffsetsEvent` completes
15419 /// exceptionally and the error propagates.
15420 #[tokio::test]
15421 async fn beginning_offsets_propagates_event_error() {
15422 let (mut consumer, mut handles) = make_test_consumer_with_channels();
15423 let tp = TopicPartition::new("t0".to_string(), 0);
15424
15425 let drainer = tokio::spawn(async move {
15426 while let Some(env) = handles.app_event_rx.recv().await {
15427 if let ApplicationEvent::ListOffsets { handle, .. } = env.event {
15428 handle.complete_with_error(Error::local_illegal_state(
15429 "Unexpected failure processing List Offsets event",
15430 ));
15431 return;
15432 }
15433 }
15434 });
15435
15436 let err = consumer
15437 .beginning_offsets_with_timeout(&[tp], Duration::from_millis(100))
15438 .await
15439 .expect_err("must err");
15440 assert!(!matches!(err, Error::Timeout(_)), "non-timeout err propagates, got {err:?}");
15441 drainer.await.expect("task ok");
15442 }
15443
15444 /// Java: `testBeginningOffsetsTimeoutException`
15445 /// — the event times out, surfacing as `Timeout` with the exact
15446 /// Java message `"Failed to get offsets by times in {timeout}ms"`.
15447 /// (Issue 25 / DoD §3: exact-message assertion.)
15448 #[tokio::test]
15449 async fn beginning_offsets_propagates_timeout() {
15450 let (mut consumer, mut handles) = make_test_consumer_with_channels();
15451 let tp = TopicPartition::new("topic".to_string(), 5);
15452
15453 let drainer = tokio::spawn(async move {
15454 while let Some(env) = handles.app_event_rx.recv().await {
15455 if let ApplicationEvent::ListOffsets { handle, .. } = env.event {
15456 handle.complete_with_error(Error::timeout(
15457 "Event did not complete in time and was expired by the reaper",
15458 ));
15459 return;
15460 }
15461 }
15462 });
15463
15464 let err = consumer
15465 .beginning_offsets_with_timeout(&[tp], Duration::from_millis(100))
15466 .await
15467 .expect_err("must err");
15468 match err {
15469 Error::Timeout(msg) => {
15470 assert_eq!(msg.message(), "Failed to get offsets by times in 100ms");
15471 },
15472 other => panic!("expected Timeout, got {other:?}"),
15473 }
15474 drainer.await.expect("task ok");
15475 }
15476
15477 /// Java: `testEndOffsetsTimeoutException`. Symmetric
15478 /// of `beginning_offsets_propagates_timeout`. Asserts the exact
15479 /// message contract for `end_offsets_with_timeout` since both routes share
15480 /// the timeout-format string in production code (Issue 25).
15481 #[tokio::test]
15482 async fn end_offsets_propagates_timeout_with_exact_message() {
15483 let (mut consumer, mut handles) = make_test_consumer_with_channels();
15484 let tp = TopicPartition::new("topic".to_string(), 5);
15485
15486 let drainer = tokio::spawn(async move {
15487 while let Some(env) = handles.app_event_rx.recv().await {
15488 if let ApplicationEvent::ListOffsets { handle, .. } = env.event {
15489 handle.complete_with_error(Error::timeout(
15490 "Event did not complete in time and was expired by the reaper",
15491 ));
15492 return;
15493 }
15494 }
15495 });
15496
15497 let err = consumer
15498 .end_offsets_with_timeout(&[tp], Duration::from_millis(250))
15499 .await
15500 .expect_err("must err");
15501 match err {
15502 Error::Timeout(msg) => {
15503 assert_eq!(msg.message(), "Failed to get offsets by times in 250ms");
15504 },
15505 other => panic!("expected Timeout, got {other:?}"),
15506 }
15507 drainer.await.expect("task ok");
15508 }
15509
15510 /// Java: `testBeginningOffsetsWithZeroTimeout`.
15511 /// `beginning_offsets_with_timeout(tp, ZERO)` enqueues the event via
15512 /// `add(...)` (NOT `add_and_get`) and returns an empty map without
15513 /// blocking. Issue 24.
15514 #[tokio::test]
15515 async fn beginning_offsets_with_zero_timeout_returns_empty_and_enqueues_event() {
15516 let (mut consumer, mut handles) = make_test_consumer_with_channels();
15517 let tp = TopicPartition::new("topic1".to_string(), 0);
15518
15519 let saw_list_offsets = Arc::new(std::sync::atomic::AtomicBool::new(false));
15520 let saw_flag = Arc::clone(&saw_list_offsets);
15521 let drainer = tokio::spawn(async move {
15522 // Pull the envelope but DO NOT complete it (Java's `add`
15523 // path leaves the handle dangling; the receiver is dropped).
15524 while let Some(env) = handles.app_event_rx.recv().await {
15525 if let ApplicationEvent::ListOffsets { .. } = env.event {
15526 saw_flag.store(true, Ordering::SeqCst);
15527 return;
15528 }
15529 }
15530 });
15531
15532 let map = consumer
15533 .beginning_offsets_with_timeout(&[tp], Duration::from_millis(0))
15534 .await
15535 .expect("zero-timeout returns Ok with empty map");
15536 assert!(map.is_empty(), "zero-timeout returns empty map");
15537 drainer.await.expect("drainer ok");
15538 assert!(saw_list_offsets.load(Ordering::SeqCst), "ListOffsets event must be enqueued");
15539 }
15540
15541 /// Java: `testOffsetsForTimesWithZeroTimeout`.
15542 /// `offsets_for_times_with_timeout(map, ZERO)` returns an empty map
15543 /// without blocking via `add_and_get`. Issue 24. (The Java analog
15544 /// asserts `never().addAndGet(ListOffsets)`; the Rust analog is
15545 /// that no event is `add_and_get`-ed — the bg path is empty.)
15546 #[tokio::test]
15547 async fn offsets_for_times_with_zero_timeout_maps_every_partition_to_none() {
15548 let (mut consumer, _handles) = make_test_consumer_with_channels();
15549 let tp = TopicPartition::new("topic1".to_string(), 0);
15550 let mut search: HashMap<TopicPartition, i64> = HashMap::new();
15551 search.insert(tp.clone(), 5);
15552
15553 let map = consumer
15554 .offsets_for_times_with_timeout(search, Duration::from_millis(0))
15555 .await
15556 .expect("zero-timeout returns Ok");
15557 // Java: `Collections.singletonMap(tp, null)`.
15558 assert_eq!(map, HashMap::from([(tp, None)]));
15559 }
15560
15561 /// Java: `testOffsetsForTimesFailsOnNegativeTargetTimes`
15562 /// Three asserts: EARLIEST_TIMESTAMP (-2),
15563 /// LATEST_TIMESTAMP (-1), MAX_TIMESTAMP (-3) all reject with
15564 /// IllegalArgument. Issue 24.
15565 #[tokio::test]
15566 async fn offsets_for_times_rejects_negative_target_times() {
15567 let (mut consumer, _handles) = make_test_consumer_with_channels();
15568 let tp = TopicPartition::new("topic1".to_string(), 1);
15569
15570 for negative in &[-2i64, -1, -3] {
15571 let mut search: HashMap<TopicPartition, i64> = HashMap::new();
15572 search.insert(tp.clone(), *negative);
15573 let err = consumer
15574 .offsets_for_times_with_timeout(search, Duration::from_millis(1))
15575 .await
15576 .expect_err("negative target rejected");
15577 assert!(
15578 matches!(err, Error::LocalIllegalArgument(ref m) if m.message().contains("negative")),
15579 "expected IllegalArgument with 'negative', got {err:?}"
15580 );
15581 }
15582 }
15583
15584 /// Java: `testOffsetsForTimesTimeoutException`.
15585 /// Asserts EXACT error message `"Failed to get offsets by times in
15586 /// {timeout}ms"`. Issue 24 / DoD §3.
15587 #[tokio::test]
15588 async fn offsets_for_times_propagates_timeout_with_exact_message() {
15589 let (mut consumer, mut handles) = make_test_consumer_with_channels();
15590 let tp = TopicPartition::new("topic1".to_string(), 1);
15591 let mut search: HashMap<TopicPartition, i64> = HashMap::new();
15592 search.insert(tp, 5);
15593
15594 let drainer = tokio::spawn(async move {
15595 while let Some(env) = handles.app_event_rx.recv().await {
15596 if let ApplicationEvent::ListOffsets { handle, .. } = env.event {
15597 handle.complete_with_error(Error::timeout(
15598 "Event did not complete in time and was expired by the reaper",
15599 ));
15600 return;
15601 }
15602 }
15603 });
15604
15605 let err = consumer
15606 .offsets_for_times_with_timeout(search, Duration::from_millis(100))
15607 .await
15608 .expect_err("must err");
15609 match err {
15610 Error::Timeout(msg) => {
15611 assert_eq!(msg.message(), "Failed to get offsets by times in 100ms");
15612 },
15613 other => panic!("expected Timeout, got {other:?}"),
15614 }
15615 drainer.await.expect("drainer ok");
15616 }
15617
15618 /// Java: `testBeginningOffsetsTimeoutOnEventProcessingTimeout`
15619 /// The `addAndGet`-thrown TimeoutException
15620 /// surfaces from `beginning_offsets(tp, 1ms)` AND the
15621 /// `ListOffsetsEvent` was actually enqueued. Distinct from
15622 /// `testBeginningOffsetsTimeoutException` (which asserts the
15623 /// exact error message): this asserts both the propagation AND
15624 /// the event-enqueue side effect. Issue 24.
15625 #[tokio::test]
15626 async fn beginning_offsets_timeout_on_event_processing_enqueues_event() {
15627 let (mut consumer, mut handles) = make_test_consumer_with_channels();
15628 let tp = TopicPartition::new("t1".to_string(), 0);
15629
15630 let saw_list_offsets = Arc::new(std::sync::atomic::AtomicBool::new(false));
15631 let saw_flag = Arc::clone(&saw_list_offsets);
15632 let drainer = tokio::spawn(async move {
15633 while let Some(env) = handles.app_event_rx.recv().await {
15634 if let ApplicationEvent::ListOffsets { handle, .. } = env.event {
15635 saw_flag.store(true, Ordering::SeqCst);
15636 handle.complete_with_error(Error::timeout("bg-side timeout"));
15637 return;
15638 }
15639 }
15640 });
15641
15642 let err = consumer
15643 .beginning_offsets_with_timeout(&[tp], Duration::from_millis(1))
15644 .await
15645 .expect_err("must err");
15646 assert!(matches!(err, Error::Timeout(_)), "got {err:?}");
15647 drainer.await.expect("drainer ok");
15648 assert!(saw_list_offsets.load(Ordering::SeqCst), "ListOffsets event must be enqueued");
15649 }
15650
15651 /// Java: `testOffsetsForTimes`. Happy-path
15652 /// resolution returns a map of `OffsetAndTimestamp` for each
15653 /// requested partition.
15654 #[tokio::test]
15655 async fn offsets_for_times_returns_event_result() {
15656 let (mut consumer, mut handles) = make_test_consumer_with_channels();
15657 let t0 = TopicPartition::new("t0".to_string(), 2);
15658 let t1 = TopicPartition::new("t0".to_string(), 3);
15659
15660 let drainer = tokio::spawn(async move {
15661 while let Some(env) = handles.app_event_rx.recv().await {
15662 if let ApplicationEvent::ListOffsets { handle, .. } = env.event {
15663 let mut result: HashMap<TopicPartition, Option<OffsetAndTimestampInternal>> = HashMap::new();
15664 result.insert(
15665 TopicPartition::new("t0".to_string(), 2),
15666 Some(OffsetAndTimestampInternal::new(5, 1, None)),
15667 );
15668 result.insert(
15669 TopicPartition::new("t0".to_string(), 3),
15670 Some(OffsetAndTimestampInternal::new(6, 3, None)),
15671 );
15672 handle.complete(result);
15673 return;
15674 }
15675 }
15676 });
15677
15678 let mut ts_search: HashMap<TopicPartition, i64> = HashMap::new();
15679 ts_search.insert(t0.clone(), 1);
15680 ts_search.insert(t1.clone(), 2);
15681 let result = consumer
15682 .offsets_for_times_with_timeout(ts_search, Duration::from_millis(100))
15683 .await
15684 .expect("ok");
15685 assert_eq!(
15686 result.get(&t0).map(|x| x.as_ref().map(OffsetAndTimestamp::offset)),
15687 Some(Some(5))
15688 );
15689 assert_eq!(
15690 result.get(&t1).map(|x| x.as_ref().map(OffsetAndTimestamp::offset)),
15691 Some(Some(6))
15692 );
15693 drainer.await.expect("task ok");
15694 }
15695
15696 /// Java: `testProcessBackgroundEventsWithoutDelay`.
15697 /// When the typed-completion is already ready, the drain helper
15698 /// returns immediately and the timer's remaining_ms equals the
15699 /// initial value. We don't model Java's `Timer.remainingMs()`
15700 /// surface directly; instead we assert the drain helper returns
15701 /// quickly when the receiver is already-completed.
15702 #[tokio::test]
15703 async fn process_background_events_until_returns_immediately_for_ready_receiver() {
15704 let (mut consumer, _handles) = make_test_consumer_with_channels();
15705 // Build a pre-completed receiver.
15706 let (handle, receiver, _erased) =
15707 crate::consumer::internals::events::CompletableEvent::make_completable_event::<()>(i64::MAX);
15708 handle.complete(());
15709
15710 let start = std::time::Instant::now();
15711 let deadline = consumer.time.milliseconds() + 1000;
15712 let result = consumer
15713 .process_background_events_until::<()>(
15714 receiver,
15715 deadline,
15716 |_| false,
15717 "should not see this msg",
15718 /* skip_assignment_events = */ false,
15719 )
15720 .await;
15721 let elapsed = start.elapsed();
15722 result.expect("ready receiver must resolve");
15723 assert!(
15724 elapsed < Duration::from_millis(50),
15725 "ready receiver should resolve immediately, elapsed: {elapsed:?}"
15726 );
15727 }
15728
15729 /// Java: `testProcessBackgroundEventsTimesOut`.
15730 /// A receiver that never completes surfaces as `Timeout`.
15731 #[tokio::test]
15732 async fn process_background_events_until_times_out_for_pending_receiver() {
15733 let (mut consumer, _handles) = make_test_consumer_with_channels();
15734 let (_handle, receiver, _erased) =
15735 crate::consumer::internals::events::CompletableEvent::make_completable_event::<()>(i64::MAX);
15736 // Keep _handle alive — never complete it.
15737
15738 let deadline = consumer.time.milliseconds() + 100;
15739 let err = consumer
15740 .process_background_events_until::<()>(
15741 receiver,
15742 deadline,
15743 |_| false,
15744 "drain helper timeout",
15745 /* skip_assignment_events = */ false,
15746 )
15747 .await
15748 .expect_err("must time out");
15749 assert!(matches!(err, Error::Timeout(_)), "unexpected err: {err:?}");
15750 }
15751
15752 /// Compile-time check: `AsyncKafkaConsumer<K, V>` is `Consumer<K, V>`.
15753 /// Asserts the trait impl is wired correctly.
15754 #[test]
15755 fn consumer_trait_impl_compiles() {
15756 fn _accept_consumer<C: crate::consumer::Consumer<Vec<u8>, Vec<u8>>>(_c: C) {}
15757 // Only the type-level check matters — no runtime assertions.
15758 let _phantom: fn(AsyncKafkaConsumer<Vec<u8>, Vec<u8>>) = _accept_consumer;
15759 }
15760
15761 /// Audit finding C1-22, through the production constructor. Java's
15762 /// `ConsumerNetworkThread.run()` (4.4) wraps every `runOnce()` in
15763 /// `catch (final Throwable e)`, logs "Unexpected error caught in consumer
15764 /// network thread" and keeps looping. The Rust constructor once drove its
15765 /// own copy of the loop without that catch, so one panicking iteration
15766 /// ended the bg thread and every later call failed with "The consumer
15767 /// background task is not running". The `consumer_network_thread` tests
15768 /// start the loop through `spawn_dedicated` directly; this one builds the
15769 /// consumer with `AsyncKafkaConsumer::new` (what `KafkaConsumer::new`
15770 /// calls), so it fails if the constructor stops going through
15771 /// `ConsumerNetworkThread::run`.
15772 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15773 async fn constructed_consumer_survives_a_panicking_background_iteration() {
15774 use std::collections::HashMap;
15775 use std::sync::atomic::AtomicUsize;
15776
15777 use crate::common::serialization::ByteArrayDeserializer;
15778 use crate::consumer::internals::PollResult;
15779 use crate::consumer::internals::RequestManager;
15780
15781 /// Panics on its first poll only, and counts every poll.
15782 struct PanicOnceRequestManager {
15783 poll_calls: Arc<AtomicUsize>,
15784 }
15785
15786 impl RequestManager for PanicOnceRequestManager {
15787 fn poll(&mut self, _current_time_ms: i64) -> PollResult {
15788 if self.poll_calls.fetch_add(1, Ordering::SeqCst) == 0 {
15789 panic!("request manager failure");
15790 }
15791 PollResult::with_time_until_next_poll_ms(10)
15792 }
15793
15794 fn maximum_time_to_wait(&self, _current_time_ms: i64) -> i64 {
15795 10
15796 }
15797 }
15798
15799 // No group.id: `assign` is then served by the bg loop alone, with no
15800 // broker round trip. The bootstrap address refuses connections.
15801 let props = HashMap::from([
15802 ("bootstrap.servers".to_string(), "127.0.0.1:1".to_string()),
15803 ("group.protocol".to_string(), "consumer".to_string()),
15804 ]);
15805 let mut consumer = AsyncKafkaConsumer::<Vec<u8>, Vec<u8>>::new(
15806 ConsumerConfig::new(&props).expect("config validates"),
15807 Box::new(ByteArrayDeserializer),
15808 Box::new(ByteArrayDeserializer),
15809 )
15810 .expect("ctor should succeed against a refused broker");
15811 let poll_calls = Arc::new(AtomicUsize::new(0));
15812 consumer
15813 .request_managers
15814 .lock()
15815 .unwrap()
15816 .add_dyn_manager_for_test(Box::new(PanicOnceRequestManager { poll_calls: Arc::clone(&poll_calls) }));
15817
15818 // Wait for the loop to run the panicking iteration and at least one
15819 // more.
15820 let start = std::time::Instant::now();
15821 while poll_calls.load(Ordering::SeqCst) < 2 && start.elapsed() < Duration::from_secs(5) {
15822 tokio::time::sleep(Duration::from_millis(10)).await;
15823 }
15824 let polls = poll_calls.load(Ordering::SeqCst);
15825
15826 let assigned = consumer.assign(vec![TopicPartition::new("topic1".to_string(), 0)]).await;
15827 assert!(
15828 polls >= 2 && assigned.is_ok(),
15829 "Java: the network thread logs the error and keeps running, so later calls still succeed; Rust: \
15830 manager polls={polls}, assign after the panicking iteration returned {assigned:?}"
15831 );
15832 consumer
15833 .close_with_options(crate::consumer::CloseOptions::new_with_timeout(Duration::ZERO))
15834 .await
15835 .expect("close");
15836 }
15837
15838 // ── Phase 21: dedicated-IO-thread `NetworkThreadCloseHandle` tests ──
15839 //
15840 // The production `AsyncKafkaConsumer::new()` runs the bg loop on a
15841 // dedicated `std::thread` hosting a `current_thread` runtime and
15842 // builds a `NetworkThreadCloseHandle::with_dedicated(...)`. These
15843 // tests model that exact construction (running flag → bg loop →
15844 // wakeup → `done` oneshot → OS thread reap) without needing a broker,
15845 // and assert the close path joins the dedicated thread cleanly within
15846 // a bounded timeout (no hang), preserving the `Spawned`-path
15847 // semantics (clean exit + panic mapping).
15848
15849 /// Builds a dedicated bg thread that mirrors the production loop:
15850 /// a `current_thread` runtime spinning a `run_once`-style loop until
15851 /// the running flag flips, firing `done` on exit. Returns the close
15852 /// handle plus the running flag and a wakeup `Notify` the loop waits
15853 /// on (so the test can prove `signal_close` + `wakeup` terminate it).
15854 fn spawn_dedicated_bg() -> (NetworkThreadCloseHandle, Arc<AtomicBool>, Arc<tokio::sync::Notify>) {
15855 let running = Arc::new(AtomicBool::new(true));
15856 let wake = Arc::new(tokio::sync::Notify::new());
15857
15858 let loop_running = Arc::clone(&running);
15859 let loop_wake = Arc::clone(&wake);
15860 let (done_tx, done_rx) = tokio::sync::oneshot::channel::<()>();
15861 let thread_handle = std::thread::Builder::new()
15862 .name("kafka-consumer-io-test".into())
15863 .spawn(move || {
15864 let rt = tokio::runtime::Builder::new_current_thread()
15865 .enable_all()
15866 .build()
15867 .expect("build test io runtime");
15868 rt.block_on(async move {
15869 // Mirror the production `while is_running() { run_once().await }`
15870 // loop: block on the wakeup `Notify` each iteration so
15871 // the loop only proceeds when woken (as the real
15872 // selector poll returns on `Selector::wakeup`).
15873 while loop_running.load(Ordering::Acquire) {
15874 loop_wake.notified().await;
15875 }
15876 // Stand-in for `cleanup().await`.
15877 });
15878 let _ = done_tx.send(());
15879 })
15880 .expect("spawn test io thread");
15881
15882 let close_running = Arc::clone(&running);
15883 let close_wake = Arc::clone(&wake);
15884 let signal_close_fn: SignalCloseFn = Box::new(move |_timeout: Duration| {
15885 close_running.store(false, Ordering::Release);
15886 close_wake.notify_one();
15887 });
15888 let wakeup_wake = Arc::clone(&wake);
15889 let wakeup_fn: Box<dyn Fn() + Send + Sync> = Box::new(move || {
15890 wakeup_wake.notify_one();
15891 });
15892
15893 let handle = NetworkThreadCloseHandle::with_dedicated(signal_close_fn, wakeup_fn, done_rx, thread_handle);
15894 (handle, running, wake)
15895 }
15896
15897 /// `signal_close()` + `wakeup()` terminate the dedicated bg loop and
15898 /// `await_join()` cleanly reaps the OS thread within a bounded
15899 /// timeout (no hang).
15900 #[tokio::test]
15901 async fn dedicated_close_handle_joins_cleanly() {
15902 let (mut handle, running, _wake) = spawn_dedicated_bg();
15903
15904 handle.signal_close(Duration::from_secs(30));
15905 handle.wakeup();
15906
15907 let result = tokio::time::timeout(Duration::from_secs(5), handle.await_join())
15908 .await
15909 .expect("await_join must not hang");
15910 assert!(result.is_ok(), "clean dedicated-thread join, got {result:?}");
15911 assert!(!running.load(Ordering::Acquire), "running flag must be cleared");
15912 }
15913
15914 /// `await_join()` on a `Dedicated` handle is idempotent: a second
15915 /// call after a successful join is a no-op `Ok(())`, not a hang.
15916 #[tokio::test]
15917 async fn dedicated_await_join_is_idempotent() {
15918 let (mut handle, _running, _wake) = spawn_dedicated_bg();
15919
15920 handle.signal_close(Duration::from_secs(30));
15921 handle.wakeup();
15922
15923 tokio::time::timeout(Duration::from_secs(5), handle.await_join())
15924 .await
15925 .expect("first await_join must not hang")
15926 .expect("first await_join clean");
15927
15928 // Second call: receiver + thread already taken → immediate Ok.
15929 let second = tokio::time::timeout(Duration::from_secs(1), handle.await_join())
15930 .await
15931 .expect("second await_join must not hang");
15932 assert!(second.is_ok(), "idempotent second join, got {second:?}");
15933 }
15934
15935 /// A panic inside the dedicated bg thread is mapped to the SAME
15936 /// `Error::local_illegal_state("Consumer network thread terminated
15937 /// with error: ...")` shape as the `Spawned` JoinError path.
15938 #[tokio::test]
15939 async fn dedicated_thread_panic_maps_to_illegal_state() {
15940 let (done_tx, done_rx) = tokio::sync::oneshot::channel::<()>();
15941 let thread_handle = std::thread::Builder::new()
15942 .name("kafka-consumer-io-test-panic".into())
15943 .spawn(move || {
15944 // Hold `done_tx` so it is dropped (not sent) on panic —
15945 // `await_join` must treat the closed receiver as "loop
15946 // exited" and still reap the panicking thread.
15947 let _done_tx = done_tx;
15948 panic!("boom");
15949 })
15950 .expect("spawn panicking test io thread");
15951
15952 let mut handle =
15953 NetworkThreadCloseHandle::with_dedicated(Box::new(|_| {}), Box::new(|| {}), done_rx, thread_handle);
15954
15955 let result = tokio::time::timeout(Duration::from_secs(5), handle.await_join())
15956 .await
15957 .expect("await_join must not hang on panic");
15958 match result {
15959 Err(Error::LocalIllegalState(msg)) => {
15960 assert!(
15961 msg.message().contains("Consumer network thread terminated with error"),
15962 "unexpected message: {msg}"
15963 );
15964 },
15965 other => panic!("expected IllegalState on thread panic, got {other:?}"),
15966 }
15967 }
15968
15969 /// Java's `initializeGroupMetadata` rejects a present-but-EMPTY `group.id`
15970 /// before building anything
15971 /// (`AsyncKafkaConsumer#initializeGroupMetadata(String,Optional)`), and the constructor's
15972 /// `catch (Throwable t)` then wraps it.
15973 ///
15974 /// Accepting it leaves the consumer internally inconsistent: `Some("")` is
15975 /// "in a group" for the coordinator / commit / heartbeat / membership
15976 /// wiring but "not in a group" for `return_error_if_group_id_not_defined`, so
15977 /// `FindCoordinator` and `ConsumerGroupHeartbeat` go on the wire with an
15978 /// empty group id while `commit_sync()` reports `InvalidGroupId`.
15979 #[tokio::test(flavor = "multi_thread")]
15980 async fn empty_group_id_is_rejected_at_construction() {
15981 use std::collections::HashMap;
15982
15983 use crate::common::serialization::Deserializer;
15984
15985 struct TestStringDeserializer;
15986 impl Deserializer<String> for TestStringDeserializer {
15987 async fn deserialize(&self, _topic: &str, data: &[u8]) -> Result<String, Error> {
15988 String::from_utf8(data.to_vec()).map_err(|e| Error::serialization(format!("invalid utf-8: {}", e)))
15989 }
15990 }
15991
15992 // A whitespace-only id trims to "" at parse (`ConfigDef.parseType`),
15993 // so it takes the same rejection.
15994 for group_id in ["", " "] {
15995 let props = HashMap::from([
15996 ("bootstrap.servers".to_string(), "127.0.0.1:1".to_string()),
15997 ("group.id".to_string(), group_id.to_string()),
15998 ]);
15999 let config = ConsumerConfig::new(&props).expect("config itself validates");
16000
16001 let err = AsyncKafkaConsumer::<String, String>::new(
16002 config,
16003 Box::new(TestStringDeserializer),
16004 Box::new(TestStringDeserializer),
16005 )
16006 .err()
16007 .unwrap_or_else(|| panic!("group.id {group_id:?} must fail construction"));
16008
16009 // Wrapped as Java wraps every constructor failure.
16010 assert_eq!("Failed to construct kafka consumer", err.message());
16011 let cause = err.source().expect("the InvalidGroupId is the cause");
16012 assert_eq!(cause.error(), crate::common::protocol::Errors::InvalidGroupId);
16013 assert_eq!(
16014 "The configured group.id should not be an empty string or whitespace.",
16015 cause.message()
16016 );
16017 }
16018 }
16019
16020 /// A `group.id` that is absent, or non-empty, still constructs — so the
16021 /// check above cannot be an unconditional rejection.
16022 #[tokio::test(flavor = "multi_thread")]
16023 async fn non_empty_and_absent_group_id_still_construct() {
16024 use std::collections::HashMap;
16025
16026 use crate::common::serialization::Deserializer;
16027
16028 struct TestStringDeserializer;
16029 impl Deserializer<String> for TestStringDeserializer {
16030 async fn deserialize(&self, _topic: &str, data: &[u8]) -> Result<String, Error> {
16031 String::from_utf8(data.to_vec()).map_err(|e| Error::serialization(format!("invalid utf-8: {}", e)))
16032 }
16033 }
16034
16035 for group_id in [None, Some("a-group")] {
16036 let mut props = HashMap::from([("bootstrap.servers".to_string(), "127.0.0.1:1".to_string())]);
16037 if let Some(g) = group_id {
16038 props.insert("group.id".to_string(), g.to_string());
16039 }
16040 let config = ConsumerConfig::new(&props).expect("config validates");
16041 let mut consumer = AsyncKafkaConsumer::<String, String>::new(
16042 config,
16043 Box::new(TestStringDeserializer),
16044 Box::new(TestStringDeserializer),
16045 )
16046 .unwrap_or_else(|e| panic!("group_id={group_id:?} must construct, got {e}"));
16047 let _ = consumer.close().await;
16048 }
16049 }
16050
16051 /// Java's `close` wraps whatever the close steps recorded:
16052 ///
16053 /// ```java
16054 /// throw new KafkaException("Failed to close kafka consumer", exception);
16055 /// ```
16056 /// (`AsyncKafkaConsumer#close(Duration,GroupMembershipOperation,boolean)`).
16057 ///
16058 /// This is what makes `catch (KafkaException e)` around `close()` — the
16059 /// canonical Java idiom — reliable. Returning the recorded error raw breaks
16060 /// it whenever that error is outside the `KafkaException` hierarchy, which
16061 /// the flat `Error` enum makes reachable (an `LocalIllegalState` from the close
16062 /// path answers `false` to `is_kafka_error()`).
16063 #[tokio::test]
16064 async fn close_wraps_the_first_error_as_failed_to_close_kafka_consumer() {
16065 let (mut consumer, mut handles) = make_test_consumer_with_channels();
16066 // Fail the leave-group step with an error that is NOT a
16067 // `KafkaException` in Java terms, so the wrap is observable in the
16068 // hierarchy answer and not only in the message.
16069 let drainer = tokio::spawn(async move {
16070 while let Some(env) = handles.app_event_rx.recv().await {
16071 match env.event {
16072 ApplicationEvent::LeaveGroupOnClose { handle, .. } => {
16073 handle.complete_with_error(Error::local_illegal_state(
16074 "Consumer background task is no longer running.",
16075 ));
16076 },
16077 ApplicationEvent::CommitSync { handle, offsets_ready, .. } => {
16078 offsets_ready.complete(());
16079 handle.complete(HashMap::new());
16080 },
16081 ApplicationEvent::CommitAsync { handle, offsets_ready, .. } => {
16082 offsets_ready.complete(());
16083 handle.complete(HashMap::new());
16084 },
16085 _ => {},
16086 }
16087 }
16088 });
16089
16090 let err = consumer.close().await.expect_err("the failed step must surface");
16091 drop(drainer);
16092
16093 assert_eq!("Failed to close kafka consumer", err.message());
16094 assert!(err.is_kafka_error(), "Java guarantees the caller a KafkaException: {err:?}");
16095 let cause = err.source().expect("the recorded error is the cause");
16096 assert!(
16097 matches!(cause, Error::LocalIllegalState(_)),
16098 "the original error must be the cause, got {cause:?}"
16099 );
16100 assert_eq!("Consumer background task is no longer running.", cause.message());
16101 // The consumer is still marked closed — the wrap happens after the
16102 // state flip, as in Java.
16103 assert!(consumer.is_closed());
16104 }
16105
16106 // ─── An OffsetCommitCallback returning an error (Java's onComplete throwing) ───
16107 //
16108 // Java's `OffsetCommitCallbackInvoker.executeCallbacks()` has no
16109 // try/catch, so an exception `onComplete` throws escapes the call that
16110 // runs the queued callbacks: `poll` (wrapped by
16111 // `maybeWrapAsKafkaException`, the inflight poll cleared), `commitSync` /
16112 // `commitAsync` (through `commit()` and, for `commitSync`,
16113 // `awaitPendingAsyncCommitsAndExecuteCommitCallbacks`) and `close` (as
16114 // the first error, wrapped in "Failed to close kafka consumer" — unless
16115 // the auto-commit on close ran the callback, which swallows it).
16116
16117 /// Returns `Err(LocalIllegalState(message))` from `on_complete`, counting
16118 /// its invocations.
16119 struct RaisingCommitCallback {
16120 message: &'static str,
16121 invoked: Arc<std::sync::atomic::AtomicUsize>,
16122 }
16123
16124 impl RaisingCommitCallback {
16125 fn new(message: &'static str) -> Arc<Self> {
16126 Arc::new(Self { message, invoked: Arc::new(std::sync::atomic::AtomicUsize::new(0)) })
16127 }
16128 }
16129
16130 #[async_trait::async_trait]
16131 impl crate::consumer::OffsetCommitCallback for RaisingCommitCallback {
16132 async fn on_complete(
16133 &self,
16134 _offsets: &HashMap<TopicPartition, OffsetAndMetadata>,
16135 _error: Option<&Error>,
16136 ) -> Result<(), Error> {
16137 self.invoked.fetch_add(1, Ordering::SeqCst);
16138 Err(Error::local_illegal_state(self.message))
16139 }
16140 }
16141
16142 /// Java's poll: `catch (Throwable t) { inflightPoll = null; throw
16143 /// ConsumerUtils.maybeWrapAsKafkaException(t); }`. The callbacks behind
16144 /// the failing one stay queued and run at the next poll.
16145 #[tokio::test]
16146 async fn poll_propagates_a_commit_callback_error_and_clears_the_inflight_poll() {
16147 let (mut consumer, _handles) = make_test_consumer_with_channels();
16148 let tp = TopicPartition::new("t".to_string(), 0);
16149 consumer
16150 .subscriptions
16151 .lock()
16152 .unwrap()
16153 .assign_from_user(HashSet::from([tp.clone()]))
16154 .unwrap();
16155 let raising = RaisingCommitCallback::new("callback failed");
16156 let next = RaisingCommitCallback::new("next callback failed");
16157 consumer.offset_commit_callback_invoker.enqueue_user_callback_invocation(
16158 raising.clone(),
16159 singleton_offsets(tp.clone(), 1),
16160 None,
16161 );
16162 consumer.offset_commit_callback_invoker.enqueue_user_callback_invocation(
16163 next.clone(),
16164 singleton_offsets(tp, 2),
16165 None,
16166 );
16167
16168 let err = consumer.poll(Duration::from_millis(0)).await.expect_err("propagates");
16169 // `maybeWrapAsKafkaException`: a non-Kafka error is wrapped, its
16170 // `toString()` the message and the original the cause.
16171 assert!(err.is_kafka_error(), "poll hands the caller a KafkaException: {err:?}");
16172 assert_eq!(err.message(), Error::local_illegal_state("callback failed").to_string());
16173 let cause = err.source().expect("the callback's error is the cause");
16174 assert!(matches!(cause, Error::LocalIllegalState(_)), "unexpected cause: {cause:?}");
16175 assert_eq!(cause.message(), "callback failed");
16176 assert!(consumer.inflight_poll.is_none(), "the inflight poll is cleared");
16177 assert_eq!(raising.invoked.load(Ordering::SeqCst), 1);
16178 assert_eq!(next.invoked.load(Ordering::SeqCst), 0, "the next callback stays queued");
16179
16180 let err = consumer.poll(Duration::from_millis(0)).await.expect_err("the next one runs");
16181 assert_eq!(err.source().expect("cause").message(), "next callback failed");
16182 assert_eq!(raising.invoked.load(Ordering::SeqCst), 1);
16183 assert_eq!(next.invoked.load(Ordering::SeqCst), 1);
16184 }
16185
16186 /// A commit callback that calls `wakeup()` after `delay`, as a user's
16187 /// callback may.
16188 struct WakingCommitCallback {
16189 wakeup_trigger: WakeupTrigger,
16190 delay: Duration,
16191 }
16192
16193 #[async_trait::async_trait]
16194 impl crate::consumer::OffsetCommitCallback for WakingCommitCallback {
16195 async fn on_complete(
16196 &self,
16197 _offsets: &HashMap<TopicPartition, OffsetAndMetadata>,
16198 _error: Option<&Error>,
16199 ) -> Result<(), Error> {
16200 if !self.delay.is_zero() {
16201 tokio::time::sleep(self.delay).await;
16202 }
16203 self.wakeup_trigger.wakeup();
16204 Ok(())
16205 }
16206 }
16207
16208 /// Java's `testWakeupAfterEmptyFetch`, for a poll with no time left when
16209 /// the wakeup arrives: a `wakeup()` sent while `checkInflightPoll` runs the
16210 /// commit callbacks is raised by that same poll, because `pollForFetches`
16211 /// calls `wakeupTrigger.setFetchAction(fetchBuffer)` after an empty
16212 /// `collectFetch()` whatever the remaining time
16213 /// (`AsyncKafkaConsumer#pollForFetches`). The next poll does not raise.
16214 ///
16215 /// Two ways of having no time left: `poll(0)`, and a poll whose deadline
16216 /// passes while the callback runs.
16217 #[tokio::test]
16218 async fn poll_raises_a_wakeup_sent_by_its_callbacks_when_no_time_is_left() {
16219 for (timeout, delay) in [
16220 (Duration::ZERO, Duration::ZERO),
16221 (Duration::from_millis(50), Duration::from_millis(150)),
16222 ] {
16223 let (mut consumer, _handles) = make_test_consumer_with_channels();
16224 let tp = TopicPartition::new("t".to_string(), 0);
16225 consumer
16226 .subscriptions
16227 .lock()
16228 .unwrap()
16229 .assign_from_user(HashSet::from([tp.clone()]))
16230 .unwrap();
16231 consumer.offset_commit_callback_invoker.enqueue_user_callback_invocation(
16232 Arc::new(WakingCommitCallback { wakeup_trigger: consumer.wakeup_trigger.clone(), delay }),
16233 singleton_offsets(tp, 1),
16234 None,
16235 );
16236
16237 let err = consumer
16238 .poll(timeout)
16239 .await
16240 .expect_err("the wakeup is raised by the poll whose callback sent it");
16241 assert!(matches!(err, Error::Wakeup(_)), "unexpected err for {timeout:?}: {err:?}");
16242 assert!(
16243 !consumer.wakeup_trigger.current_token().is_cancelled(),
16244 "the token is rotated after raising"
16245 );
16246 consumer
16247 .poll(Duration::ZERO)
16248 .await
16249 .expect("the wakeup was consumed: the next poll does not raise");
16250 }
16251 }
16252
16253 /// Java's `commit()` runs `executeCallbacks()` before adding the event: a
16254 /// queued callback's exception escapes `commitSync` / `commitAsync`
16255 /// unwrapped, and the commit is not submitted. A handle's commit runs the
16256 /// same `commit()`.
16257 #[tokio::test]
16258 async fn commit_propagates_a_queued_commit_callback_error_before_submitting() {
16259 let (mut consumer, mut handles) = make_test_consumer_with_channels();
16260 let tp = TopicPartition::new("t".to_string(), 0);
16261 let handle = consumer.handle();
16262 let _in_callback = consumer.enter_callback_scope();
16263 for raising in [
16264 RaisingCommitCallback::new("sync"),
16265 RaisingCommitCallback::new("async"),
16266 RaisingCommitCallback::new("handle"),
16267 ] {
16268 consumer.offset_commit_callback_invoker.enqueue_user_callback_invocation(
16269 raising.clone(),
16270 singleton_offsets(tp.clone(), 1),
16271 None,
16272 );
16273 let err = if raising.message == "sync" {
16274 consumer.commit_sync_with_offsets(singleton_offsets(tp.clone(), 5)).await
16275 } else if raising.message == "handle" {
16276 // A callback committing through the handle runs the same
16277 // `commit()`.
16278 handle.commit_sync_with_offsets(singleton_offsets(tp.clone(), 5)).await
16279 } else {
16280 consumer
16281 .commit_async_with_offsets_callback(
16282 singleton_offsets(tp.clone(), 5),
16283 RaisingCommitCallback::new("unused"),
16284 )
16285 .await
16286 }
16287 .expect_err("propagates");
16288 assert!(
16289 matches!(err, Error::LocalIllegalState(_)),
16290 "unwrapped, as Java's commit(): {err:?}"
16291 );
16292 assert_eq!(err.message(), raising.message);
16293 assert_eq!(raising.invoked.load(Ordering::SeqCst), 1);
16294 assert!(handles.app_event_rx.try_recv().is_err(), "the commit is not submitted");
16295 }
16296 }
16297
16298 /// Java's `commitSync` runs `awaitPendingAsyncCommitsAndExecuteCommitCallbacks`
16299 /// after submitting its own commit: the callback of an async commit that
16300 /// completes meanwhile runs there, and its exception escapes `commitSync`.
16301 #[tokio::test]
16302 async fn commit_sync_propagates_the_error_of_a_pending_async_commit_callback() {
16303 let (mut consumer, mut handles) = make_test_consumer_with_channels();
16304 let tp = TopicPartition::new("t".to_string(), 0);
16305 let completer = tokio::spawn(async move {
16306 let mut pending_async = None;
16307 while let Some(env) = handles.app_event_rx.recv().await {
16308 match env.event {
16309 ApplicationEvent::CommitAsync { handle, offsets_ready, offsets } => {
16310 offsets_ready.complete(());
16311 pending_async = Some((handle, offsets.unwrap_or_default()));
16312 },
16313 ApplicationEvent::CommitSync { handle, offsets_ready, offsets } => {
16314 offsets_ready.complete(());
16315 // The async commit completes once the sync commit
16316 // submitted, so its callback is queued after
16317 // `commit()` ran the queue.
16318 let (async_handle, async_offsets) = pending_async.take().expect("async commit first");
16319 async_handle.complete(async_offsets);
16320 handle.complete(offsets.unwrap_or_default());
16321 return;
16322 },
16323 _ => {},
16324 }
16325 }
16326 });
16327
16328 let raising = RaisingCommitCallback::new("pending callback failed");
16329 consumer
16330 .commit_async_with_offsets_callback(singleton_offsets(tp.clone(), 1), raising.clone())
16331 .await
16332 .expect("async commit submitted");
16333 let err = consumer
16334 .commit_sync_with_offsets(singleton_offsets(tp, 2))
16335 .await
16336 .expect_err("propagates");
16337 assert!(matches!(err, Error::LocalIllegalState(_)), "unexpected error: {err:?}");
16338 assert_eq!(err.message(), "pending callback failed");
16339 assert_eq!(raising.invoked.load(Ordering::SeqCst), 1);
16340 completer.await.expect("completer");
16341 }
16342
16343 /// Drains the close-path events, completing each successfully. The
16344 /// caller keeps the rest of its `ConsumerTestHandles` (the background
16345 /// event sender) alive across the close.
16346 fn spawn_close_drainer(
16347 mut app_event_rx: mpsc::UnboundedReceiver<ApplicationEventEnvelope>,
16348 ) -> tokio::task::JoinHandle<()> {
16349 tokio::spawn(async move {
16350 while let Some(env) = app_event_rx.recv().await {
16351 match env.event {
16352 ApplicationEvent::LeaveGroupOnClose { handle, .. } => {
16353 handle.complete(());
16354 },
16355 ApplicationEvent::CommitSync { handle, offsets_ready, .. }
16356 | ApplicationEvent::CommitAsync { handle, offsets_ready, .. } => {
16357 offsets_ready.complete(());
16358 handle.complete(HashMap::new());
16359 },
16360 _ => {},
16361 }
16362 }
16363 })
16364 }
16365
16366 /// Java's close records the error of
16367 /// `awaitPendingAsyncCommitsAndExecuteCommitCallbacks` (logged at ERROR)
16368 /// and throws `KafkaException("Failed to close kafka consumer", it)`.
16369 #[tokio::test]
16370 async fn close_wraps_a_commit_callback_error() {
16371 let (mut consumer, handles) = make_test_consumer_with_channels();
16372 consumer.auto_commit_enabled = false;
16373 let raising = RaisingCommitCallback::new("callback failed on close");
16374 consumer.offset_commit_callback_invoker.enqueue_user_callback_invocation(
16375 raising.clone(),
16376 singleton_offsets(TopicPartition::new("t".to_string(), 0), 1),
16377 None,
16378 );
16379 let drainer = spawn_close_drainer(handles.app_event_rx);
16380
16381 let err = consumer.close().await.expect_err("the callback error surfaces");
16382 drainer.abort();
16383 assert_eq!(err.message(), "Failed to close kafka consumer");
16384 assert!(err.is_kafka_error());
16385 let cause = err.source().expect("the callback's error is the cause");
16386 assert!(matches!(cause, Error::LocalIllegalState(_)), "unexpected cause: {cause:?}");
16387 assert_eq!(cause.message(), "callback failed on close");
16388 assert_eq!(raising.invoked.load(Ordering::SeqCst), 1);
16389 assert!(consumer.is_closed());
16390 }
16391
16392 /// With auto-commit on, Java's close first runs `commitSyncAllConsumed`,
16393 /// whose `commit()` runs the queued callback and whose `catch (Exception
16394 /// e)` only logs "Synchronous auto-commit failed": close succeeds.
16395 #[tokio::test]
16396 async fn close_swallows_a_commit_callback_error_raised_by_the_auto_commit() {
16397 let (mut consumer, handles) = make_test_consumer_with_channels();
16398 consumer.auto_commit_enabled = true;
16399 let raising = RaisingCommitCallback::new("callback failed in auto-commit");
16400 consumer.offset_commit_callback_invoker.enqueue_user_callback_invocation(
16401 raising.clone(),
16402 singleton_offsets(TopicPartition::new("t".to_string(), 0), 1),
16403 None,
16404 );
16405 let drainer = spawn_close_drainer(handles.app_event_rx);
16406
16407 consumer.close().await.expect("the auto-commit swallows it");
16408 drainer.abort();
16409 assert_eq!(raising.invoked.load(Ordering::SeqCst), 1);
16410 }
16411
16412 // ─── ConsumerHandle commits (Java callbacks calling back into the consumer) ───
16413 //
16414 // A Java callback commits by calling `consumer.commitSync()` /
16415 // `commitAsync()` on the consumer it captured, which runs the consumer's
16416 // own commit path. A Rust callback commits through a `ConsumerHandle`;
16417 // these tests translate the Java cases with the commit made through it.
16418
16419 /// The commit a [`NestingCallback`] makes through the handle.
16420 type NestedCommit = (
16421 ConsumerHandle,
16422 HashMap<TopicPartition, OffsetAndMetadata>,
16423 Arc<dyn crate::consumer::OffsetCommitCallback>,
16424 );
16425
16426 /// Records `label` (and `label-end` after its nested commit) into `order`;
16427 /// with `nested`, commits through the handle from inside the callback.
16428 struct NestingCallback {
16429 label: &'static str,
16430 order: Arc<std::sync::Mutex<Vec<String>>>,
16431 nested: Option<NestedCommit>,
16432 errors: Arc<std::sync::Mutex<Vec<Option<String>>>>,
16433 }
16434
16435 #[async_trait::async_trait]
16436 impl crate::consumer::OffsetCommitCallback for NestingCallback {
16437 async fn on_complete(
16438 &self,
16439 _offsets: &HashMap<TopicPartition, OffsetAndMetadata>,
16440 error: Option<&Error>,
16441 ) -> Result<(), Error> {
16442 self.errors.lock().unwrap().push(error.map(|e| e.to_string()));
16443 self.order.lock().unwrap().push(self.label.to_string());
16444 if let Some((handle, offsets, callback)) = &self.nested {
16445 handle
16446 .commit_async_with_offsets_callback(offsets.clone(), Arc::clone(callback))
16447 .await
16448 .expect("nested commit_async");
16449 self.order.lock().unwrap().push(format!("{}-end", self.label));
16450 }
16451 Ok(())
16452 }
16453 }
16454
16455 fn nesting_callback(
16456 label: &'static str,
16457 order: &Arc<std::sync::Mutex<Vec<String>>>,
16458 nested: Option<NestedCommit>,
16459 ) -> Arc<dyn crate::consumer::OffsetCommitCallback> {
16460 Arc::new(NestingCallback {
16461 label,
16462 order: Arc::clone(order),
16463 nested,
16464 errors: Arc::new(std::sync::Mutex::new(Vec::new())),
16465 })
16466 }
16467
16468 /// Stands in for the background task: completes every commit event at
16469 /// once with its explicit offsets.
16470 fn spawn_commit_completer(mut app_event_rx: mpsc::UnboundedReceiver<ApplicationEventEnvelope>) -> JoinHandle<()> {
16471 tokio::spawn(async move {
16472 while let Some(env) = app_event_rx.recv().await {
16473 match env.event {
16474 ApplicationEvent::CommitAsync { handle, offsets_ready, offsets }
16475 | ApplicationEvent::CommitSync { handle, offsets_ready, offsets } => {
16476 offsets_ready.complete(());
16477 handle.complete(offsets.unwrap_or_default());
16478 },
16479 ApplicationEvent::LeaveGroupOnClose { handle, .. } => {
16480 handle.complete(());
16481 },
16482 _ => {},
16483 }
16484 }
16485 })
16486 }
16487
16488 async fn wait_last_pending_async_commit(consumer: &AsyncKafkaConsumer<Vec<u8>, Vec<u8>>) {
16489 if let Some(mut pending) = consumer.commit_state.last_pending_async_commit() {
16490 pending.wait().await;
16491 }
16492 }
16493
16494 /// Java's `commit()` runs `offsetCommitCallbackInvoker.executeCallbacks()`
16495 /// first (`AsyncKafkaConsumer#commit`), so a callback calling
16496 /// `commitAsync` runs the callbacks queued behind it before that call
16497 /// returns. Through the handle: callback "a" commits from inside, and "b",
16498 /// queued after "a", runs before the nested commit returns.
16499 #[tokio::test]
16500 async fn handle_commit_async_inside_callback_runs_later_queued_callbacks_first() {
16501 let (mut consumer, handles) = make_test_consumer_with_channels();
16502 let completer = spawn_commit_completer(handles.app_event_rx);
16503 let tp = TopicPartition::new("foo".to_string(), 0);
16504 let order = Arc::new(std::sync::Mutex::new(Vec::new()));
16505
16506 let nested = (
16507 consumer.handle(),
16508 singleton_offsets(tp.clone(), 3),
16509 nesting_callback("c", &order, None),
16510 );
16511 let a = nesting_callback("a", &order, Some(nested));
16512 consumer
16513 .commit_async_with_offsets_callback(singleton_offsets(tp.clone(), 1), a)
16514 .await
16515 .expect("commit a");
16516 consumer
16517 .commit_async_with_offsets_callback(singleton_offsets(tp.clone(), 2), nesting_callback("b", &order, None))
16518 .await
16519 .expect("commit b");
16520 wait_last_pending_async_commit(&consumer).await;
16521
16522 // Java's `forceCommitCallbackInvocation()`.
16523 consumer
16524 .offset_commit_callback_invoker
16525 .invoke_pending_callbacks()
16526 .await
16527 .unwrap();
16528 assert_eq!(*order.lock().unwrap(), vec!["a", "b", "a-end"]);
16529
16530 // The nested commit is the consumer's last pending async commit, so
16531 // the consumer's next drain runs its callback.
16532 wait_last_pending_async_commit(&consumer).await;
16533 consumer
16534 .offset_commit_callback_invoker
16535 .invoke_pending_callbacks()
16536 .await
16537 .unwrap();
16538 assert_eq!(*order.lock().unwrap(), vec!["a", "b", "a-end", "c"]);
16539 completer.abort();
16540 }
16541
16542 /// Java: `testCommitSyncAwaitsCommitAsyncCompletionWithNonEmptyOffsets`
16543 /// with the async commit made through a handle: the consumer's
16544 /// `commit_sync` waits for it.
16545 #[tokio::test]
16546 async fn commit_sync_awaits_handle_commit_async_completion() {
16547 let tp = TopicPartition::new("foo".to_string(), 0);
16548 let IncompleteAsyncCommit { mut consumer, async_commit, _bg_event_tx } =
16549 set_up_consumer_with_incomplete_async_commit_via(&tp, None, true).await;
16550
16551 let err = consumer
16552 .commit_sync_with_offsets_timeout(singleton_offsets(tp.clone(), 20), Duration::from_millis(100))
16553 .await
16554 .expect_err("commit sync must time out");
16555 assert_pending_async_commit_timeout(&err);
16556
16557 async_commit.complete(HashMap::new());
16558
16559 consumer
16560 .commit_sync_with_offsets_timeout(singleton_offsets(tp, 20), Duration::from_millis(100))
16561 .await
16562 .expect("commit sync must succeed");
16563 }
16564
16565 /// Java: `testCommitSyncAwaitsCommitAsyncCompletionWithNonEmptyOffsets`
16566 /// with the sync commit made through a handle: the handle's
16567 /// `commit_sync` waits for the consumer's async commit, as `commitSync`
16568 /// does (`awaitPendingAsyncCommitsAndExecuteCommitCallbacks`).
16569 #[tokio::test]
16570 async fn handle_commit_sync_awaits_commit_async_completion() {
16571 let tp = TopicPartition::new("foo".to_string(), 0);
16572 let IncompleteAsyncCommit { mut consumer, async_commit, _bg_event_tx } =
16573 set_up_consumer_with_incomplete_async_commit(&tp, None).await;
16574 // The handle's commit_sync is bounded by `default.api.timeout.ms`.
16575 consumer.default_api_timeout_ms = 100;
16576 let handle = consumer.handle();
16577 let _in_callback = consumer.enter_callback_scope();
16578
16579 let err = handle
16580 .commit_sync_with_offsets(singleton_offsets(tp.clone(), 20))
16581 .await
16582 .expect_err("commit sync must time out");
16583 assert_pending_async_commit_timeout(&err);
16584
16585 async_commit.complete(HashMap::new());
16586
16587 handle
16588 .commit_sync_with_offsets(singleton_offsets(tp, 20))
16589 .await
16590 .expect("commit sync must succeed");
16591 }
16592
16593 /// Java: `testCommitSyncAwaitsCommitAsyncButDoesNotFail` through a handle,
16594 /// both commits made through it: a failed async commit is reported to its
16595 /// callback, and the handle's `commit_sync` runs that callback.
16596 #[tokio::test]
16597 async fn handle_commit_sync_awaits_handle_commit_async_but_does_not_fail() {
16598 let tp = TopicPartition::new("foo".to_string(), 0);
16599 let order = Arc::new(std::sync::Mutex::new(Vec::new()));
16600 let errors = Arc::new(std::sync::Mutex::new(Vec::new()));
16601 let callback: Arc<dyn crate::consumer::OffsetCommitCallback> = Arc::new(NestingCallback {
16602 label: "async",
16603 order: Arc::clone(&order),
16604 nested: None,
16605 errors: Arc::clone(&errors),
16606 });
16607 let IncompleteAsyncCommit { mut consumer, async_commit, _bg_event_tx } =
16608 set_up_consumer_with_incomplete_async_commit_via(&tp, Some(callback), true).await;
16609 consumer.default_api_timeout_ms = 100;
16610 let handle = consumer.handle();
16611 let _in_callback = consumer.enter_callback_scope();
16612
16613 let err = handle
16614 .commit_sync_with_offsets(singleton_offsets(tp.clone(), 20))
16615 .await
16616 .expect_err("commit sync must time out");
16617 assert_pending_async_commit_timeout(&err);
16618 assert!(order.lock().unwrap().is_empty(), "the async commit has not completed");
16619
16620 async_commit.complete_with_error(Error::kafka_message("Test error"));
16621
16622 handle
16623 .commit_sync_with_offsets(singleton_offsets(tp, 20))
16624 .await
16625 .expect("commit sync must not fail");
16626 assert_eq!(*order.lock().unwrap(), vec!["async"]);
16627 assert_eq!(
16628 *errors.lock().unwrap(),
16629 vec![Some(Error::kafka_message("Test error").to_string())]
16630 );
16631 }
16632
16633 /// Java: `testCloseAwaitPendingAsyncCommitComplete` with the async commit
16634 /// made through a handle: `close` waits for it and runs its callback.
16635 #[tokio::test]
16636 async fn close_awaits_handle_commit_async() {
16637 let (mut consumer, mut handles) = make_test_consumer_with_channels();
16638 let drainer = tokio::spawn(async move {
16639 while let Some(env) = handles.app_event_rx.recv().await {
16640 match env.event {
16641 ApplicationEvent::CommitAsync { handle, offsets_ready, offsets } => {
16642 offsets_ready.complete(());
16643 // Complete later, so close finds the commit in flight.
16644 tokio::spawn(async move {
16645 tokio::time::sleep(Duration::from_millis(200)).await;
16646 handle.complete(offsets.unwrap_or_default());
16647 });
16648 },
16649 ApplicationEvent::CommitSync { handle, offsets_ready, .. } => {
16650 offsets_ready.complete(());
16651 handle.complete(HashMap::new());
16652 },
16653 ApplicationEvent::LeaveGroupOnClose { handle, .. } => {
16654 handle.complete(());
16655 },
16656 _ => {},
16657 }
16658 }
16659 });
16660
16661 let tp = TopicPartition::new("foo".to_string(), 0);
16662 let order = Arc::new(std::sync::Mutex::new(Vec::new()));
16663 {
16664 let _in_callback = consumer.enter_callback_scope();
16665 consumer
16666 .handle()
16667 .commit_async_with_offsets_callback(singleton_offsets(tp, 5), nesting_callback("async", &order, None))
16668 .await
16669 .expect("commit_async");
16670 }
16671 assert!(order.lock().unwrap().is_empty(), "the commit is still in flight");
16672
16673 consumer.close().await.expect("close ok");
16674 assert_eq!(*order.lock().unwrap(), vec!["async"], "close must wait for the handle's commit");
16675 drop(drainer);
16676 }
16677
16678 /// Counts `on_commit` calls; one instance per chain, sharing the count.
16679 struct CountingInterceptor {
16680 on_commit_count: Arc<std::sync::atomic::AtomicUsize>,
16681 }
16682
16683 impl crate::consumer::ConsumerInterceptor<Vec<u8>, Vec<u8>> for CountingInterceptor {
16684 fn on_consume(&self, _records: &mut ConsumerRecords<Vec<u8>, Vec<u8>>) {}
16685 fn on_commit(&self, _offsets: &HashMap<TopicPartition, OffsetAndMetadata>) {
16686 self.on_commit_count.fetch_add(1, Ordering::SeqCst);
16687 }
16688 }
16689
16690 /// A consumer whose commit-sync and async-commit interceptor chains both
16691 /// count into the returned counter (Java's
16692 /// `MockConsumerInterceptor.ON_COMMIT_COUNT`).
16693 fn make_test_consumer_with_counting_interceptors() -> (
16694 AsyncKafkaConsumer<Vec<u8>, Vec<u8>>,
16695 ConsumerTestHandles,
16696 Arc<std::sync::atomic::AtomicUsize>,
16697 ) {
16698 let count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
16699 let (consumer, handles) =
16700 make_test_consumer_with_channels_and_invoker_interceptors(vec![Box::new(CountingInterceptor {
16701 on_commit_count: Arc::clone(&count),
16702 })]);
16703 *consumer.interceptors.lock().unwrap() =
16704 ConsumerInterceptors::new(vec![Box::new(CountingInterceptor { on_commit_count: Arc::clone(&count) })]);
16705 (consumer, handles, count)
16706 }
16707
16708 /// Completes every commit event with `error` (Java's
16709 /// `completeCommit{Sync,Async}ApplicationEventExceptionally`).
16710 fn spawn_failing_commit_completer(
16711 mut app_event_rx: mpsc::UnboundedReceiver<ApplicationEventEnvelope>,
16712 error: Error,
16713 ) -> JoinHandle<()> {
16714 tokio::spawn(async move {
16715 while let Some(env) = app_event_rx.recv().await {
16716 if let ApplicationEvent::CommitAsync { handle, offsets_ready, .. }
16717 | ApplicationEvent::CommitSync { handle, offsets_ready, .. } = env.event
16718 {
16719 offsets_ready.complete(());
16720 handle.complete_with_error(error.clone());
16721 }
16722 }
16723 })
16724 }
16725
16726 /// Java: `testInterceptorCommitSync`, committing through a handle.
16727 #[tokio::test]
16728 async fn handle_interceptor_commit_sync() {
16729 let (consumer, handles, count) = make_test_consumer_with_counting_interceptors();
16730 let completer = spawn_commit_completer(handles.app_event_rx);
16731 let _in_callback = consumer.enter_callback_scope();
16732
16733 consumer
16734 .handle()
16735 .commit_sync_with_offsets(singleton_offsets(TopicPartition::new("t".to_string(), 0), 1))
16736 .await
16737 .expect("commit sync");
16738 assert_eq!(count.load(Ordering::SeqCst), 1);
16739 completer.abort();
16740 }
16741
16742 /// Java: `testNoInterceptorCommitSyncFailed`, committing through a handle.
16743 #[tokio::test]
16744 async fn handle_no_interceptor_commit_sync_failed() {
16745 let (consumer, handles, count) = make_test_consumer_with_counting_interceptors();
16746 let expected = Error::kafka_message("Test error");
16747 let completer = spawn_failing_commit_completer(handles.app_event_rx, expected.clone());
16748 let _in_callback = consumer.enter_callback_scope();
16749
16750 let actual = consumer
16751 .handle()
16752 .commit_sync_with_offsets(singleton_offsets(TopicPartition::new("t".to_string(), 0), 1))
16753 .await
16754 .expect_err("commit sync must fail");
16755 assert!(matches!(actual, Error::KafkaError(_)), "unexpected error: {actual:?}");
16756 assert_eq!(actual.message(), expected.message());
16757 assert_eq!(count.load(Ordering::SeqCst), 0);
16758 completer.abort();
16759 }
16760
16761 /// Java: `testInterceptorCommitAsync`, committing through a handle.
16762 #[tokio::test]
16763 async fn handle_interceptor_commit_async() {
16764 let (consumer, handles, count) = make_test_consumer_with_counting_interceptors();
16765 let completer = spawn_commit_completer(handles.app_event_rx);
16766 let _in_callback = consumer.enter_callback_scope();
16767
16768 consumer
16769 .handle()
16770 .commit_async_with_offsets_callback(
16771 singleton_offsets(TopicPartition::new("t".to_string(), 0), 1),
16772 Arc::new(NoopCallback),
16773 )
16774 .await
16775 .expect("commit async");
16776 assert_eq!(count.load(Ordering::SeqCst), 0);
16777
16778 // Java's `forceCommitCallbackInvocation()`.
16779 wait_last_pending_async_commit(&consumer).await;
16780 consumer
16781 .offset_commit_callback_invoker
16782 .invoke_pending_callbacks()
16783 .await
16784 .unwrap();
16785 assert_eq!(count.load(Ordering::SeqCst), 1);
16786 completer.abort();
16787 }
16788
16789 /// Java: `testNoInterceptorCommitAsyncFailed`, committing through a handle.
16790 #[tokio::test]
16791 async fn handle_no_interceptor_commit_async_failed() {
16792 let (consumer, handles, count) = make_test_consumer_with_counting_interceptors();
16793 let completer = spawn_failing_commit_completer(handles.app_event_rx, Error::kafka_message("Test error"));
16794 let _in_callback = consumer.enter_callback_scope();
16795
16796 consumer
16797 .handle()
16798 .commit_async_with_offsets_callback(
16799 singleton_offsets(TopicPartition::new("t".to_string(), 0), 1),
16800 Arc::new(NoopCallback),
16801 )
16802 .await
16803 .expect("commit async");
16804 assert_eq!(count.load(Ordering::SeqCst), 0);
16805
16806 wait_last_pending_async_commit(&consumer).await;
16807 consumer
16808 .offset_commit_callback_invoker
16809 .invoke_pending_callbacks()
16810 .await
16811 .unwrap();
16812 assert_eq!(count.load(Ordering::SeqCst), 0);
16813 completer.abort();
16814 }
16815
16816 /// Java: `testCommitAsyncUserSuppliedCallbackNoException` through the
16817 /// handle's `commit_async_with_callback` (no explicit offsets): the
16818 /// callback receives the committed offsets without an error.
16819 #[tokio::test]
16820 async fn handle_commit_async_with_callback_invokes_callback() {
16821 let (consumer, mut handles) = make_test_consumer_with_channels();
16822 let tp = TopicPartition::new("t".to_string(), 0);
16823 let committed = singleton_offsets(tp.clone(), 7);
16824 let committed_for_completer = committed.clone();
16825 let completer = tokio::spawn(async move {
16826 while let Some(env) = handles.app_event_rx.recv().await {
16827 if let ApplicationEvent::CommitAsync { handle, offsets_ready, offsets } = env.event {
16828 assert!(offsets.is_none(), "commit_async_with_callback commits all consumed offsets");
16829 offsets_ready.complete(());
16830 handle.complete(committed_for_completer.clone());
16831 }
16832 }
16833 });
16834
16835 /// The offsets a callback was handed, and whether it got an error.
16836 type Seen = (HashMap<TopicPartition, OffsetAndMetadata>, bool);
16837 struct RecordingCallback {
16838 seen: Arc<std::sync::Mutex<Vec<Seen>>>,
16839 }
16840 #[async_trait::async_trait]
16841 impl crate::consumer::OffsetCommitCallback for RecordingCallback {
16842 async fn on_complete(
16843 &self,
16844 offsets: &HashMap<TopicPartition, OffsetAndMetadata>,
16845 error: Option<&Error>,
16846 ) -> Result<(), Error> {
16847 self.seen.lock().unwrap().push((offsets.clone(), error.is_some()));
16848 Ok(())
16849 }
16850 }
16851 let seen = Arc::new(std::sync::Mutex::new(Vec::new()));
16852 let _in_callback = consumer.enter_callback_scope();
16853 consumer
16854 .handle()
16855 .commit_async_with_callback(Arc::new(RecordingCallback { seen: Arc::clone(&seen) }))
16856 .await
16857 .expect("commit async");
16858
16859 wait_last_pending_async_commit(&consumer).await;
16860 consumer
16861 .offset_commit_callback_invoker
16862 .invoke_pending_callbacks()
16863 .await
16864 .unwrap();
16865 assert_eq!(*seen.lock().unwrap(), vec![(committed, false)]);
16866 completer.abort();
16867 }
16868
16869 /// Java's `commit()` checks `group.id` first (`throwIfGroupIdNotDefined`),
16870 /// so a handle commit on a consumer without one fails as the consumer's.
16871 #[tokio::test]
16872 async fn handle_commits_without_group_id_fail() {
16873 let (consumer, _handles) = make_test_consumer_without_group_id();
16874 let handle = consumer.handle();
16875 let _in_callback = consumer.enter_callback_scope();
16876 let expected = "To use the group management or offset commit APIs, you must provide a valid group.id in \
16877 the consumer configuration.";
16878
16879 let err = handle
16880 .commit_async_with_callback(Arc::new(NoopCallback))
16881 .await
16882 .expect_err("no group.id");
16883 assert!(err.is_invalid_group_id_error(), "unexpected error: {err:?}");
16884 assert_eq!(err.message(), expected);
16885
16886 let tp = TopicPartition::new("t".to_string(), 0);
16887 let err = handle
16888 .commit_async_with_offsets_callback(singleton_offsets(tp.clone(), 1), Arc::new(NoopCallback))
16889 .await
16890 .expect_err("no group.id");
16891 assert_eq!(err.message(), expected);
16892
16893 let err = handle
16894 .commit_sync_with_offsets(singleton_offsets(tp, 1))
16895 .await
16896 .expect_err("no group.id");
16897 assert_eq!(err.message(), expected);
16898 }
16899
16900 /// The handle's `commit_sync` records the consumer's `commit-sync`
16901 /// metric, as `commitSync`'s `finally` does
16902 /// (`kafkaConsumerMetrics.recordCommitSync`).
16903 #[tokio::test]
16904 async fn handle_commit_sync_records_commit_sync_metric() {
16905 let (consumer, handles) = make_test_consumer_with_channels();
16906 let completer = spawn_commit_completer(handles.app_event_rx);
16907 let total = || {
16908 consumer
16909 .metrics
16910 .metrics()
16911 .snapshot()
16912 .iter()
16913 .find(|(name, _)| name.name() == "commit-sync-time-ns-total")
16914 .and_then(|(_, metric)| crate::common::Metric::metric_value(metric.as_ref()).as_double())
16915 .expect("commit-sync-time-ns-total registered")
16916 };
16917 let before = total();
16918 let _in_callback = consumer.enter_callback_scope();
16919 consumer
16920 .handle()
16921 .commit_sync_with_offsets(singleton_offsets(TopicPartition::new("t".to_string(), 0), 1))
16922 .await
16923 .expect("commit sync");
16924 assert!(total() > before, "commit-sync-time-ns-total must grow");
16925 completer.abort();
16926 }
16927
16928 // ─── KAFKA-20780 / KAFKA-20315: the inflight `AsyncPoll` lifecycle ───
16929
16930 /// Assigns and positions `topic1-0` so `poll()` gets past its
16931 /// preconditions without the background task.
16932 fn assign_and_seek_for_poll(handles: &ConsumerTestHandles) -> TopicPartition {
16933 let tp = TopicPartition::new("topic1", 0);
16934 let mut subs = handles.subscriptions.lock().unwrap();
16935 subs.assign_from_user(HashSet::from([tp.clone()])).unwrap();
16936 subs.seek(&tp, 0).unwrap();
16937 tp
16938 }
16939
16940 /// Drains every queued application event, returning the `AsyncPoll`
16941 /// states and the number of `CreateFetchRequests` events seen.
16942 fn drain_poll_events(handles: &mut ConsumerTestHandles) -> (Vec<Arc<AsyncPollState>>, usize) {
16943 let mut polls = Vec::new();
16944 let mut prefetches = 0;
16945 while let Ok(env) = handles.app_event_rx.try_recv() {
16946 match env.event {
16947 ApplicationEvent::AsyncPoll { state, .. } => polls.push(state),
16948 ApplicationEvent::CreateFetchRequests { .. } => prefetches += 1,
16949 _ => {},
16950 }
16951 }
16952 (polls, prefetches)
16953 }
16954
16955 /// Translated from `AsyncKafkaConsumerTest.testInflightPollResubmittedAfterCompletionWithEmptyBuffer`
16956 /// (KAFKA-20780). One `poll()` runs two internal passes over an empty
16957 /// buffer; the poll event submitted on the first completes during the
16958 /// first wait, and the second pass must submit a fresh one so a fetch stays
16959 /// pending. Before the fix only the first pass submitted (the second was
16960 /// starved until a third pass cleared the completed event).
16961 ///
16962 /// Java completes the event from a mocked `awaitWakeup` and advances a
16963 /// `MockTime` 150 ms per pass under a 200 ms timeout. Here the stand-in
16964 /// background task completes each event 30 ms after it arrives, while the
16965 /// real wait is bounded at `retry.backoff.ms` (100 ms: the partition is
16966 /// fetchable with nothing buffered), so the 200 ms poll also runs exactly
16967 /// two passes and the event completes inside the first wait.
16968 #[tokio::test]
16969 async fn test_inflight_poll_resubmitted_after_completion_with_empty_buffer() {
16970 let (mut consumer, mut handles) = make_test_consumer_with_channels();
16971 assign_and_seek_for_poll(&handles);
16972 consumer.max_time_to_wait_ms.store(150, Ordering::Release);
16973
16974 let submitted = Arc::new(std::sync::atomic::AtomicUsize::new(0));
16975 let submitted_in_task = Arc::clone(&submitted);
16976 let drainer = tokio::spawn(async move {
16977 while let Some(env) = handles.app_event_rx.recv().await {
16978 if let ApplicationEvent::AsyncPoll { state, .. } = env.event {
16979 submitted_in_task.fetch_add(1, Ordering::SeqCst);
16980 tokio::spawn(async move {
16981 tokio::time::sleep(Duration::from_millis(30)).await;
16982 state.mark_validate_positions_complete();
16983 state.complete_successfully();
16984 });
16985 }
16986 }
16987 });
16988
16989 let records = consumer.poll(Duration::from_millis(200)).await.expect("poll ok");
16990 assert!(records.is_empty());
16991 // Let the drainer count the last event.
16992 tokio::time::sleep(Duration::from_millis(10)).await;
16993
16994 // A fresh poll event on each of the two passes.
16995 assert_eq!(2, submitted.load(Ordering::SeqCst));
16996 drainer.abort();
16997 }
16998
16999 /// Translated from `AsyncKafkaConsumerTest.testPollSurfacesInflightPollErrorAndResumes`
17000 /// (KAFKA-20780): an inflight poll that completed with an error surfaces it
17001 /// from `poll()` and is cleared, so the next `poll()` submits a fresh event.
17002 ///
17003 /// Java's mock completes the event inside `add()`, so the first `poll()`
17004 /// throws. The Rust handler's `add` is a channel send, so the event is
17005 /// completed between two `poll()` calls and surfaces at the start of the
17006 /// second; the third resumes.
17007 #[tokio::test]
17008 async fn test_poll_surfaces_inflight_poll_error_and_resumes() {
17009 let (mut consumer, mut handles) = make_test_consumer_with_channels();
17010 assign_and_seek_for_poll(&handles);
17011
17012 consumer.poll(Duration::ZERO).await.expect("first poll ok");
17013 let (polls, _) = drain_poll_events(&mut handles);
17014 assert_eq!(1, polls.len());
17015 // The inflight poll completes with an error (e.g. a failed fetch);
17016 // poll() must surface it and clear it.
17017 polls[0].complete_with_error(Error::kafka_message("fetch failed"));
17018 let err = consumer.poll(Duration::ZERO).await.expect_err("the inflight error surfaces");
17019 assert_eq!("fetch failed", err.message());
17020 assert!(err.is_kafka_error());
17021 assert!(consumer.inflight_poll.is_none(), "the errored event is cleared");
17022
17023 // The next (successful) poll submits a fresh event rather than
17024 // re-raising.
17025 consumer.poll(Duration::ZERO).await.expect("poll resumes");
17026 let (polls, _) = drain_poll_events(&mut handles);
17027 assert_eq!(1, polls.len(), "two AsyncPoll events in all");
17028 }
17029
17030 /// Translated from `AsyncKafkaConsumerTest.testBufferedRecordsReturnedWithoutResubmittingPollEvent`
17031 /// (KAFKA-20780): when the inflight poll completed and filled the buffer,
17032 /// the next `poll()` returns those records without submitting a new poll
17033 /// event (a fresh one would re-run validate-positions and starve them).
17034 ///
17035 /// Also carries the assertion 20e952c783 added to Java's
17036 /// `testClearWakeupTriggerAfterPoll`:
17037 /// when `poll()` returns records the next
17038 /// fetch is pipelined (`CreateFetchRequests`), so a fetch request stays
17039 /// pending while the application processes them.
17040 #[tokio::test]
17041 async fn test_buffered_records_returned_without_resubmitting_poll_event() {
17042 use crate::common::compress::Compression;
17043 use crate::common::record::TimestampType;
17044 use crate::common::record::internal::{MemoryRecords, SimpleRecord};
17045 use crate::consumer::internals::CompletedFetch;
17046 use crate::fetch_response_data::PartitionData;
17047
17048 let (mut consumer, mut handles) = make_test_consumer_with_channels();
17049 let tp = assign_and_seek_for_poll(&handles);
17050
17051 // The first poll submits a poll event; it stays in flight.
17052 consumer.poll(Duration::ZERO).await.expect("first poll ok");
17053 let (polls, _) = drain_poll_events(&mut handles);
17054 assert_eq!(1, polls.len());
17055
17056 // That poll event now completes and its fetch has filled the buffer.
17057 polls[0].mark_validate_positions_complete();
17058 polls[0].complete_successfully();
17059 let records = MemoryRecords::with_records_with_magic_initial_offset_timestamp_type(
17060 2,
17061 0,
17062 Compression::none().build(),
17063 TimestampType::CreateTime,
17064 &[SimpleRecord::with_timestamp_key_value(
17065 0,
17066 Some(b"key".to_vec()),
17067 Some(b"value".to_vec()),
17068 )],
17069 );
17070 let mut partition_data = PartitionData::new();
17071 partition_data.set_partition_index(0);
17072 partition_data.set_high_watermark(1);
17073 partition_data.set_records(Some(bytes::Bytes::from(records.buffer().to_vec())));
17074 consumer.fetch_buffer.add(CompletedFetch::new(tp, partition_data));
17075
17076 // The next poll returns the buffered records and submits no new poll
17077 // event (only the original one exists).
17078 let polled = consumer.poll(Duration::ZERO).await.expect("poll ok");
17079 assert_eq!(1, polled.count());
17080 let (polls, prefetches) = drain_poll_events(&mut handles);
17081 assert!(polls.is_empty(), "no AsyncPoll event resubmitted");
17082 assert!(prefetches >= 1, "the next fetch is pipelined when poll() returns records");
17083 }
17084
17085 /// Translated from `AsyncKafkaConsumerTest.testPollDoesNotAddNewAsyncPollEventWhenOneIsAlreadyInFlight`
17086 /// (KAFKA-20315, 0fd8327920): while the poll event stays in flight, the
17087 /// many internal passes of one `poll()` add it only once.
17088 ///
17089 /// Java bounds each wait with a mocked `awaitWakeup` over `MockTime`
17090 /// (`maximumTimeToWait` 100 ms, 450 ms timeout). Here the waits are real,
17091 /// bounded at `retry.backoff.ms` (100 ms).
17092 #[tokio::test]
17093 async fn test_poll_does_not_add_new_async_poll_event_when_one_is_already_in_flight() {
17094 let (mut consumer, mut handles) = make_test_consumer_with_channels();
17095 assign_and_seek_for_poll(&handles);
17096 consumer.max_time_to_wait_ms.store(100, Ordering::Release);
17097
17098 let started = std::time::Instant::now();
17099 let records = consumer.poll(Duration::from_millis(450)).await.expect("poll ok");
17100 assert!(records.is_empty());
17101 // The "wait for fetches" path ran, i.e. more than one pass.
17102 assert!(started.elapsed() >= Duration::from_millis(400));
17103
17104 let (polls, prefetches) = drain_poll_events(&mut handles);
17105 assert_eq!(1, polls.len(), "only one AsyncPoll event despite multiple loop iterations");
17106 assert!(prefetches >= 2, "each pass waited for fetches ({prefetches} prefetches)");
17107 }
17108
17109 // ─── KAFKA-20854: the `pollForFetches` wait bound ────────────────────
17110
17111 /// Translated from `AsyncKafkaConsumerTest.testPollWithManualAssignmentDoesNotBusyLoop`,
17112 /// as KAFKA-20854 amended it. With a manually assigned, positioned
17113 /// partition whose data is already buffered, and the background task
17114 /// reporting `maximumTimeToWait() == Long.MAX_VALUE` (the UNSUBSCRIBED
17115 /// heartbeat manager, KAFKA-20426), nothing bounds the wait: the
17116 /// application task waits the full 500 ms user timeout, not 0.
17117 ///
17118 /// Java drives `poll(Duration.ofMillis(500))` with a mocked buffer and
17119 /// captures the `Timer` handed to `awaitWakeup`; that `Timer` is
17120 /// [`AsyncKafkaConsumer::poll_for_fetches_timeout_ms`]'s value, asserted
17121 /// here directly.
17122 #[tokio::test]
17123 async fn test_poll_with_manual_assignment_does_not_busy_loop() {
17124 use crate::consumer::internals::CompletedFetch;
17125 use crate::fetch_response_data::PartitionData;
17126
17127 let (consumer, handles) = make_test_consumer_with_channels();
17128 let tp = TopicPartition::new("topic1", 0);
17129 {
17130 let mut subs = handles.subscriptions.lock().unwrap();
17131 subs.assign_from_user(HashSet::from([tp.clone()])).unwrap();
17132 subs.seek(&tp, 0).unwrap();
17133 }
17134 consumer.max_time_to_wait_ms.store(i64::MAX, Ordering::Release);
17135 // The partition is fetchable but already buffered, so the wait is not bounded.
17136 consumer.fetch_buffer.add(CompletedFetch::new(tp.clone(), PartitionData::new()));
17137
17138 let poll_timeout_ms = 500;
17139 assert_eq!(
17140 poll_timeout_ms,
17141 consumer.poll_for_fetches_timeout_ms(poll_timeout_ms),
17142 "Expected poll wait timer to use the full user timeout (no busy loop)"
17143 );
17144 }
17145
17146 /// The other branches of the same bound (KAFKA-20854; Java has no
17147 /// dedicated test): each "background progress may enable fetching" state
17148 /// waits exactly `retry.backoff.ms`, never 0 and never the full timeout.
17149 /// The first case is the bootstrapping consumer (KIP-909): nothing
17150 /// assigned yet.
17151 #[tokio::test]
17152 async fn test_poll_for_fetches_timeout_bounded_by_retry_backoff() {
17153 use crate::consumer::internals::CompletedFetch;
17154 use crate::fetch_response_data::PartitionData;
17155
17156 let (consumer, handles) = make_test_consumer_with_channels();
17157 let retry_backoff_ms = consumer.retry_backoff_ms;
17158 assert_eq!(retry_backoff_ms, 100, "the fixture's retry.backoff.ms default");
17159 consumer.max_time_to_wait_ms.store(i64::MAX, Ordering::Release);
17160
17161 // No assigned partitions (bootstrap resolution / group join pending).
17162 assert_eq!(retry_backoff_ms, consumer.poll_for_fetches_timeout_ms(30_000));
17163
17164 // Assigned, but without a valid position.
17165 let tp0 = TopicPartition::new("topic1", 0);
17166 let tp1 = TopicPartition::new("topic1", 1);
17167 handles
17168 .subscriptions
17169 .lock()
17170 .unwrap()
17171 .assign_from_user(HashSet::from([tp0.clone(), tp1.clone()]))
17172 .unwrap();
17173 assert_eq!(retry_backoff_ms, consumer.poll_for_fetches_timeout_ms(30_000));
17174
17175 // Positioned, but tp1 is fetchable with nothing buffered (skipped for
17176 // backoff / an in-flight request / a missing leader).
17177 {
17178 let mut subs = handles.subscriptions.lock().unwrap();
17179 subs.seek(&tp0, 0).unwrap();
17180 subs.seek(&tp1, 0).unwrap();
17181 }
17182 consumer
17183 .fetch_buffer
17184 .add(CompletedFetch::new(tp0.clone(), PartitionData::new()));
17185 assert_eq!(retry_backoff_ms, consumer.poll_for_fetches_timeout_ms(30_000));
17186
17187 // Everything fetchable is buffered: the full timeout.
17188 consumer
17189 .fetch_buffer
17190 .add(CompletedFetch::new(tp1.clone(), PartitionData::new()));
17191 assert_eq!(30_000, consumer.poll_for_fetches_timeout_ms(30_000));
17192
17193 // Below the backoff, the smaller of the background bound and the
17194 // caller's timer wins unchanged — including 0, which the caller turns
17195 // into an immediate return rather than a wait.
17196 assert_eq!(40, consumer.poll_for_fetches_timeout_ms(40));
17197 consumer.max_time_to_wait_ms.store(0, Ordering::Release);
17198 assert_eq!(0, consumer.poll_for_fetches_timeout_ms(30_000));
17199 }
17200
17201 /// Critic 93 Issue 1: while `maximumTimeToWait` is 0 (Java's value, for
17202 /// instance while a JOINING member's first heartbeat is in flight with a
17203 /// zero interval), `poll()` loops without blocking. It must still yield
17204 /// every iteration, or it monopolises its tokio worker: on a
17205 /// `current_thread` runtime a co-scheduled task would not run until
17206 /// `poll()` returns. In the integration harness that task was the
17207 /// `BrokerProxy` forwarding the very heartbeat response the spin was
17208 /// waiting for, so every first join stalled for the 30 s request timeout.
17209 ///
17210 /// Mutation check: without the `yield_now` in `poll_inner` the spawned
17211 /// task has not run at all by the time `poll()` returns 300 ms later, and
17212 /// the test fails.
17213 #[tokio::test(flavor = "current_thread")]
17214 async fn test_poll_yields_while_maximum_time_to_wait_is_zero() {
17215 let (mut consumer, handles) = make_test_consumer_with_channels();
17216 let tp = TopicPartition::new("topic1", 0);
17217 {
17218 let mut subs = handles.subscriptions.lock().unwrap();
17219 subs.assign_from_user(HashSet::from([tp.clone()])).unwrap();
17220 subs.seek(&tp, 0).unwrap();
17221 }
17222 // The background task's published bound: no wait at all.
17223 consumer.max_time_to_wait_ms.store(0, Ordering::Release);
17224
17225 let start = std::time::Instant::now();
17226 let ran_after = Arc::new(Mutex::new(None::<Duration>));
17227 let ran_after_task = Arc::clone(&ran_after);
17228 tokio::spawn(async move {
17229 *ran_after_task.lock().unwrap() = Some(start.elapsed());
17230 });
17231
17232 let poll_timeout = Duration::from_millis(300);
17233 let records = crate::consumer::Consumer::poll(&mut consumer, poll_timeout)
17234 .await
17235 .expect("poll");
17236 let poll_took = start.elapsed();
17237 assert!(records.is_empty());
17238 // Waits out (about) its timeout: the deadline is on the fixture's
17239 // millisecond clock, so allow for rounding.
17240 assert!(
17241 poll_took >= Duration::from_millis(250),
17242 "poll waits out its timeout: {poll_took:?}"
17243 );
17244
17245 let ran_after = ran_after.lock().unwrap().expect("the co-scheduled task ran during poll()");
17246 assert!(
17247 ran_after < Duration::from_millis(100),
17248 "a task on the same current_thread runtime first ran {ran_after:?} into a {poll_timeout:?} poll(): \
17249 poll() spun without yielding while maximumTimeToWait was 0"
17250 );
17251 drop(handles.subscriptions);
17252 }
17253
17254 // ── Duration::MAX timeouts (audit finding C1-03) ───────────────────
17255 //
17256 // Rust saturates a `Duration` beyond `i64::MAX` ms to `i64::MAX` ms
17257 // (`ConsumerUtils::duration_to_millis`) and never rejects it, so each test
17258 // below asserts that single outcome: the call behaves as Java does for
17259 // `Long.MAX_VALUE` ms.
17260
17261 /// Audit finding C1-03: `poll(Duration::MAX)` must keep blocking.
17262 /// Java: `poll(Duration)` with `Long.MAX_VALUE` ms blocks until records
17263 /// arrive (`AsyncKafkaConsumer.java` `poll` → `time.timer(timeout)`). An
17264 /// unchecked `timeout.as_millis() as i64` wrapped `Duration::MAX` to
17265 /// -1 ms, so the deadline had already passed and `poll` returned an empty
17266 /// batch at once.
17267 #[tokio::test]
17268 async fn poll_with_duration_max_blocks() {
17269 let (mut consumer, mut handles) = make_test_consumer_with_channels();
17270 assign_and_seek_for_poll(&handles);
17271 // Stand-in bg task: complete every AsyncPoll so poll() is gated only
17272 // by its own deadline (no records ever arrive).
17273 let drainer = tokio::spawn(async move {
17274 while let Some(env) = handles.app_event_rx.recv().await {
17275 if let ApplicationEvent::AsyncPoll { state, .. } = env.event {
17276 state.mark_validate_positions_complete();
17277 state.complete_successfully();
17278 }
17279 }
17280 });
17281
17282 let start = std::time::Instant::now();
17283 let outcome = tokio::time::timeout(Duration::from_millis(500), consumer.poll(Duration::MAX)).await;
17284 let took = start.elapsed();
17285 drainer.abort();
17286 assert!(
17287 outcome.is_err(),
17288 "poll(Duration::MAX) must still be blocked when the 500 ms test timeout fires (Java: \
17289 Long.MAX_VALUE ms blocks until records arrive); it returned {:?} after {took:?}",
17290 outcome.as_ref().map(|r| r.as_ref().map(|recs| recs.count()))
17291 );
17292 }
17293
17294 /// Audit finding C1-03: `commit_sync_with_offsets_timeout(.., Duration::MAX)` must wait
17295 /// for a commit that completes 50 ms later and return `Ok`. Java:
17296 /// `commitSync(offsets, Duration)` with `Long.MAX_VALUE` ms waits for the
17297 /// commit; it never fails with a retriable `TimeoutException` before the
17298 /// commit completes.
17299 #[tokio::test]
17300 async fn commit_sync_with_duration_max_does_not_time_out_immediately() {
17301 let (mut consumer, mut handles) = make_test_consumer_with_channels();
17302 let tp = TopicPartition::new("t0".to_string(), 0);
17303 let offsets = singleton_offsets(tp, 10);
17304 let completer = tokio::spawn(async move {
17305 while let Some(env) = handles.app_event_rx.recv().await {
17306 if let ApplicationEvent::CommitSync { handle, offsets_ready, .. } = env.event {
17307 offsets_ready.complete(());
17308 tokio::time::sleep(Duration::from_millis(50)).await;
17309 handle.complete(HashMap::new());
17310 }
17311 }
17312 });
17313
17314 let outcome = tokio::time::timeout(
17315 Duration::from_secs(2),
17316 consumer.commit_sync_with_offsets_timeout(offsets, Duration::MAX),
17317 )
17318 .await;
17319 completer.abort();
17320 assert!(
17321 matches!(outcome, Ok(Ok(()))),
17322 "commit_sync_with_offsets_timeout(Duration::MAX) must wait for a commit that completes 50 ms \
17323 later and return Ok (Java: Long.MAX_VALUE ms waits for the commit); got {outcome:?}"
17324 );
17325 }
17326
17327 /// Audit finding C1-03 (ConsumerHandle path): `ConsumerHandle::position_with_timeout(..,
17328 /// Duration::MAX)` must keep waiting for a position. An unchecked cast
17329 /// wrapped the timeout to -1 ms and returned a retriable `TimeoutError` at once.
17330 #[tokio::test]
17331 async fn handle_position_with_duration_max_does_not_time_out_immediately() {
17332 let (consumer, handles) = make_test_consumer_with_channels();
17333 let tp = TopicPartition::new("topic1", 0);
17334 {
17335 // Assigned but no position yet: position() must wait for one.
17336 let mut subs = handles.subscriptions.lock().unwrap();
17337 subs.assign_from_user(HashSet::from([tp.clone()])).unwrap();
17338 }
17339 // `ConsumerHandle` operations run only inside a callback the consumer
17340 // is running; outside one they fail with `ConcurrentModification`.
17341 let _in_callback = consumer.enter_callback_scope();
17342 let handle = consumer.handle();
17343 let outcome =
17344 tokio::time::timeout(Duration::from_millis(500), handle.position_with_timeout(&tp, Duration::MAX)).await;
17345 assert!(
17346 outcome.is_err(),
17347 "ConsumerHandle::position_with_timeout(Duration::MAX) must still be waiting when the 500 ms \
17348 test timeout fires (Java: position(tp, Duration) with Long.MAX_VALUE ms waits for a \
17349 position); got {outcome:?}"
17350 );
17351 drop(handles.app_event_rx);
17352 }
17353
17354 /// Audit finding C1-03 (close path): `close_with_options(timeout(Duration::MAX))` must
17355 /// not close with a zero budget. Java caps the close timer at
17356 /// `request.timeout.ms` (`createTimerForCloseRequests`,
17357 /// `AsyncKafkaConsumer.java`), so `Long.MAX_VALUE` ms hands the
17358 /// network thread ~`request.timeout.ms`. An unchecked cast gave `min(-1, request_timeout_ms)`
17359 /// = -1 ms, so auto-commit-on-close, leave-group and the network-thread
17360 /// cleanup all got 0 ms.
17361 #[tokio::test]
17362 async fn close_with_duration_max_keeps_the_request_timeout_budget() {
17363 use crate::consumer::CloseOptions;
17364
17365 let (mut consumer, mut handles) = make_test_consumer_with_channels();
17366 let request_timeout_ms = consumer.config.request_timeout_ms() as i64;
17367 let drainer = tokio::spawn(async move {
17368 while let Some(env) = handles.app_event_rx.recv().await {
17369 match env.event {
17370 ApplicationEvent::LeaveGroupOnClose { handle, .. } => {
17371 handle.complete(());
17372 },
17373 ApplicationEvent::CommitSync { handle, offsets_ready, .. } => {
17374 offsets_ready.complete(());
17375 handle.complete(HashMap::new());
17376 },
17377 _ => {},
17378 }
17379 }
17380 });
17381 let result = consumer.close_with_options(CloseOptions::new_with_timeout(Duration::MAX)).await;
17382 drainer.abort();
17383
17384 // The bg task gets what is left of the close timer
17385 // (`close_deadline_ms - now`, on the fixture's real `SystemTime`), so the
17386 // few milliseconds the close steps take are subtracted from the
17387 // `request.timeout.ms` cap; 100 ms covers that and still fails a cap
17388 // that is even slightly off.
17389 let signalled = handles.signal_close_timeout_ms.load(Ordering::Acquire);
17390 assert!(
17391 (request_timeout_ms - 100..=request_timeout_ms).contains(&signalled),
17392 "close(Duration::MAX) must hand the bg task the request.timeout.ms cap \
17393 ({request_timeout_ms} ms, less the elapsed close time) (Java: createTimerForCloseRequests \
17394 caps Long.MAX_VALUE ms at request.timeout.ms); got {signalled} ms (result {result:?})"
17395 );
17396 }
17397
17398 // ─── Background events wait for poll() (§31, audit finding VC-01) ───
17399 //
17400 // Java's `AsyncKafkaConsumer` runs `processBackgroundEvents()` — the only
17401 // path that invokes `onPartitionsRevoked/Assigned/Lost` for a queued
17402 // `PartitionsRemovedEvent` / `PartitionsAssignedEvent` or throws a queued
17403 // `ErrorEvent` — only from `poll()` (`checkInflightPoll`,
17404 // `updateAssignmentMetadataIfNeeded`) and `unsubscribe()`. `commitSync`,
17405 // `committed`, `seek`, `pause`, `assign` and `close` block with a plain
17406 // `applicationEventHandler.addAndGet(event)` / `ConsumerUtils.getResult`,
17407 // running at most the offset-commit callbacks.
17408
17409 /// A rebalance listener that only records how often each method ran.
17410 struct QueuedEventCountingListener {
17411 revoked: Arc<AtomicUsize>,
17412 assigned: Arc<AtomicUsize>,
17413 }
17414
17415 #[async_trait::async_trait]
17416 impl ConsumerRebalanceListener for QueuedEventCountingListener {
17417 async fn on_partitions_revoked(&self, _: &[TopicPartition]) -> Result<(), Error> {
17418 self.revoked.fetch_add(1, Ordering::SeqCst);
17419 Ok(())
17420 }
17421 async fn on_partitions_assigned(&self, _: &[TopicPartition]) -> Result<(), Error> {
17422 self.assigned.fetch_add(1, Ordering::SeqCst);
17423 Ok(())
17424 }
17425 }
17426
17427 /// Installs a [`QueuedEventCountingListener`] and returns its
17428 /// `(revoked, assigned)` counters.
17429 fn install_counting_listener(
17430 consumer: &AsyncKafkaConsumer<Vec<u8>, Vec<u8>>,
17431 ) -> (Arc<AtomicUsize>, Arc<AtomicUsize>) {
17432 let revoked = Arc::new(AtomicUsize::new(0));
17433 let assigned = Arc::new(AtomicUsize::new(0));
17434 *consumer.rebalance_listener.lock().unwrap() = Some(Arc::new(QueuedEventCountingListener {
17435 revoked: Arc::clone(&revoked),
17436 assigned: Arc::clone(&assigned),
17437 }) as Arc<dyn ConsumerRebalanceListener>);
17438 (revoked, assigned)
17439 }
17440
17441 /// Fake background task: completes every event these tests submit at
17442 /// once, counting the `ApplyAssignment`s, and ignores the rest
17443 /// (`AsyncPoll` stays in flight).
17444 fn spawn_queued_event_test_completer(
17445 mut app_event_rx: mpsc::UnboundedReceiver<ApplicationEventEnvelope>,
17446 apply_assignments: Arc<AtomicUsize>,
17447 ) -> tokio::task::JoinHandle<()> {
17448 tokio::spawn(async move {
17449 while let Some(env) = app_event_rx.recv().await {
17450 match env.event {
17451 ApplicationEvent::CommitSync { handle, offsets_ready, .. }
17452 | ApplicationEvent::CommitAsync { handle, offsets_ready, .. } => {
17453 offsets_ready.complete(());
17454 handle.complete(HashMap::new());
17455 },
17456 ApplicationEvent::FetchCommittedOffsets { handle, .. } => {
17457 handle.complete(HashMap::new());
17458 },
17459 ApplicationEvent::PausePartitions { handle, .. }
17460 | ApplicationEvent::SeekUnvalidated { handle, .. }
17461 | ApplicationEvent::AssignmentChange { handle, .. } => {
17462 handle.complete(());
17463 },
17464 ApplicationEvent::ApplyAssignment { handle, .. } => {
17465 apply_assignments.fetch_add(1, Ordering::SeqCst);
17466 handle.complete(());
17467 },
17468 _ => {},
17469 }
17470 }
17471 })
17472 }
17473
17474 fn assign_for_queued_event_test(consumer: &AsyncKafkaConsumer<Vec<u8>, Vec<u8>>, tp: &TopicPartition) {
17475 let mut subs = consumer.subscriptions.lock().unwrap();
17476 subs.assign_from_user(HashSet::from([tp.clone()])).unwrap();
17477 }
17478
17479 fn queue_revoke_callback(
17480 bg_event_tx: &mpsc::UnboundedSender<BackgroundEventEnvelope>,
17481 tp: &TopicPartition,
17482 ) -> tokio::sync::oneshot::Receiver<Result<(), Error>> {
17483 let (ack_tx, ack_rx) = tokio::sync::oneshot::channel::<Result<(), Error>>();
17484 bg_event_tx
17485 .send(BackgroundEventEnvelope {
17486 event: BackgroundEvent::PartitionsRemoved {
17487 method_name: crate::consumer::internals::ConsumerRebalanceListenerMethodName::OnPartitionsRevoked,
17488 partitions: vec![tp.clone()],
17489 ack: ack_tx,
17490 ack_through_application_events: false,
17491 },
17492 enqueued_ms: 0,
17493 })
17494 .expect("bg event sent");
17495 ack_rx
17496 }
17497
17498 fn queue_background_error(bg_event_tx: &mpsc::UnboundedSender<BackgroundEventEnvelope>, msg: &str) {
17499 bg_event_tx
17500 .send(BackgroundEventEnvelope {
17501 event: BackgroundEvent::Error { error: Error::local_illegal_state(msg.to_string()) },
17502 enqueued_ms: 0,
17503 })
17504 .expect("bg event sent");
17505 }
17506
17507 /// A boxed consumer call, for the shared bodies below.
17508 type QueuedEventOp = for<'a> fn(
17509 &'a mut AsyncKafkaConsumer<Vec<u8>, Vec<u8>>,
17510 TopicPartition,
17511 ) -> std::pin::Pin<Box<dyn Future<Output = Result<(), Error>> + Send + 'a>>;
17512
17513 /// Shared body: a revoke callback is queued; `op` must not run it, the
17514 /// next `poll()` must.
17515 async fn assert_queued_listener_waits_for_poll(op_name: &str, op: QueuedEventOp) {
17516 let (mut consumer, handles) = make_test_consumer_with_channels();
17517 let ConsumerTestHandles { app_event_rx, bg_event_tx, subscriptions, .. } = handles;
17518 let completer = spawn_queued_event_test_completer(app_event_rx, Arc::new(AtomicUsize::new(0)));
17519 let tp = TopicPartition::new("t".to_string(), 0);
17520 assign_for_queued_event_test(&consumer, &tp);
17521 let (revoked, _assigned) = install_counting_listener(&consumer);
17522
17523 let mut ack_rx = queue_revoke_callback(&bg_event_tx, &tp);
17524
17525 op(&mut consumer, tp.clone())
17526 .await
17527 .unwrap_or_else(|e| panic!("{op_name} must succeed: {e}"));
17528 assert_eq!(
17529 revoked.load(Ordering::SeqCst),
17530 0,
17531 "{op_name} invoked onPartitionsRevoked for a queued PartitionsRemovedEvent; Java's \
17532 AsyncKafkaConsumer.{op_name} waits with applicationEventHandler.addAndGet / getResult and \
17533 never runs processBackgroundEvents(), so the listener only runs from the next poll()"
17534 );
17535 assert!(
17536 matches!(ack_rx.try_recv(), Err(tokio::sync::oneshot::error::TryRecvError::Empty)),
17537 "{op_name} must leave the callback-needed event (and its ack) for poll()"
17538 );
17539
17540 consumer.poll(Duration::ZERO).await.expect("poll ok");
17541 assert_eq!(
17542 revoked.load(Ordering::SeqCst),
17543 1,
17544 "the next poll() must run the queued onPartitionsRevoked exactly once (Java checkInflightPoll -> \
17545 processBackgroundEvents)"
17546 );
17547 assert!(ack_rx.await.expect("ack sent").is_ok(), "poll() acks the callback");
17548 completer.abort();
17549 drop(subscriptions);
17550 }
17551
17552 /// Shared body: a background error is queued; `op` must not consume it,
17553 /// the next `poll()` must return it.
17554 async fn assert_queued_error_waits_for_poll(op_name: &str, op: QueuedEventOp) {
17555 let (mut consumer, handles) = make_test_consumer_with_channels();
17556 let ConsumerTestHandles { app_event_rx, bg_event_tx, subscriptions, .. } = handles;
17557 let completer = spawn_queued_event_test_completer(app_event_rx, Arc::new(AtomicUsize::new(0)));
17558 let tp = TopicPartition::new("t".to_string(), 0);
17559 assign_for_queued_event_test(&consumer, &tp);
17560
17561 queue_background_error(&bg_event_tx, "queued background error");
17562
17563 let result = op(&mut consumer, tp.clone()).await;
17564 assert!(
17565 result.is_ok(),
17566 "{op_name} returned the queued background ErrorEvent ({:?}); Java's AsyncKafkaConsumer.{op_name} \
17567 does not run processBackgroundEvents(), so the error stays queued and is thrown by the next poll()",
17568 result.err()
17569 );
17570
17571 let err = consumer
17572 .poll(Duration::ZERO)
17573 .await
17574 .expect_err("the next poll() must throw the queued background error, as Java's does");
17575 // Java's `poll()` throws it through `maybeWrapAsKafkaException`.
17576 assert_eq!(
17577 err.message(),
17578 "LocalIllegalStateError: queued background error",
17579 "poll() must surface the queued background error"
17580 );
17581 completer.abort();
17582 drop(subscriptions);
17583 }
17584
17585 #[tokio::test]
17586 async fn commit_sync_leaves_queued_rebalance_callback_for_poll() {
17587 assert_queued_listener_waits_for_poll("commitSync", |c, _tp| {
17588 Box::pin(c.commit_sync_with_timeout(Duration::from_secs(5)))
17589 })
17590 .await;
17591 }
17592
17593 #[tokio::test]
17594 async fn committed_leaves_queued_rebalance_callback_for_poll() {
17595 assert_queued_listener_waits_for_poll("committed", |c, tp| {
17596 Box::pin(async move { c.committed(&[tp]).await.map(|_| ()) })
17597 })
17598 .await;
17599 }
17600
17601 #[tokio::test]
17602 async fn pause_leaves_queued_background_error_for_poll() {
17603 assert_queued_error_waits_for_poll("pause", |c, tp| Box::pin(async move { c.pause(&[tp]).await })).await;
17604 }
17605
17606 #[tokio::test]
17607 async fn seek_leaves_queued_background_error_for_poll() {
17608 assert_queued_error_waits_for_poll("seek", |c, tp| Box::pin(c.seek_with_offset(tp, 5))).await;
17609 }
17610
17611 #[tokio::test]
17612 async fn commit_sync_leaves_queued_background_error_for_poll() {
17613 assert_queued_error_waits_for_poll("commitSync", |c, _tp| {
17614 Box::pin(c.commit_sync_with_timeout(Duration::from_secs(5)))
17615 })
17616 .await;
17617 }
17618
17619 /// `commitAsync` with a queued `ErrorEvent`: Java's `commitAsync` waits on
17620 /// `offsetsReady` without processing background events
17621 /// (`AsyncKafkaConsumer#commit(CommitEvent)`), so it returns normally, its
17622 /// callback runs once the commit completes (from the next `poll()`'s
17623 /// `executeCallbacks()`), and that `poll()` then throws the queued error.
17624 /// Draining here used to fail `commitAsync` after the commit had been
17625 /// sent, so its callback was never invoked.
17626 #[tokio::test]
17627 async fn commit_async_with_queued_background_error_still_runs_its_callback() {
17628 struct CountingCallback {
17629 invoked: Arc<AtomicUsize>,
17630 }
17631 #[async_trait::async_trait]
17632 impl crate::consumer::OffsetCommitCallback for CountingCallback {
17633 async fn on_complete(
17634 &self,
17635 _offsets: &HashMap<TopicPartition, OffsetAndMetadata>,
17636 error: Option<&Error>,
17637 ) -> Result<(), Error> {
17638 assert!(error.is_none(), "the commit succeeded, got {error:?}");
17639 self.invoked.fetch_add(1, Ordering::SeqCst);
17640 Ok(())
17641 }
17642 }
17643
17644 let (mut consumer, handles) = make_test_consumer_with_channels();
17645 let ConsumerTestHandles { app_event_rx, bg_event_tx, subscriptions, .. } = handles;
17646 let completer = spawn_queued_event_test_completer(app_event_rx, Arc::new(AtomicUsize::new(0)));
17647 let tp = TopicPartition::new("t".to_string(), 0);
17648 assign_for_queued_event_test(&consumer, &tp);
17649 queue_background_error(&bg_event_tx, "queued background error");
17650
17651 let invoked = Arc::new(AtomicUsize::new(0));
17652 let result = consumer
17653 .commit_async_with_offsets_callback(
17654 singleton_offsets(tp, 5),
17655 Arc::new(CountingCallback { invoked: Arc::clone(&invoked) }),
17656 )
17657 .await;
17658 assert!(
17659 result.is_ok(),
17660 "commitAsync returned the queued background ErrorEvent ({:?}) after sending the commit; Java's \
17661 commitAsync does not run processBackgroundEvents()",
17662 result.err()
17663 );
17664
17665 // The commit completes on the fake bg task; its callback is queued by
17666 // the spawned continuation.
17667 let deadline = std::time::Instant::now() + Duration::from_secs(5);
17668 while consumer.commit_state.last_pending_async_commit().is_some_and(|c| !c.is_done()) {
17669 assert!(std::time::Instant::now() < deadline, "the commit never completed");
17670 tokio::time::sleep(Duration::from_millis(5)).await;
17671 }
17672 tokio::time::sleep(Duration::from_millis(20)).await;
17673
17674 let err = consumer
17675 .poll(Duration::ZERO)
17676 .await
17677 .expect_err("the next poll() must throw the queued background error");
17678 assert_eq!(err.message(), "LocalIllegalStateError: queued background error");
17679 assert_eq!(
17680 invoked.load(Ordering::SeqCst),
17681 1,
17682 "poll() runs the commit callback (executeCallbacks) before throwing the background error"
17683 );
17684 completer.abort();
17685 drop(subscriptions);
17686 }
17687
17688 /// `assign` with a queued `PartitionsAssignedEvent`: Java's `assign`
17689 /// waits with `addAndGet(AssignmentChangeEvent)`, so the queued
17690 /// assignment is neither applied (`ApplyAssignmentEvent`) nor delivered
17691 /// to `onPartitionsAssigned` by it; the next `poll()` does both.
17692 #[tokio::test]
17693 async fn assign_does_not_apply_a_queued_pending_assignment() {
17694 let (mut consumer, handles) = make_test_consumer_with_channels();
17695 let ConsumerTestHandles { app_event_rx, bg_event_tx, subscriptions, .. } = handles;
17696 let apply_assignments = Arc::new(AtomicUsize::new(0));
17697 let completer = spawn_queued_event_test_completer(app_event_rx, Arc::clone(&apply_assignments));
17698 let (_revoked, assigned) = install_counting_listener(&consumer);
17699 let tp = TopicPartition::new("t".to_string(), 0);
17700 // The fake bg task does not apply the `AssignmentChange`; seed the
17701 // assignment it would have applied, so `poll()` can run.
17702 assign_for_queued_event_test(&consumer, &tp);
17703 let stale = TopicPartition::new("t".to_string(), 1);
17704 let (ack_tx, mut ack_rx) = tokio::sync::oneshot::channel::<Result<(), Error>>();
17705 bg_event_tx
17706 .send(BackgroundEventEnvelope {
17707 event: BackgroundEvent::PartitionsAssigned {
17708 assigned_partitions: vec![stale.clone()],
17709 added_partitions: vec![stale],
17710 ack: ack_tx,
17711 },
17712 enqueued_ms: 0,
17713 })
17714 .expect("bg event sent");
17715
17716 consumer.assign(vec![tp]).await.expect("assign ok");
17717 assert_eq!(
17718 apply_assignments.load(Ordering::SeqCst),
17719 0,
17720 "assign applied a queued PartitionsAssignedEvent; Java's assign() never runs processBackgroundEvents()"
17721 );
17722 assert_eq!(assigned.load(Ordering::SeqCst), 0, "assign must not run onPartitionsAssigned");
17723 assert!(
17724 matches!(ack_rx.try_recv(), Err(tokio::sync::oneshot::error::TryRecvError::Empty)),
17725 "assign must leave the PartitionsAssignedEvent for poll()"
17726 );
17727
17728 consumer.poll(Duration::ZERO).await.expect("poll ok");
17729 assert_eq!(
17730 apply_assignments.load(Ordering::SeqCst),
17731 1,
17732 "poll() applies the queued assignment"
17733 );
17734 assert_eq!(assigned.load(Ordering::SeqCst), 1, "poll() runs onPartitionsAssigned once");
17735 assert!(ack_rx.await.expect("ack sent").is_ok());
17736 completer.abort();
17737 drop(subscriptions);
17738 }
17739
17740 /// Java's `close()` never runs `processBackgroundEvents()`: its steps are
17741 /// `commitSyncAllConsumed` (plain `commitSync`), `runRebalanceCallbacksOnClose`
17742 /// (the listener called directly) and `leaveGroupOnClose`
17743 /// (`applicationEventHandler.addAndGet`), in
17744 /// `AsyncKafkaConsumer#close(Duration,GroupMembershipOperation,boolean)`.
17745 /// A queued `ErrorEvent` is therefore never thrown by `close()`.
17746 #[tokio::test]
17747 async fn close_does_not_fail_with_queued_background_error() {
17748 let (mut consumer, handles) = make_test_consumer_with_channels();
17749 consumer.auto_commit_enabled = false;
17750 queue_background_error(&handles.bg_event_tx, "queued background error");
17751 let drainer = spawn_close_drainer(handles.app_event_rx);
17752 let result = consumer.close().await;
17753 drainer.abort();
17754 assert!(
17755 result.is_ok(),
17756 "close() failed with the queued background ErrorEvent ({:?}); Java's close() does not run \
17757 processBackgroundEvents(), so it does not throw it",
17758 result.err()
17759 );
17760 }
17761
17762 /// With a queued `ErrorEvent`, `close()` still waits for its
17763 /// `LeaveGroupOnClose` event, so the bg task sends the leave heartbeat
17764 /// before it is shut down: Java's `leaveGroupOnClose` blocks in
17765 /// `addAndGet` until the leave completes or the close timer expires
17766 /// (`AsyncKafkaConsumer#leaveGroupOnClose`). The fake bg task completes the
17767 /// leave only after a delay; `close()` must not return before it.
17768 #[tokio::test]
17769 async fn close_with_queued_background_error_still_waits_for_the_leave() {
17770 let (mut consumer, handles) = make_test_consumer_with_channels();
17771 consumer.auto_commit_enabled = false;
17772 queue_background_error(&handles.bg_event_tx, "queued background error");
17773 let leave_completed = Arc::new(AtomicBool::new(false));
17774 let mut app_event_rx = handles.app_event_rx;
17775 let drainer = {
17776 let leave_completed = Arc::clone(&leave_completed);
17777 tokio::spawn(async move {
17778 while let Some(env) = app_event_rx.recv().await {
17779 if let ApplicationEvent::LeaveGroupOnClose { handle, .. } = env.event {
17780 tokio::time::sleep(Duration::from_millis(100)).await;
17781 leave_completed.store(true, Ordering::SeqCst);
17782 handle.complete(());
17783 }
17784 }
17785 })
17786 };
17787
17788 let result = consumer.close().await;
17789 drainer.abort();
17790 assert!(result.is_ok(), "close() must succeed: {:?}", result.err());
17791 assert!(
17792 leave_completed.load(Ordering::SeqCst),
17793 "close() stopped waiting for LeaveGroupOnClose because of a queued background error; Java's \
17794 leaveGroupOnClose waits for the leave with addAndGet"
17795 );
17796 }
17797
17798 /// With auto-commit on, Java's `close()` commits through `commitSync`
17799 /// (`AsyncKafkaConsumer#commitSyncAllConsumed`), which
17800 /// does not run `processBackgroundEvents()`: a queued revoke callback is
17801 /// not invoked by it, and `runRebalanceCallbacksOnClose` invokes the
17802 /// listener only for the group assignment snapshot (empty here).
17803 #[tokio::test]
17804 async fn close_auto_commit_does_not_invoke_queued_rebalance_listener() {
17805 let (mut consumer, handles) = make_test_consumer_with_channels();
17806 consumer.auto_commit_enabled = true;
17807 let (revoked, _assigned) = install_counting_listener(&consumer);
17808 let _ack_rx = queue_revoke_callback(&handles.bg_event_tx, &TopicPartition::new("t".to_string(), 0));
17809 let drainer = spawn_close_drainer(handles.app_event_rx);
17810 let result = consumer.close().await;
17811 drainer.abort();
17812 assert!(result.is_ok(), "close() must succeed: {:?}", result.err());
17813 assert_eq!(
17814 revoked.load(Ordering::SeqCst),
17815 0,
17816 "close() invoked onPartitionsRevoked for a queued PartitionsRemovedEvent (through its auto-commit); \
17817 Java's close() commits with commitSync, which never runs processBackgroundEvents()"
17818 );
17819 }
17820}