Skip to main content

confluent_kafka/producer/
kafka_producer.rs

1// Copyright 2025 Confluent Inc.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15#![cfg_attr(not(test), expect(dead_code))]
16//! A Kafka client that publishes records to the Kafka cluster.
17//!
18//! Translated from `org.apache.kafka.clients.producer.KafkaProducer`.
19//!
20//! The producer is thread-safe and sharing a single producer instance across
21//! threads will generally be faster than having multiple instances.
22//!
23//! Transactional methods are translated: see [`KafkaProducer::init_transactions`]
24//! and its four siblings.
25
26use crate::common::requests::TxnOffsetCommitRequest;
27use std::collections::{HashMap, HashSet};
28use std::sync::atomic::{AtomicBool, Ordering};
29use std::sync::{Arc, Mutex};
30use std::time::Duration;
31
32use tokio::sync::Notify;
33use tokio::task::JoinHandle;
34
35use crate::ClientUtils;
36use crate::KafkaClient;
37use crate::Metadata;
38use crate::NetworkClient;
39use crate::common::Cluster;
40use crate::common::Error;
41use crate::common::KafkaFuture;
42use crate::common::PartitionInfo;
43use crate::common::TopicPartition;
44use crate::common::compress::Compression;
45use crate::common::errors::TimeoutError;
46use crate::common::header::Headers;
47use crate::common::header::RecordHeader;
48use crate::common::header::RecordHeaders;
49use crate::common::internals::ClusterResourceListeners;
50use crate::common::metrics::{MetricConfig, Metrics, MetricsView, RecordingLevel};
51use crate::common::network::ChannelBuilders;
52use crate::common::network::Selector;
53use crate::common::record::internal::AbstractRecords;
54use crate::common::record::internal::CompressionType;
55use crate::common::record::internal::RecordBatch;
56use crate::common::serialization::{DynSerializer, Serializer};
57use crate::common::utils::internals::LogContext;
58use crate::common::utils::{SystemTime, Time, Utils};
59use crate::consumer::OffsetAndMetadata;
60use crate::consumer::internals::panic_payload_message;
61use crate::consumer::{ConsumerGroupMetadata, ConsumerGroupMetadataImpl};
62use crate::producer::Callback;
63use crate::producer::Partitioner;
64use crate::producer::Producer;
65use crate::producer::ProducerConfig;
66use crate::producer::ProducerRecord;
67use crate::producer::RecordMetadata;
68use crate::producer::internals::BufferPool;
69use crate::producer::internals::Caller;
70use crate::producer::internals::ChunkedRecordAccumulator;
71use crate::producer::internals::FutureRecordMetadata;
72use crate::producer::internals::KafkaProducerMetrics;
73use crate::producer::internals::PendingRequests;
74use crate::producer::internals::ProducerMetadata;
75use crate::producer::internals::ProducerMetrics;
76use crate::producer::internals::Sender;
77use crate::producer::internals::SenderMetricsRegistry;
78use crate::producer::internals::SenderStatics;
79use crate::producer::internals::TransactionManager;
80use crate::producer::internals::buffer_pool::AllocationMode;
81use crate::producer::internals::{BuiltInPartitioner, KeyHasher, PartitionerConfig, RecordAccumulator};
82use crate::{ApiVersions, DefaultHostResolver};
83use crate::{kafka_debug, kafka_error, kafka_info, kafka_trace, kafka_warn};
84use futures_util::FutureExt;
85use std::panic::AssertUnwindSafe;
86
87/// Metadata and time spent waiting for it.
88#[derive(Debug)]
89#[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer$ClusterAndWaitTime")]
90struct ClusterAndWaitTime {
91    /// The cluster metadata.
92    cluster: Arc<Cluster>,
93    /// Time in ms spent waiting for metadata.
94    waited_on_metadata_ms: i64,
95}
96
97/// Whether a send may block before the record reaches the accumulator.
98///
99/// Java's `KafkaProducer.send` blocks for up to `max.block.ms` at two points
100/// before `RecordAccumulator.append` succeeds: `waitOnMetadata`
101/// (`KafkaProducer`) and `BufferPool.allocate`. This enum has no Java
102/// counterpart (DoD #7): it exists so the crate-internal
103/// [`KafkaProducer::try_send`] can share the whole `send` path and differ from
104/// it only at those two points.
105#[derive(Clone, Copy, Debug, PartialEq, Eq)]
106enum SendWait {
107    /// Block for up to `max.block.ms`, as Java does.
108    MaxBlock,
109    /// Never block: a wait at either point is reported as
110    /// [`TrySendOutcome::WouldBlock`], with the callback un-invoked.
111    Never,
112}
113
114/// The outcome of a non-blocking [`KafkaProducer::try_send`].
115///
116/// There is no Java counterpart (DoD #7). Java's `KafkaProducer.send` always
117/// blocks for up to `max.block.ms` at its two pre-append wait points —
118/// `waitOnMetadata` (`KafkaProducer`) and `BufferPool.allocate` inside
119/// `RecordAccumulator.append` — and a caller who cannot afford to block has no
120/// way to ask for anything else. The C FFI's non-blocking send needs exactly
121/// that: it appends on the caller's own thread when the producer is ready and
122/// must fall back to its submission queue, rather than stall that thread,
123/// whenever the producer would have to wait. This enum is how the producer
124/// reports which of the two happened. Crate-internal: the public surface stays
125/// Java's `send`.
126pub(crate) enum TrySendOutcome {
127    /// The record was handed to the producer and resolved the way `send` would
128    /// have resolved it: appended to a batch, or rejected with an `ApiException`
129    /// (a record too large, say) — in which case the future is already failed
130    /// and the callback has already fired, exactly as on `send`.
131    Accepted(KafkaFuture<RecordMetadata>),
132    /// The producer would have had to block — on metadata for the topic, or on
133    /// buffer memory — so it did nothing: no batch was touched, no future was
134    /// created and the callback was **not** invoked. The callback is handed back
135    /// so the caller can retry the record with `send` once it can afford to
136    /// wait.
137    WouldBlock(Option<Box<dyn Callback>>),
138}
139
140impl TrySendOutcome {
141    /// The blocking path's view of an outcome.
142    ///
143    /// [`SendWait::MaxBlock`] never produces [`WouldBlock`](Self::WouldBlock), so
144    /// the second arm is unreachable by construction; it is an `Err` rather than
145    /// a `panic!` per CLAUDE.md §12.1.
146    fn into_accepted_future(self) -> Result<KafkaFuture<RecordMetadata>, Error> {
147        match self {
148            TrySendOutcome::Accepted(future) => Ok(future),
149            TrySendOutcome::WouldBlock(_) => {
150                Err(Error::local_illegal_state("A blocking send reported that it would block"))
151            },
152        }
153    }
154}
155
156impl std::fmt::Debug for TrySendOutcome {
157    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
158        match self {
159            TrySendOutcome::Accepted(future) => f.debug_tuple("Accepted").field(future).finish(),
160            // A `Box<dyn Callback>` is not `Debug`, so only its presence is printable.
161            TrySendOutcome::WouldBlock(callback) => f
162                .debug_tuple("WouldBlock")
163                .field(&callback.as_ref().map(|_| "callback"))
164                .finish(),
165        }
166    }
167}
168
169/// A Kafka client that publishes records to the Kafka cluster.
170///
171/// The producer is thread-safe and sharing a single producer instance across
172/// tasks will generally be faster than having multiple instances.
173///
174/// The producer consists of a pool of buffer space that holds records that
175/// haven't yet been transmitted to the server, as well as a background I/O
176/// task that is responsible for turning these records into requests and
177/// transmitting them to the cluster. Failure to close the producer after use
178/// will leak these resources.
179///
180/// The [`send`](KafkaProducer::send) method is asynchronous. When called, it
181/// adds the record to a buffer of pending record sends and immediately returns.
182/// This allows the producer to batch together individual records for efficiency.
183#[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer")]
184pub struct KafkaProducer<K, V> {
185    /// The client ID used for this producer.
186    client_id: String,
187    /// The key serializer.
188    key_serializer: Box<dyn DynSerializer<K>>,
189    /// The value serializer.
190    value_serializer: Box<dyn DynSerializer<V>>,
191    /// Set by the first `close`, which closes both serializers: `close` is
192    /// idempotent, and a serializer's `close` runs at most once.
193    serializers_closed: AtomicBool,
194    /// The maximum size of a request in bytes.
195    max_request_size: i32,
196    /// The total memory size for the buffer pool.
197    total_memory_size: i64,
198    /// The record accumulator that batches records.
199    accumulator: Arc<RecordAccumulator>,
200    /// The incremental-strategy accumulator (KIP-1332), when `buffer.memory.allocation.strategy`
201    /// is `incremental` and `batch.size` admits it; its base is [`Self::accumulator`].
202    ///
203    /// Java holds one `accumulator` field typed `RecordAccumulator` and lets virtual dispatch pick
204    /// `ChunkedRecordAccumulator.append`. Rust composes the chunked accumulator over the base
205    /// (PLAN §2.3), so the one call site that differs — `append` on the send path — dispatches on
206    /// this field, and every other use keeps the shared base.
207    chunked_accumulator: Option<ChunkedRecordAccumulator>,
208    /// The producer metadata.
209    metadata: Arc<ProducerMetadata>,
210    /// All the state related to transactions, in particular the producer id,
211    /// producer epoch, and sequence numbers; `None` when idempotence is disabled.
212    ///
213    /// Translated from `KafkaProducer#transactionManager`, which is
214    /// nullable — hence [`Option`].
215    ///
216    /// Shared with the [`Sender`] task and the [`RecordAccumulator`] behind a
217    /// `std::sync::Mutex` (`.claude/rules/producer-transactions.md` §2 and
218    /// PLAN §6.3): the Sender is moved into a `tokio::task::spawn`, so this
219    /// struct cannot reach it any other way. No guard is ever held across an
220    /// `.await` (rules §4).
221    transaction_manager: Option<Arc<Mutex<TransactionManager>>>,
222    /// The queue of transactional requests waiting for the [`Sender`] to send them.
223    ///
224    /// Translated from `TransactionManager.pendingRequests`
225    /// (`TransactionManager#pendingRequests`). It lives outside the manager and is shared
226    /// with the `Sender` — see [`Sender::pending_requests`] for the full rationale
227    /// and for the lock order (`pending_requests` → `transaction_manager`) every
228    /// site below observes.
229    ///
230    /// Java's four transactional entry points on this class all enqueue into it from
231    /// the application thread, which is why the producer needs a handle at all. Cited
232    /// at the `transactionManager.<m>(..)` call statement in each, so the line and the
233    /// method cannot drift apart:
234    ///
235    /// | method | call statement | line |
236    /// |---|---|---|
237    /// | `initTransactions` | `initializeTransactions(false)` | 652 |
238    /// | `sendOffsetsToTransaction` | `sendOffsetsToTransaction(..)` | 740 |
239    /// | `commitTransaction` | `beginCommit()` | 783 |
240    /// | `abortTransaction` | `beginAbort()` | 818 |
241    ///
242    /// [`Sender::pending_requests`]: crate::producer::internals::Sender
243    pending_requests: Arc<Mutex<PendingRequests>>,
244    /// The compression type for records.
245    compression_type: CompressionType,
246    /// The maximum time to block on send/partitionsFor.
247    max_block_ms: i64,
248    /// The custom partitioner, if one is configured; `None` uses the built-in
249    /// default partitioner (the keyed CRC-32 / murmur2 path plus adaptive
250    /// partitioning).
251    ///
252    /// Translated from `KafkaProducer#partitionerPlugin`, a
253    /// `Plugin<Partitioner>`. The `Plugin<>` wrapper is `Monitorable`/metrics
254    /// plumbing (KIP-877) with no Rust counterpart in this milestone, so the
255    /// bare [`Partitioner`] trait object is held directly. Nullable in Java
256    /// (built-in partitioning when unset) — hence [`Option`]. Resolved from
257    /// `partitioner.type` via
258    /// [`ProducerConfig::resolve_partitioner`](crate::producer::ProducerConfig)
259    /// or set with
260    /// [`ProducerConfig::set_partitioner`](crate::producer::ProducerConfig::set_partitioner).
261    partitioner: Option<Box<dyn Partitioner<K, V>>>,
262    /// Whether to ignore keys for partitioning.
263    partitioner_ignore_keys: bool,
264    /// Which hash the keyed partition path uses, resolved from
265    /// `partitioner.type` (default [`KeyHasher::Crc32`], librdkafka
266    /// `consistent_random` parity). Copy-cheap; consulted once per keyed record.
267    key_hasher: KeyHasher,
268    /// Whether the sender task is still running.
269    running: Arc<AtomicBool>,
270    /// Whether the caller wants to force-close.
271    force_close: Arc<AtomicBool>,
272    /// Wakeup notification for the sender task.
273    wakeup: Arc<Notify>,
274    /// Handle to the sender background task.
275    /// Wrapped in `Mutex<Option<_>>` so `close_with_timeout` can take ownership
276    /// and `.await` it even though we only have `&self` (not `&mut self`).
277    sender_handle: Mutex<Option<JoinHandle<()>>>,
278    /// Java: `private final Time time`.
279    time: Arc<dyn Time>,
280    /// The metrics registry owned by this producer.
281    ///
282    /// Translated from Java's `Metrics metrics` field. Shared into
283    /// `producer_metrics`; exposed via [`metrics()`](Producer::metrics). Later
284    /// phases wire the Sender / BufferPool / RecordAccumulator sensors against
285    /// this same registry.
286    metrics: Arc<Metrics>,
287    /// Producer-level latency metrics (flush, metadata-wait, txn timings).
288    ///
289    /// Translated from Java's `KafkaProducerMetrics producerMetrics` field.
290    producer_metrics: KafkaProducerMetrics,
291    /// Contextual log message prefix.
292    ///
293    /// Translated from Java's `LogContext logContext` field in `KafkaProducer`.
294    log_context: LogContext,
295}
296
297impl<K, V> std::fmt::Debug for KafkaProducer<K, V> {
298    // The client id only, never the configuration.
299    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
300        f.debug_struct("KafkaProducer")
301            .field("client_id", &self.client_id)
302            .finish_non_exhaustive()
303    }
304}
305
306/// Parameters for [`KafkaProducer::with_options`].
307///
308/// This struct has **no Java counterpart** (DoD #7). It exists solely to satisfy
309/// CLAUDE.md §2's cap on derived overload names: Java's widest `KafkaProducer`
310/// constructor (`KafkaProducer#KafkaProducer`, marked `// visible for testing`)
311/// differs from the constructor group's parameter-name intersection —
312/// `{config, keySerializer, valueSerializer}`, which [`KafkaProducer::new`] owns
313/// — by ten parameters, far past the cap of three. So the derived name collapses
314/// to `with_options` and this struct becomes the method's *only* parameter,
315/// carrying every parameter including the intersection.
316///
317/// `config` is borrowed, not owned, exactly as the Java constructor borrows it:
318/// the producer reads it and stores only derived values.
319///
320/// Construct it with [`KafkaProducerOptionsBuilder::new`];
321/// [`KafkaProducerOptionsBuilder::build`] validates the mandatory parameters.
322#[non_exhaustive]
323pub(crate) struct KafkaProducerOptions<'a, K, V> {
324    /// The producer configuration. Java's `config`.
325    pub config: &'a ProducerConfig,
326    /// The key serializer. Java's `keySerializer`.
327    pub key_serializer: Box<dyn DynSerializer<K>>,
328    /// The value serializer. Java's `valueSerializer`.
329    pub value_serializer: Box<dyn DynSerializer<V>>,
330    /// The producer metadata. Java's `metadata`.
331    pub metadata: Arc<ProducerMetadata>,
332    /// The record accumulator. Java's `accumulator`. A producer built through these options
333    /// uses the full strategy: no caller (all of them tests) hands it a
334    /// `ChunkedRecordAccumulator`.
335    pub accumulator: Arc<RecordAccumulator>,
336    /// Whether the sender task is running. Part of Rust's decomposition of
337    /// Java's `Sender sender` / `Sender.SenderThread ioThread` pair; defaults to
338    /// `true`, the value [`KafkaProducer::with_client_options`] supplies itself.
339    pub running: Arc<AtomicBool>,
340    /// Whether force-close has been requested. Rust-side sender lifecycle, as
341    /// for [`Self::running`]; defaults to `false`.
342    pub force_close: Arc<AtomicBool>,
343    /// Notification that wakes the sender task. Rust-side sender lifecycle, as
344    /// for [`Self::running`]. Defaults to a *fresh* [`Notify`], attached to no
345    /// client — unlike [`KafkaProducer::with_client_options`], which takes this
346    /// from the client it was handed. A caller needing the producer and a client
347    /// to share one must set it.
348    pub wakeup: Arc<Notify>,
349    /// Handle to the sender background task, or `None` when no task was spawned.
350    /// Rust's counterpart of Java's `ioThread`, which the test-only constructor also
351    /// (`KafkaProducer#KafkaProducer(ProducerConfig,LogContext,Metrics,Serializer,Serializer,ProducerMetadata,RecordAccumulator,TransactionManager,Sender,ProducerInterceptors,Partitioner,Time,SenderThread,Optional)`)
352    /// accepts as a pre-built value.
353    pub sender_handle: Option<JoinHandle<()>>,
354    /// The clock. Java's `time`.
355    pub time: Arc<dyn Time>,
356    /// The shared transaction state, or `None` when idempotence is disabled.
357    /// Java's `transactionManager`, which that constructor also accepts as `null`.
358    pub transaction_manager: Option<Arc<Mutex<TransactionManager>>>,
359    /// The transactional request queue this producer shares with the [`Sender`].
360    pub pending_requests: Arc<Mutex<PendingRequests>>,
361    /// The custom partitioner instance, or `None` for the built-in default.
362    /// Java's `partitioner`. Stored as-is and **not** configured, mirroring
363    /// `KafkaProducer#KafkaProducer`, which wraps a pre-built `Partitioner` without
364    /// calling `configure`.
365    pub partitioner: Option<Box<dyn Partitioner<K, V>>>,
366}
367
368/// Fluent builder for [`KafkaProducerOptions`].
369///
370/// Per CLAUDE.md §2 [`Self::new`] takes no parameters, every parameter has a
371/// fluent setter, and [`Self::build`] validates the mandatory ones — returning
372/// [`Error::LocalIllegalArgument`] if they were not set. Like
373/// [`KafkaProducerOptions`] it has no Java counterpart and exists solely to
374/// satisfy that naming rule (DoD #7).
375pub(crate) struct KafkaProducerOptionsBuilder<'a, K, V> {
376    config: Option<&'a ProducerConfig>,
377    key_serializer: Option<Box<dyn DynSerializer<K>>>,
378    value_serializer: Option<Box<dyn DynSerializer<V>>>,
379    metadata: Option<Arc<ProducerMetadata>>,
380    accumulator: Option<Arc<RecordAccumulator>>,
381    running: Option<Arc<AtomicBool>>,
382    force_close: Option<Arc<AtomicBool>>,
383    wakeup: Option<Arc<Notify>>,
384    sender_handle: Option<JoinHandle<()>>,
385    time: Option<Arc<dyn Time>>,
386    transaction_manager: Option<Arc<Mutex<TransactionManager>>>,
387    pending_requests: Option<Arc<Mutex<PendingRequests>>>,
388    partitioner: Option<Box<dyn Partitioner<K, V>>>,
389}
390
391impl<K, V> Default for KafkaProducerOptionsBuilder<'_, K, V> {
392    fn default() -> Self {
393        Self::new()
394    }
395}
396
397impl<'a, K, V> KafkaProducerOptionsBuilder<'a, K, V> {
398    /// Creates a builder with every mandatory parameter unset and every other
399    /// parameter at the value the constructor supplies on its caller's behalf.
400    pub(crate) fn new() -> Self {
401        Self {
402            config: None,
403            key_serializer: None,
404            value_serializer: None,
405            metadata: None,
406            accumulator: None,
407            running: None,
408            force_close: None,
409            wakeup: None,
410            sender_handle: None,
411            time: None,
412            transaction_manager: None,
413            pending_requests: None,
414            partitioner: None,
415        }
416    }
417
418    /// Sets [`KafkaProducerOptions::config`], a mandatory parameter.
419    pub(crate) fn set_config(mut self, config: &'a ProducerConfig) -> Self {
420        self.config = Some(config);
421        self
422    }
423    /// Sets [`KafkaProducerOptions::key_serializer`], a mandatory parameter.
424    pub(crate) fn set_key_serializer(mut self, key_serializer: Box<dyn DynSerializer<K>>) -> Self {
425        self.key_serializer = Some(key_serializer);
426        self
427    }
428    /// Sets [`KafkaProducerOptions::value_serializer`], a mandatory parameter.
429    pub(crate) fn set_value_serializer(mut self, value_serializer: Box<dyn DynSerializer<V>>) -> Self {
430        self.value_serializer = Some(value_serializer);
431        self
432    }
433    /// Sets [`KafkaProducerOptions::metadata`], a mandatory parameter.
434    pub(crate) fn set_metadata(mut self, metadata: Arc<ProducerMetadata>) -> Self {
435        self.metadata = Some(metadata);
436        self
437    }
438    /// Sets [`KafkaProducerOptions::accumulator`], a mandatory parameter.
439    pub(crate) fn set_accumulator(mut self, accumulator: Arc<RecordAccumulator>) -> Self {
440        self.accumulator = Some(accumulator);
441        self
442    }
443    /// Sets [`KafkaProducerOptions::running`]; defaults to `true`.
444    pub(crate) fn set_running(mut self, running: Arc<AtomicBool>) -> Self {
445        self.running = Some(running);
446        self
447    }
448    /// Sets [`KafkaProducerOptions::force_close`]; defaults to `false`.
449    pub(crate) fn set_force_close(mut self, force_close: Arc<AtomicBool>) -> Self {
450        self.force_close = Some(force_close);
451        self
452    }
453    /// Sets [`KafkaProducerOptions::wakeup`]; defaults to a fresh [`Notify`].
454    pub(crate) fn set_wakeup(mut self, wakeup: Arc<Notify>) -> Self {
455        self.wakeup = Some(wakeup);
456        self
457    }
458    /// Sets [`KafkaProducerOptions::sender_handle`]; defaults to `None`.
459    pub(crate) fn set_sender_handle(mut self, sender_handle: Option<JoinHandle<()>>) -> Self {
460        self.sender_handle = sender_handle;
461        self
462    }
463    /// Sets [`KafkaProducerOptions::time`], a mandatory parameter.
464    pub(crate) fn set_time(mut self, time: Arc<dyn Time>) -> Self {
465        self.time = Some(time);
466        self
467    }
468    /// Sets [`KafkaProducerOptions::transaction_manager`]; defaults to `None`.
469    pub(crate) fn set_transaction_manager(
470        mut self,
471        transaction_manager: Option<Arc<Mutex<TransactionManager>>>,
472    ) -> Self {
473        self.transaction_manager = transaction_manager;
474        self
475    }
476    /// Sets [`KafkaProducerOptions::pending_requests`], a mandatory parameter.
477    pub(crate) fn set_pending_requests(mut self, pending_requests: Arc<Mutex<PendingRequests>>) -> Self {
478        self.pending_requests = Some(pending_requests);
479        self
480    }
481    /// Sets [`KafkaProducerOptions::partitioner`]; defaults to `None`.
482    pub(crate) fn set_partitioner(mut self, partitioner: Option<Box<dyn Partitioner<K, V>>>) -> Self {
483        self.partitioner = partitioner;
484        self
485    }
486
487    /// Returns the built options.
488    ///
489    /// Per CLAUDE.md §2 the mandatory parameters are validated here rather than
490    /// being named in the constructor, so a later Java version that makes one of
491    /// them optional changes the set this accepts instead of adding a second
492    /// constructor. Today there is one mandatory set: `config`,
493    /// `key_serializer`, `value_serializer`, `metadata`, `accumulator`,
494    /// `time` and `pending_requests`.
495    ///
496    /// The other six are not in it for two distinct reasons. `sender_handle`,
497    /// `transaction_manager` and `partitioner` are the parameters Java's test-only constructor
498    /// itself accepts as `null`. `running`, `force_close` and `wakeup` have no
499    /// Java counterpart at all — they are Rust's decomposition of Java's
500    /// `Sender` / `SenderThread` pair, and are values a constructor supplies on
501    /// the caller's behalf rather than values the caller must choose:
502    /// [`KafkaProducer::with_client_options`] builds `running` and `force_close`
503    /// with exactly these defaults itself, and derives `wakeup` from the client
504    /// it was handed.
505    ///
506    /// `pending_requests` is deliberately mandatory although a fresh queue would
507    /// be a plausible default: it is shared with the [`Sender`], and defaulting
508    /// it would silently hand back a producer whose transactional requests
509    /// nobody drains.
510    ///
511    /// # Errors
512    ///
513    /// Returns [`Error::LocalIllegalArgument`] naming the first parameter of the
514    /// mandatory set which was not given a setter call. Only presence is checked
515    /// here; semantic validation belongs to the method the options are passed to
516    /// (CLAUDE.md §2).
517    pub(crate) fn build(self) -> Result<KafkaProducerOptions<'a, K, V>, Error> {
518        Ok(KafkaProducerOptions {
519            config: self.config.ok_or_else(|| Self::missing("config"))?,
520            key_serializer: self.key_serializer.ok_or_else(|| Self::missing("key_serializer"))?,
521            value_serializer: self.value_serializer.ok_or_else(|| Self::missing("value_serializer"))?,
522            metadata: self.metadata.ok_or_else(|| Self::missing("metadata"))?,
523            accumulator: self.accumulator.ok_or_else(|| Self::missing("accumulator"))?,
524            running: self.running.unwrap_or_else(|| Arc::new(AtomicBool::new(true))),
525            force_close: self.force_close.unwrap_or_else(|| Arc::new(AtomicBool::new(false))),
526            wakeup: self.wakeup.unwrap_or_else(|| Arc::new(Notify::new())),
527            sender_handle: self.sender_handle,
528            time: self.time.ok_or_else(|| Self::missing("time"))?,
529            transaction_manager: self.transaction_manager,
530            pending_requests: self.pending_requests.ok_or_else(|| Self::missing("pending_requests"))?,
531            partitioner: self.partitioner,
532        })
533    }
534
535    /// Builds the [`Error::LocalIllegalArgument`] naming a mandatory parameter
536    /// [`Self::build`] found unset.
537    fn missing(parameter: &str) -> Error {
538        Error::local_illegal_argument(format!(
539            "KafkaProducerOptionsBuilder::build: mandatory parameter `{parameter}` was not set"
540        ))
541    }
542}
543
544/// Parameters for [`KafkaProducer::with_client_options`].
545///
546/// This struct has **no Java counterpart** (DoD #7), and exists for the same
547/// reason as [`KafkaProducerOptions`]: Java's other `// visible for testing`
548/// constructor (`KafkaProducer#KafkaProducer`) differs from the group's
549/// parameter-name intersection by ten parameters, past CLAUDE.md §2's cap of
550/// three, so it is served by an options struct rather than by a name listing
551/// every parameter.
552///
553/// Note the method it parameterizes is `with_client_options`, not the bare
554/// `with_options` §2's rule would literally derive: both widest Java
555/// constructors collapse to the same derived name, so the one discriminating
556/// parameter that distinguishes this overload from the test-only one — Java's
557/// `kafkaClient` — is kept in the name.
558#[non_exhaustive]
559pub(crate) struct KafkaProducerClientOptions<'a, K, V, C> {
560    /// The producer configuration. Java's `config`.
561    pub config: &'a ProducerConfig,
562    /// The key serializer. Java's `keySerializer`.
563    pub key_serializer: Box<dyn DynSerializer<K>>,
564    /// The value serializer. Java's `valueSerializer`.
565    pub value_serializer: Box<dyn DynSerializer<V>>,
566    /// The producer metadata. Java's `metadata`.
567    pub metadata: Arc<ProducerMetadata>,
568    /// The record accumulator. Java has no `accumulator` parameter on `KafkaProducer#KafkaProducer(ProducerConfig,Serializer,Serializer,ProducerMetadata,KafkaClient,ProducerInterceptors,ApiVersions,Time)`;
569    /// Rust builds it in [`KafkaProducer::new_inner`] and injects it here.
570    pub accumulator: Arc<RecordAccumulator>,
571    /// The incremental-strategy accumulator over `accumulator`, or `None` for the full strategy
572    /// (see `KafkaProducer::chunked_accumulator`).
573    pub chunked_accumulator: Option<ChunkedRecordAccumulator>,
574    /// The network client the sender task drives. Java's `kafkaClient`.
575    pub client: C,
576    /// The clock. Java's `time`.
577    pub time: Arc<dyn Time>,
578    /// The producer's metrics registry.
579    pub metrics: Arc<Metrics>,
580    /// The producer-level metrics wrapper.
581    pub producer_metrics: KafkaProducerMetrics,
582    /// The sender-level metrics registry.
583    pub sender_metrics_registry: SenderMetricsRegistry,
584    /// The shared transaction state, or `None` when idempotence is disabled.
585    pub transaction_manager: Option<Arc<Mutex<TransactionManager>>>,
586    /// The transactional request queue this producer shares with the [`Sender`].
587    pub pending_requests: Arc<Mutex<PendingRequests>>,
588    /// The custom partitioner instance (already `configure`d by the caller), or
589    /// `None` for the built-in default partitioner. Callers that resolve a
590    /// partitioner also gate adaptive partitioning on its absence before
591    /// building the `accumulator` they pass in (see `KafkaProducer::new`).
592    pub partitioner: Option<Box<dyn Partitioner<K, V>>>,
593}
594
595/// Fluent builder for [`KafkaProducerClientOptions`].
596///
597/// Per CLAUDE.md §2 [`Self::new`] takes no parameters, every parameter has a
598/// fluent setter, and [`Self::build`] validates the mandatory ones — returning
599/// [`Error::LocalIllegalArgument`] if they were not set.
600pub(crate) struct KafkaProducerClientOptionsBuilder<'a, K, V, C> {
601    config: Option<&'a ProducerConfig>,
602    key_serializer: Option<Box<dyn DynSerializer<K>>>,
603    value_serializer: Option<Box<dyn DynSerializer<V>>>,
604    metadata: Option<Arc<ProducerMetadata>>,
605    accumulator: Option<Arc<RecordAccumulator>>,
606    chunked_accumulator: Option<ChunkedRecordAccumulator>,
607    client: Option<C>,
608    time: Option<Arc<dyn Time>>,
609    metrics: Option<Arc<Metrics>>,
610    producer_metrics: Option<KafkaProducerMetrics>,
611    sender_metrics_registry: Option<SenderMetricsRegistry>,
612    transaction_manager: Option<Arc<Mutex<TransactionManager>>>,
613    pending_requests: Option<Arc<Mutex<PendingRequests>>>,
614    partitioner: Option<Box<dyn Partitioner<K, V>>>,
615}
616
617impl<K, V, C> Default for KafkaProducerClientOptionsBuilder<'_, K, V, C> {
618    fn default() -> Self {
619        Self::new()
620    }
621}
622
623impl<'a, K, V, C> KafkaProducerClientOptionsBuilder<'a, K, V, C> {
624    /// Creates a builder with every mandatory parameter unset and every other
625    /// parameter at the value the constructor supplies on its caller's behalf.
626    pub(crate) fn new() -> Self {
627        Self {
628            config: None,
629            key_serializer: None,
630            value_serializer: None,
631            metadata: None,
632            accumulator: None,
633            chunked_accumulator: None,
634            client: None,
635            time: None,
636            metrics: None,
637            producer_metrics: None,
638            sender_metrics_registry: None,
639            transaction_manager: None,
640            pending_requests: None,
641            partitioner: None,
642        }
643    }
644
645    /// Sets [`KafkaProducerClientOptions::config`], a mandatory parameter.
646    pub(crate) fn set_config(mut self, config: &'a ProducerConfig) -> Self {
647        self.config = Some(config);
648        self
649    }
650    /// Sets [`KafkaProducerClientOptions::key_serializer`], a mandatory parameter.
651    pub(crate) fn set_key_serializer(mut self, key_serializer: Box<dyn DynSerializer<K>>) -> Self {
652        self.key_serializer = Some(key_serializer);
653        self
654    }
655    /// Sets [`KafkaProducerClientOptions::value_serializer`], a mandatory parameter.
656    pub(crate) fn set_value_serializer(mut self, value_serializer: Box<dyn DynSerializer<V>>) -> Self {
657        self.value_serializer = Some(value_serializer);
658        self
659    }
660    /// Sets [`KafkaProducerClientOptions::metadata`], a mandatory parameter.
661    pub(crate) fn set_metadata(mut self, metadata: Arc<ProducerMetadata>) -> Self {
662        self.metadata = Some(metadata);
663        self
664    }
665    /// Sets [`KafkaProducerClientOptions::accumulator`], a mandatory parameter.
666    pub(crate) fn set_accumulator(mut self, accumulator: Arc<RecordAccumulator>) -> Self {
667        self.accumulator = Some(accumulator);
668        self
669    }
670    /// Sets [`KafkaProducerClientOptions::chunked_accumulator`]; `None` (the full strategy) if
671    /// unset.
672    pub(crate) fn set_chunked_accumulator(mut self, chunked_accumulator: Option<ChunkedRecordAccumulator>) -> Self {
673        self.chunked_accumulator = chunked_accumulator;
674        self
675    }
676    /// Sets [`KafkaProducerClientOptions::client`], a mandatory parameter.
677    pub(crate) fn set_client(mut self, client: C) -> Self {
678        self.client = Some(client);
679        self
680    }
681    /// Sets [`KafkaProducerClientOptions::time`], a mandatory parameter.
682    pub(crate) fn set_time(mut self, time: Arc<dyn Time>) -> Self {
683        self.time = Some(time);
684        self
685    }
686    /// Sets [`KafkaProducerClientOptions::metrics`], a mandatory parameter.
687    pub(crate) fn set_metrics(mut self, metrics: Arc<Metrics>) -> Self {
688        self.metrics = Some(metrics);
689        self
690    }
691    /// Sets [`KafkaProducerClientOptions::producer_metrics`], a mandatory parameter.
692    pub(crate) fn set_producer_metrics(mut self, producer_metrics: KafkaProducerMetrics) -> Self {
693        self.producer_metrics = Some(producer_metrics);
694        self
695    }
696    /// Sets [`KafkaProducerClientOptions::sender_metrics_registry`], a mandatory parameter.
697    pub(crate) fn set_sender_metrics_registry(mut self, sender_metrics_registry: SenderMetricsRegistry) -> Self {
698        self.sender_metrics_registry = Some(sender_metrics_registry);
699        self
700    }
701    /// Sets [`KafkaProducerClientOptions::transaction_manager`]; defaults to `None`.
702    pub(crate) fn set_transaction_manager(
703        mut self,
704        transaction_manager: Option<Arc<Mutex<TransactionManager>>>,
705    ) -> Self {
706        self.transaction_manager = transaction_manager;
707        self
708    }
709    /// Sets [`KafkaProducerClientOptions::pending_requests`], a mandatory parameter.
710    pub(crate) fn set_pending_requests(mut self, pending_requests: Arc<Mutex<PendingRequests>>) -> Self {
711        self.pending_requests = Some(pending_requests);
712        self
713    }
714    /// Sets [`KafkaProducerClientOptions::partitioner`]; defaults to `None`.
715    pub(crate) fn set_partitioner(mut self, partitioner: Option<Box<dyn Partitioner<K, V>>>) -> Self {
716        self.partitioner = partitioner;
717        self
718    }
719
720    /// Returns the built options.
721    ///
722    /// Per CLAUDE.md §2 the mandatory parameters are validated here rather than
723    /// being named in the constructor. Every parameter is mandatory except
724    /// `transaction_manager` and `partitioner`, the two that Java's two
725    /// package-private constructors also accept as `null`.
726    ///
727    /// # Errors
728    ///
729    /// Returns [`Error::LocalIllegalArgument`] naming the first parameter of the
730    /// mandatory set which was not given a setter call. Only presence is checked
731    /// here; semantic validation belongs to the method the options are passed to
732    /// (CLAUDE.md §2).
733    pub(crate) fn build(self) -> Result<KafkaProducerClientOptions<'a, K, V, C>, Error> {
734        Ok(KafkaProducerClientOptions {
735            config: self.config.ok_or_else(|| Self::missing("config"))?,
736            key_serializer: self.key_serializer.ok_or_else(|| Self::missing("key_serializer"))?,
737            value_serializer: self.value_serializer.ok_or_else(|| Self::missing("value_serializer"))?,
738            metadata: self.metadata.ok_or_else(|| Self::missing("metadata"))?,
739            accumulator: self.accumulator.ok_or_else(|| Self::missing("accumulator"))?,
740            chunked_accumulator: self.chunked_accumulator,
741            client: self.client.ok_or_else(|| Self::missing("client"))?,
742            time: self.time.ok_or_else(|| Self::missing("time"))?,
743            metrics: self.metrics.ok_or_else(|| Self::missing("metrics"))?,
744            producer_metrics: self.producer_metrics.ok_or_else(|| Self::missing("producer_metrics"))?,
745            sender_metrics_registry: self
746                .sender_metrics_registry
747                .ok_or_else(|| Self::missing("sender_metrics_registry"))?,
748            transaction_manager: self.transaction_manager,
749            pending_requests: self.pending_requests.ok_or_else(|| Self::missing("pending_requests"))?,
750            partitioner: self.partitioner,
751        })
752    }
753
754    /// Builds the [`Error::LocalIllegalArgument`] naming a mandatory parameter
755    /// [`Self::build`] found unset.
756    fn missing(parameter: &str) -> Error {
757        Error::local_illegal_argument(format!(
758            "KafkaProducerClientOptionsBuilder::build: mandatory parameter `{parameter}` was not set"
759        ))
760    }
761}
762
763impl<K, V> KafkaProducer<K, V> {
764    /// Network thread name prefix.
765    pub const NETWORK_THREAD_PREFIX: &str = "kafka-producer-network-thread";
766
767    /// Producer metric group name.
768    pub const PRODUCER_METRIC_GROUP_NAME: &str = "producer-metrics";
769
770    /// Timeout reason appended to the [`Error::Timeout`] message when
771    /// [`KafkaProducer::init_transactions`] does not complete within `max.block.ms`.
772    const INIT_TXN_TIMEOUT_MSG: &str = "InitTransactions timed out - did not complete coordinator discovery or \
773         receive the InitProducerId response within max.block.ms.";
774
775    /// Timeout reason appended when [`KafkaProducer::send_offsets_to_transaction`]
776    /// does not complete within `max.block.ms`.
777    const SEND_OFFSETS_TIMEOUT_MSG: &str = "SendOffsetsToTransaction timed out - did not reach the coordinator or \
778         receive the TxnOffsetCommit/AddOffsetsToTxn response within max.block.ms";
779
780    /// Timeout reason appended when [`KafkaProducer::commit_transaction`] does not
781    /// complete within `max.block.ms`.
782    const COMMIT_TXN_TIMEOUT_MSG: &str =
783        "CommitTransaction timed out - did not complete EndTxn with the transaction coordinator within max.block.ms";
784
785    /// Timeout reason appended when [`KafkaProducer::abort_transaction`] does not
786    /// complete within `max.block.ms`.
787    const ABORT_TXN_TIMEOUT_MSG: &str = "AbortTransaction timed out - did not complete EndTxn(abort) with the transaction coordinator within max.block.ms";
788
789    /// Creates a `KafkaProducer` from individual pre-built components.
790    ///
791    /// Translates Java's `KafkaProducer(ProducerConfig, LogContext, Metrics,
792    /// Serializer, Serializer, ProducerMetadata, RecordAccumulator,
793    /// TransactionManager, Sender, ProducerInterceptors, Partitioner, Time,
794    /// Sender.SenderThread, Optional<ClientTelemetryReporter>)`
795    /// (`KafkaProducer#KafkaProducer`), which Java marks `// visible for testing`.
796    /// This is the injection seam tests use when they supply every collaborator
797    /// themselves; [`Self::new`] is the user-facing constructor.
798    ///
799    /// `pub(crate)`, matching Java's package-private visibility: the parameters
800    /// name `ProducerMetadata`, `RecordAccumulator`, `TransactionManager` and
801    /// `PendingRequests`, all `pub(crate)` under `producer::internals`, so an
802    /// external caller could never have named or constructed them even while
803    /// this method was nominally `pub`.
804    ///
805    /// # Arguments
806    ///
807    /// * `options` - Every parameter, built through
808    ///   [`KafkaProducerOptionsBuilder`]. [`KafkaProducerOptions`] is this
809    ///   method's only parameter because the derived name would list ten
810    ///   parameters, past CLAUDE.md §2's cap of three.
811    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#KafkaProducer")]
812    pub(crate) fn with_options(options: KafkaProducerOptions<'_, K, V>) -> Self {
813        let KafkaProducerOptions {
814            config,
815            key_serializer,
816            value_serializer,
817            metadata,
818            accumulator,
819            running,
820            force_close,
821            wakeup,
822            sender_handle,
823            time,
824            transaction_manager,
825            pending_requests,
826            partitioner,
827        } = options;
828        let log_context = LogContext::new(format!("[Producer clientId={}] ", config.client_id));
829        let (metrics, producer_metrics) = Self::create_metrics(config);
830        Self {
831            client_id: config.client_id.clone(),
832            key_serializer,
833            value_serializer,
834            serializers_closed: AtomicBool::new(false),
835            max_request_size: config.max_request_size,
836            total_memory_size: config.buffer_memory,
837            accumulator,
838            chunked_accumulator: None,
839            metadata,
840            transaction_manager,
841            pending_requests,
842            compression_type: config.compression_type,
843            max_block_ms: config.max_block_ms,
844            partitioner,
845            partitioner_ignore_keys: config.partitioner_ignore_keys,
846            key_hasher: config.key_hasher(),
847            running,
848            force_close,
849            wakeup,
850            sender_handle: Mutex::new(sender_handle),
851            time,
852            metrics,
853            producer_metrics,
854            log_context,
855        }
856    }
857
858    // Java's no-serializer constructors, `KafkaProducer(Map)` and
859    // `KafkaProducer(Properties)`, are deliberately not translated: they delegate
860    // to `this(configs, null, null)` so that the private constructor can fill a
861    // `null` serializer in reflectively from the `key.serializer` /
862    // `value.serializer` `Type.CLASS` entries. Rust has no reflection, and unlike
863    // `partitioner.type` (mapped to built-in types by
864    // `ProducerConfig::resolve_partitioner`) a serializer is typed in the
865    // producer's own `K` / `V`, so no such mapping can be written for an arbitrary
866    // `K`. `ProducerConfig` therefore carries no serializer state, and a
867    // `key.serializer` entry is ignored as an unknown key.
868    //
869    // The Java group's parameter intersection is `{configs}`, whose overload is
870    // untranslatable, so the plain name `new` stays on this, the only
871    // general-purpose constructor (as `AsyncKafkaConsumer::new` does).
872    //
873    // It wires up all infrastructure components: it resolves the bootstrap
874    // addresses, creates `ProducerMetadata`, the channel builder, `Selector` and
875    // `NetworkClient`, the `BufferPool` and `RecordAccumulator`, and spawns the
876    // background sender task via the crate-internal `with_client_options`.
877    /// A producer is instantiated by providing a set of key-value pairs as
878    /// configuration, a key and a value [`Serializer`].
879    ///
880    /// Note: after creating a `KafkaProducer` you must always
881    /// [`close`](Producer::close) it to avoid resource leaks.
882    ///
883    /// # Arguments
884    ///
885    /// - `config` - The producer configs
886    /// - `key_serializer` - The serializer for key that implements [`Serializer`]. The
887    ///   `configure` method won't be called in the producer when the serializer is passed in
888    ///   directly.
889    /// - `value_serializer` - The serializer for value that implements [`Serializer`]. The
890    ///   `configure` method won't be called in the producer when the serializer is passed in
891    ///   directly.
892    ///
893    /// # Errors
894    ///
895    /// - `Error::is_kafka_error` if the producer cannot be constructed; the cause is its
896    ///   source.
897    ///
898    /// # Examples
899    ///
900    /// ```no_run
901    /// use std::collections::HashMap;
902    /// use confluent_kafka::producer::KafkaProducer;
903    /// use confluent_kafka::producer::ProducerConfig;
904    /// use confluent_kafka::common::serialization::StringSerializer;
905    ///
906    /// let props = HashMap::from([
907    ///     ("bootstrap.servers".to_string(), "localhost:9092".to_string()),
908    /// ]);
909    /// let config = ProducerConfig::new(&props).expect("Invalid config");
910    ///
911    /// let producer = KafkaProducer::<String, String>::new(
912    ///     config,
913    ///     StringSerializer::default(),
914    ///     StringSerializer::default(),
915    /// ).expect("Failed to create producer");
916    /// ```
917    ///
918    /// # Translation notes
919    ///
920    /// Java's constructors without serializers are not translated: Rust cannot
921    /// instantiate the class named by `key.serializer`.
922    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#KafkaProducer(Map,Serializer,Serializer)")]
923    pub fn new(
924        config: ProducerConfig,
925        key_serializer: impl Serializer<K> + 'static,
926        value_serializer: impl Serializer<V> + 'static,
927    ) -> Result<Self, Error>
928    where
929        K: 'static,
930        V: 'static,
931    {
932        Self::with_boxed_serializers(config, Box::new(key_serializer), Box::new(value_serializer))
933    }
934
935    /// [`new`](Self::new) over serializers already boxed as `dyn`.
936    ///
937    /// For the C binding, whose serializers are trait objects before they reach
938    /// the producer: boxing them as `dyn` a second time would nest one
939    /// `InlineFuture` in another, which no longer fits its buffer, so every
940    /// record would allocate on the send path (CLAUDE.md §13).
941    pub(crate) fn with_boxed_serializers(
942        config: ProducerConfig,
943        key_serializer: Box<dyn DynSerializer<K>>,
944        value_serializer: Box<dyn DynSerializer<V>>,
945    ) -> Result<Self, Error>
946    where
947        K: 'static,
948        V: 'static,
949    {
950        // Java wraps the whole constructor body in `catch (Throwable t)` and
951        // relabels every failure (`KafkaProducer#KafkaProducer`):
952        //
953        //     throw new KafkaException("Failed to construct kafka producer", t);
954        //
955        // so a caller has exactly one class to guard construction with, whatever
956        // went wrong inside. Without this, a bad `ssl.truststore.location` reached
957        // the caller as something for which `is_kafka_error()` is `false`.
958        //
959        // The `close(Duration.ofMillis(0), true)` half of Java's catch (KAFKA-2121)
960        // matters for the serializers and the partitioner only. No socket and no
961        // spawned task can leak: the sender task is spawned by the final, infallible
962        // step, and an unconnected `Selector` / `NetworkClient` dropped on an error
963        // owns no socket. But the serializers (`KafkaProducer`) and a
964        // configured custom partitioner (`KafkaProducer`) must be closed: `new_inner` closes
965        // the serializers on every failure (`CloseSerializersOnError`) and the
966        // partitioner on every fallible step after it is configured.
967        Self::new_inner(config, key_serializer, value_serializer)
968            .map_err(|e| Error::kafka_message_source("Failed to construct kafka producer", e))
969    }
970
971    /// The body of [`new`](Self::new) and [`with_boxed_serializers`](Self::with_boxed_serializers).
972    ///
973    /// The partitioner is resolved from `partitioner.type` (a built-in name, or
974    /// the partitioner set with
975    /// [`ProducerConfig::set_partitioner`](crate::producer::ProducerConfig::set_partitioner))
976    /// via `ProducerConfig::resolve_partitioner`. A resolved partitioner is
977    /// `configure`d exactly once here (`originals` + `client.id`, Java
978    /// `KafkaProducer#KafkaProducer`), and adaptive partitioning is gated on its
979    /// absence. Errors escape raw from here; `new` relabels them
980    /// (`KafkaProducer#KafkaProducer`).
981    fn new_inner(
982        mut config: ProducerConfig,
983        key_serializer: Box<dyn DynSerializer<K>>,
984        value_serializer: Box<dyn DynSerializer<V>>,
985    ) -> Result<Self, Error>
986    where
987        K: 'static,
988        V: 'static,
989    {
990        // Java's `ProducerConfig` is built with `doLog = true`, which logs
991        // every parsed value (`AbstractConfig.logAll`).
992        config.log_values();
993        let log_context = LogContext::new(format!("[Producer clientId={}] ", config.client_id));
994
995        kafka_trace!(log_context, "Starting the Kafka producer");
996
997        // From here on, a failed construction closes the serializers (Java's
998        // catch, `KafkaProducer#KafkaProducer`); see `CloseSerializersOnError`.
999        let serializers = CloseSerializersOnError::new(key_serializer, value_serializer, log_context.clone());
1000
1001        // 1. (KIP-909) Bootstrap address resolution moved below, to
1002        //    `ClientUtils::maybe_bootstrap_metadata_synchronously` right after the
1003        //    metadata is created, as in Java 4.4.
1004
1005        // 2. Validate delivery timeout configuration
1006        //    Translated from KafkaProducer.configureDeliveryTimeout().
1007        let delivery_timeout_ms = Self::configure_delivery_timeout(&config, &log_context)?;
1008
1009        // The MILESTONE-11 GUARD that used to sit here is gone. Its idempotence arm
1010        // was removed in Phase 4, when `enable.idempotence` began to be honoured for
1011        // real; its transactional arm is removed here, now that
1012        // `init_transactions` / `begin_transaction` / `send_offsets_to_transaction` /
1013        // `commit_transaction` / `abort_transaction` are implemented. PLAN §7.1 named
1014        // this removal as an explicit Phase-6 deliverable, so nothing is left behind
1015        // (CLAUDE.md §7).
1016
1017        // 3. Derive compression from config
1018        //    Translated from KafkaProducer.configureCompression().
1019        let compression = Self::configure_compression(&config)?;
1020
1021        // 4. The system clock, Java's `Time.SYSTEM`
1022        let time: Arc<dyn Time> = Arc::new(SystemTime);
1023
1024        // 5. Create ProducerMetadata. With `bootstrap.resolve.timeout.ms=0` (the
1025        //    default) bootstrap it synchronously from `bootstrap.servers`, so an
1026        //    unresolvable address fails construction (`ConfigException`); with a
1027        //    positive value the NetworkClient resolves them asynchronously
1028        //    (KIP-909, `KafkaProducer#KafkaProducer`).
1029        let metadata = Arc::new(ProducerMetadata::with_log_context(
1030            config.retry_backoff_ms,
1031            config.retry_backoff_max_ms,
1032            config.metadata_max_age_ms,
1033            config.metadata_max_idle_ms,
1034            ClusterResourceListeners::new(),
1035            log_context.clone(),
1036        ));
1037        ClientUtils::maybe_bootstrap_metadata_synchronously(
1038            config.bootstrap_resolve_timeout_ms,
1039            config.client_dns_lookup,
1040            &config.bootstrap_servers,
1041            &metadata,
1042        )?;
1043
1044        // 6. Get the shared Metadata Arc from ProducerMetadata so the NetworkClient
1045        //    uses the same Metadata instance. This mirrors Java's inheritance where
1046        //    ProducerMetadata extends Metadata.
1047        let shared_metadata = metadata.metadata_arc();
1048
1049        // 7. Create Selector + NetworkClient
1050        let channel_builder = ChannelBuilders::client_channel_builder(
1051            config.security_protocol,
1052            Some(&config.ssl_config),
1053            Some(&config.sasl_config),
1054            None,
1055            &config.client_id,
1056            log_context.clone(),
1057        )
1058        // Java's reachable failure here is `SslFactory.configure` throwing
1059        // `ConfigException` (`SslFactory#configure`) — the missing-argument
1060        // `IllegalArgumentException`s in `ChannelBuilders.create` cannot be reached
1061        // from a `ProducerConfig`, which always supplies both sub-configs. So the
1062        // class is `ConfigException`, inside the `KafkaException` hierarchy;
1063        // `illegal_argument` put it outside, where `is_kafka_error()` is `false`.
1064        .map_err(|e| Error::config_message(format!("Failed to create channel builder: {}", e)))?;
1065        let mut selector = Selector::with_defaults_and_log_context(
1066            config.connections_max_idle_ms,
1067            channel_builder,
1068            log_context.clone(),
1069        );
1070        selector.set_time(Arc::clone(&time));
1071        let api_versions = Arc::new(ApiVersions::new());
1072
1073        let mut client = NetworkClient::with_metadata_rebootstrap_trigger_ms(
1074            selector,
1075            shared_metadata,
1076            &config.client_id,
1077            config.max_in_flight_requests_per_connection as usize,
1078            config.reconnect_backoff_ms,
1079            config.reconnect_backoff_max_ms,
1080            config.send_buffer_bytes,
1081            config.receive_buffer_bytes,
1082            config.request_timeout_ms,
1083            config.socket_connection_setup_timeout_ms,
1084            config.socket_connection_setup_timeout_max_ms,
1085            true, // discover_broker_versions
1086            Arc::clone(&api_versions),
1087            DefaultHostResolver::new(),
1088            // `ClientUtils.createNetworkClient` reads
1089            // both from the config; the default strategy is `rebootstrap`.
1090            config.metadata_recovery_rebootstrap_trigger_ms,
1091            config.metadata_recovery_strategy,
1092            log_context.clone(),
1093        );
1094        // `ClientUtils.createNetworkClient` passes `metadata.cluster.check.enable`
1095        // to the `NetworkClient` constructor (`ClientUtils#createNetworkClient`, KIP-1242).
1096        client.set_metadata_cluster_check_enable(config.metadata_cluster_check_enable);
1097        // `ClientUtils.createNetworkClient` passes
1098        // `bootstrapConfiguration(config, bootstrap.servers)` to the
1099        // `NetworkClient` constructor (KIP-909).
1100        client.set_bootstrap_configuration(ClientUtils::bootstrap_configuration(
1101            config.bootstrap_resolve_timeout_ms,
1102            config.client_dns_lookup,
1103            config.retry_backoff_ms,
1104            &config.bootstrap_servers,
1105        )?);
1106
1107        // 8. Create the metrics registry. Java creates `this.metrics` early in
1108        //    the constructor (`KafkaProducer#KafkaProducer`), before the
1109        //    `RecordAccumulator`/`BufferPool` (`KafkaProducer#KafkaProducer`), because both are
1110        //    handed the same `Metrics` instance.
1111        let (metrics, producer_metrics) = Self::create_metrics(&config);
1112
1113        // 9. Create the TransactionManager, before the accumulator and the Sender
1114        //    because both of them need it (PLAN §6.3). Java's field assignment sits
1115        //    at the same point in the constructor (`KafkaProducer#KafkaProducer`, ahead
1116        //    of the `RecordAccumulator` at `KafkaProducer#KafkaProducer` and the `Sender` at `KafkaProducer#KafkaProducer`).
1117        let transaction_manager =
1118            Self::configure_transaction_state(&config, &api_versions, &metadata.metadata_arc(), &log_context);
1119
1120        // 9b. Resolve and configure the partitioner. Translated from
1121        //     `KafkaProducer#KafkaProducer`: Java reflectively instantiates
1122        //     `partitioner.class` and calls `partitioner.configure(originals +
1123        //     {client.id -> clientId})`. Rust has no reflection, so the built-in
1124        //     `partitioner.type` names resolve here, and a user-written partitioner
1125        //     is the value set with `ProducerConfig::set_partitioner`. It is configured with the user
1126        //     config map (`originals`) plus the resolved (possibly generated)
1127        //     `client.id`, so a generated `producer-N` id is visible to `configure`
1128        //     just as in Java. Resolved after metrics/`TransactionManager` creation
1129        //     (matching Java's ordering) and before the `RecordAccumulator`, whose
1130        //     adaptive-partitioning flag is gated on the partitioner's absence below.
1131        //
1132        //     KAFKA-2121: Java's constructor `catch (Throwable)` calls
1133        //     `close(Duration.ofMillis(0), true)`, which closes an already-configured
1134        //     partitioner (`KafkaProducer#KafkaProducer` -> `KafkaProducer#close`).
1135        //     `resolve_partitioner` fails only before resolving one (a partitioner
1136        //     written for other record types), and `configure` is infallible. Every
1137        //     fallible step *after* this point lives in `build_accumulator`, and its
1138        //     error closes the partitioner before propagating, so a step added there
1139        //     later is covered too.
1140        let mut partitioner = config.resolve_partitioner::<K, V>()?;
1141        if let Some(partitioner) = partitioner.as_mut() {
1142            let mut configs = config.originals();
1143            configs.insert(ProducerConfig::CLIENT_ID_CONFIG.to_string(), config.client_id.clone());
1144            partitioner.configure(&configs);
1145        }
1146
1147        let (accumulator, chunked_accumulator) = match Self::build_accumulator(
1148            &config,
1149            compression,
1150            delivery_timeout_ms,
1151            partitioner.is_none(),
1152            &metrics,
1153            &time,
1154            &transaction_manager,
1155            &log_context,
1156        ) {
1157            Ok(accumulator) => accumulator,
1158            Err(error) => {
1159                // Java's `close` closes the serializers, then the partitioner
1160                // (`KafkaProducer#close`).
1161                drop(serializers);
1162                if let Some(partitioner) = partitioner.as_ref() {
1163                    partitioner.close();
1164                }
1165                return Err(error);
1166            },
1167        };
1168
1169        // 11. Wire the produce-throttle-time sensor into the network client.
1170        //    Java creates the throttle sensor (`Sender.throttleTimeSensor(...)`)
1171        //    and hands it to the `NetworkClient` at construction
1172        //    (`KafkaProducer` via `ClientUtils.createNetworkClient`);
1173        //    the client then records every response's throttle time into it. We
1174        //    set it on the concrete `NetworkClient` here, before it moves into
1175        //    the generic sender task.
1176        // Java: `new ProducerMetrics(this.metrics).senderMetrics`.
1177        let sender_metrics_registry = ProducerMetrics::new(Arc::clone(&metrics)).sender_metrics;
1178        let throttle_sensor = SenderStatics::throttle_time_sensor(&sender_metrics_registry)
1179            .expect("registering produce-throttle-time sensor");
1180        client.set_throttle_time_sensor(throttle_sensor);
1181        client.set_time(Arc::clone(&time));
1182
1183        // 12. Wire up the Sender and spawn the I/O background task. Every
1184        //     fallible step is behind us.
1185        let (key_serializer, value_serializer) = serializers.disarm();
1186        // Java's `config.logUnused()` (`KafkaProducer#KafkaProducer`), once the
1187        // producer has read its settings.
1188        config.mark_read_keys_used();
1189        config.log_unused();
1190        Ok(Self::with_client_options(
1191            KafkaProducerClientOptionsBuilder::new()
1192                .set_config(&config)
1193                .set_key_serializer(key_serializer)
1194                .set_value_serializer(value_serializer)
1195                .set_metadata(metadata)
1196                .set_accumulator(accumulator)
1197                .set_chunked_accumulator(chunked_accumulator)
1198                .set_client(client)
1199                .set_time(time)
1200                .set_metrics(metrics)
1201                .set_producer_metrics(producer_metrics)
1202                .set_sender_metrics_registry(sender_metrics_registry)
1203                .set_transaction_manager(transaction_manager)
1204                .set_pending_requests(Arc::new(Mutex::new(PendingRequests::new())))
1205                .set_partitioner(partitioner)
1206                .build()
1207                .expect("KafkaProducerClientOptionsBuilder::build: every mandatory parameter is set above"),
1208        ))
1209    }
1210
1211    /// Step 10 of [`new_inner`](Self::new_inner): the `BufferPool` and the
1212    /// `RecordAccumulator` — a `ChunkedRecordAccumulator` for the incremental
1213    /// `buffer.memory.allocation.strategy` (KIP-1332) — i.e. every fallible constructor step that
1214    /// follows the custom partitioner's `configure`. `new_inner` closes the partitioner when this
1215    /// returns an error, as Java's constructor `catch` does (KAFKA-2121), so a fallible step added
1216    /// after the partitioner belongs here: the incremental strategy's compression
1217    /// `ConfigException` is one.
1218    ///
1219    /// Returns the base accumulator, which the `Sender` and the producer's flush / close paths
1220    /// share, and, for the incremental strategy, the chunked accumulator whose `append` the send
1221    /// path calls instead of the base one (PLAN §2.3).
1222    ///
1223    /// Translated from `KafkaProducer#KafkaProducer`.
1224    #[expect(clippy::too_many_arguments)]
1225    fn build_accumulator(
1226        config: &ProducerConfig,
1227        compression: Compression,
1228        delivery_timeout_ms: i32,
1229        no_custom_partitioner: bool,
1230        metrics: &Arc<Metrics>,
1231        time: &Arc<dyn Time>,
1232        transaction_manager: &Option<Arc<Mutex<TransactionManager>>>,
1233        log_context: &LogContext,
1234    ) -> Result<(Arc<RecordAccumulator>, Option<ChunkedRecordAccumulator>), Error> {
1235        // 10. Create BufferPool and RecordAccumulator, threading the shared
1236        //    `Arc<Metrics>` and `time` into both (`KafkaProducer#KafkaProducer`
1237        //    passes `metrics`/`time` to the `BufferPool` and `RecordAccumulator`).
1238        let partitioner_config = PartitionerConfig::new(
1239            // Java `KafkaProducer#KafkaProducer`: "There is no need to do work
1240            // required for adaptive partitioning, if we use a custom
1241            // partitioner." So adaptive partitioning is enabled only when no
1242            // custom partitioner is present AND the config opts in.
1243            no_custom_partitioner && config.partitioner_adaptive_partitioning_enable,
1244            config.partitioner_availability_timeout_ms,
1245            // KIP-1123 (`KafkaProducer#KafkaProducer`): passed through whether or
1246            // not a custom partitioner is set, as Java does, so a blank
1247            // `client.rack` with `partitioner.rack.aware=true` fails construction
1248            // either way.
1249            config.partitioner_rack_aware,
1250            &config.client_rack,
1251        )?;
1252        // As per Kafka producer configuration documentation batch.size may be set to 0
1253        // to explicitly disable batching which in practice actually means using a batch size of 1.
1254        let batch_size = config.batch_size.max(1);
1255        let allocation_strategy = config.buffer_memory_allocation_strategy.to_lowercase();
1256        let incremental = allocation_strategy == ProducerConfig::BUFFER_MEMORY_ALLOCATION_STRATEGY_INCREMENTAL;
1257        // Use the chunked path only when a batch is at least one full chunk
1258        // (batch.size >= CHUNK_SIZE). Below that, a batch can't fill even one chunk, so chunking
1259        // would over-reserve and the producer falls back to the full strategy instead.
1260        let use_incremental = incremental && batch_size >= ChunkedRecordAccumulator::CHUNK_SIZE;
1261        if incremental && !use_incremental {
1262            kafka_warn!(
1263                log_context,
1264                "Ignoring {}={} and falling back to {}: {} is {} bytes, below the {} byte chunk size, so a batch \
1265                 cannot fill a single chunk.",
1266                ProducerConfig::BUFFER_MEMORY_ALLOCATION_STRATEGY_CONFIG,
1267                ProducerConfig::BUFFER_MEMORY_ALLOCATION_STRATEGY_INCREMENTAL,
1268                ProducerConfig::BUFFER_MEMORY_ALLOCATION_STRATEGY_FULL,
1269                ProducerConfig::BATCH_SIZE_CONFIG,
1270                batch_size,
1271                ChunkedRecordAccumulator::CHUNK_SIZE
1272            );
1273        }
1274        // The chunked path does not support compression yet (KAFKA-20579).
1275        if use_incremental && compression.compression_type() != CompressionType::None {
1276            return Err(Error::config_message(Self::incremental_compression_error_message()));
1277        }
1278        if use_incremental {
1279            let buffer_pool = Arc::new(BufferPool::with_allocation_mode(
1280                config.buffer_memory,
1281                ChunkedRecordAccumulator::CHUNK_SIZE as usize,
1282                Arc::clone(metrics),
1283                Arc::clone(time),
1284                Self::PRODUCER_METRIC_GROUP_NAME,
1285                AllocationMode::Incremental,
1286            ));
1287            let accumulator = ChunkedRecordAccumulator::new(
1288                batch_size,
1289                compression,
1290                Self::linger_ms(config),
1291                config.retry_backoff_ms,
1292                config.retry_backoff_max_ms,
1293                delivery_timeout_ms,
1294                partitioner_config,
1295                Arc::clone(metrics),
1296                Self::PRODUCER_METRIC_GROUP_NAME,
1297                Arc::clone(time),
1298                buffer_pool,
1299                transaction_manager.clone(),
1300                log_context.clone(),
1301            )?;
1302            return Ok((Arc::clone(accumulator.base()), Some(accumulator)));
1303        }
1304        let buffer_pool = Arc::new(BufferPool::with_allocation_mode(
1305            config.buffer_memory,
1306            batch_size as usize,
1307            Arc::clone(metrics),
1308            Arc::clone(time),
1309            Self::PRODUCER_METRIC_GROUP_NAME,
1310            AllocationMode::Full,
1311        ));
1312        let accumulator = Arc::new(RecordAccumulator::with_log_context(
1313            batch_size,
1314            compression,
1315            Self::linger_ms(config),
1316            config.retry_backoff_ms,
1317            config.retry_backoff_max_ms,
1318            delivery_timeout_ms,
1319            partitioner_config,
1320            Arc::clone(metrics),
1321            Self::PRODUCER_METRIC_GROUP_NAME,
1322            Arc::clone(time),
1323            buffer_pool,
1324            transaction_manager.clone(),
1325            log_context.clone(),
1326        ));
1327        Ok((accumulator, None))
1328    }
1329
1330    /// The `ConfigException` message for the incremental strategy combined with compression
1331    /// (`KafkaProducer#KafkaProducer`).
1332    fn incremental_compression_error_message() -> String {
1333        format!(
1334            "The {} {} does not support compression yet. {} must be set to none.",
1335            ProducerConfig::BUFFER_MEMORY_ALLOCATION_STRATEGY_INCREMENTAL,
1336            ProducerConfig::BUFFER_MEMORY_ALLOCATION_STRATEGY_CONFIG,
1337            ProducerConfig::COMPRESSION_TYPE_CONFIG
1338        )
1339    }
1340
1341    /// Builds the [`TransactionManager`] when idempotence is enabled.
1342    ///
1343    /// Translated from `KafkaProducer.configureTransactionState`
1344    /// (`KafkaProducer#configureTransactionState`).
1345    ///
1346    /// Java returns `null` when `enable.idempotence` is `false`; that is `None`
1347    /// here. Java's `else` branch only marks `transaction.timeout.ms` as consumed
1348    /// so `AbstractConfig` does not warn about it, which has no Rust analogue —
1349    /// `ProducerConfig` parses every key eagerly.
1350    ///
1351    /// Returns `None` when idempotence is disabled, mirroring Java's null
1352    /// `transactionManager`. It no longer returns a `Result`: the only error it
1353    /// ever carried was [`Self::new`]'s temporary guard on
1354    /// `transactional.id` (PLAN §7.1), which Phase 6 removed.
1355    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#configureTransactionState")]
1356    fn configure_transaction_state(
1357        config: &ProducerConfig,
1358        api_versions: &Arc<ApiVersions>,
1359        metadata: &Arc<Metadata>,
1360        log_context: &LogContext,
1361    ) -> Option<Arc<Mutex<TransactionManager>>> {
1362        if !config.enable_idempotence {
1363            return None;
1364        }
1365
1366        let transaction_manager = TransactionManager::new(
1367            log_context.clone(),
1368            config.transactional_id.clone(),
1369            config.transaction_timeout_ms,
1370            config.retry_backoff_ms,
1371            Arc::clone(api_versions),
1372            Arc::clone(metadata),
1373            config.two_phase_commit_enable,
1374        );
1375
1376        if transaction_manager.is_transactional() {
1377            kafka_info!(log_context, "Instantiated a transactional producer.");
1378        } else {
1379            kafka_info!(log_context, "Instantiated an idempotent producer.");
1380        }
1381
1382        Some(Arc::new(Mutex::new(transaction_manager)))
1383    }
1384
1385    /// Creates a `KafkaProducer` from pre-built collaborators and spawns the
1386    /// sender task.
1387    ///
1388    /// Translates Java's `KafkaProducer(ProducerConfig, Serializer, Serializer,
1389    /// ProducerMetadata, KafkaClient, ProducerInterceptors, ApiVersions, Time)`
1390    /// (`KafkaProducer#KafkaProducer`), which Java marks `// visible for testing`.
1391    /// [`Self::new`] is the user-facing constructor and the analogue of Java's
1392    /// public `KafkaProducer` constructor; this is the injection seam
1393    /// [`Self::new_inner`] delegates to, and the seam through which a mock
1394    /// [`KafkaClient`] enters.
1395    ///
1396    /// Marked `pub(crate)`, not `pub`: `metadata` and `accumulator` are `Arc`s of
1397    /// `ProducerMetadata` and `RecordAccumulator`, both `pub(crate)` under
1398    /// `producer::internals`, so an external caller could not have named or
1399    /// constructed them even when this method itself was `pub` — making it
1400    /// `pub(crate)` only makes that existing unreachability explicit, it does
1401    /// not remove any capability external callers actually had.
1402    ///
1403    /// (Through Phase 5 that unreachability was also the reason the temporary
1404    /// MILESTONE-11 GUARD in [`Self::new`] was not duplicated here. Phase 6
1405    /// removed the guard, so nothing turns on it any more; the visibility note above
1406    /// stands on its own.)
1407    ///
1408    /// # Type Parameters
1409    ///
1410    /// * `C` - The KafkaClient implementation type
1411    ///
1412    /// # Arguments
1413    ///
1414    /// * `options` - Every parameter, built through
1415    ///   [`KafkaProducerClientOptionsBuilder`].
1416    ///   [`KafkaProducerClientOptions`] is this method's only parameter because
1417    ///   the derived name would list ten parameters, past CLAUDE.md §2's cap of
1418    ///   three; see that struct for why `client` is nonetheless kept in the name.
1419    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#KafkaProducer")]
1420    pub(crate) fn with_client_options<C: KafkaClient + Send + 'static>(
1421        options: KafkaProducerClientOptions<'_, K, V, C>,
1422    ) -> Self {
1423        let KafkaProducerClientOptions {
1424            config,
1425            key_serializer,
1426            value_serializer,
1427            metadata,
1428            accumulator,
1429            chunked_accumulator,
1430            client,
1431            time,
1432            metrics,
1433            producer_metrics,
1434            sender_metrics_registry,
1435            transaction_manager,
1436            pending_requests,
1437            partitioner,
1438        } = options;
1439        let log_context = LogContext::new(format!("[Producer clientId={}] ", config.client_id));
1440        let running = Arc::new(AtomicBool::new(true));
1441        let force_close = Arc::new(AtomicBool::new(false));
1442        let wakeup = client.wakeup_notify();
1443
1444        let guarantee_message_order = config.max_in_flight_requests_per_connection == 1;
1445        let acks = config.acks;
1446        let retries = config.retries;
1447
1448        let mut sender = Sender::new(
1449            client,
1450            Arc::clone(&metadata),
1451            Arc::clone(&accumulator),
1452            guarantee_message_order,
1453            config.max_request_size,
1454            acks,
1455            retries,
1456            config.request_timeout_ms,
1457            config.retry_backoff_ms,
1458            sender_metrics_registry,
1459            Arc::clone(&running),
1460            Arc::clone(&force_close),
1461            Arc::clone(&time),
1462            transaction_manager.clone(),
1463            Arc::clone(&pending_requests),
1464            log_context.clone(),
1465        );
1466
1467        let io_thread_name = format!("{} | {}", Self::NETWORK_THREAD_PREFIX, config.client_id);
1468        let task_log_context = log_context.clone();
1469        let sender_handle = tokio::task::spawn(async move {
1470            kafka_debug!(task_log_context, "Starting {} I/O task", io_thread_name);
1471            sender.run().await;
1472        });
1473
1474        kafka_debug!(log_context, "Kafka producer started");
1475
1476        Self {
1477            client_id: config.client_id.clone(),
1478            key_serializer,
1479            value_serializer,
1480            serializers_closed: AtomicBool::new(false),
1481            max_request_size: config.max_request_size,
1482            total_memory_size: config.buffer_memory,
1483            accumulator,
1484            chunked_accumulator,
1485            metadata,
1486            transaction_manager,
1487            pending_requests,
1488            compression_type: config.compression_type,
1489            max_block_ms: config.max_block_ms,
1490            partitioner,
1491            partitioner_ignore_keys: config.partitioner_ignore_keys,
1492            key_hasher: config.key_hasher(),
1493            running,
1494            force_close,
1495            wakeup,
1496            sender_handle: Mutex::new(Some(sender_handle)),
1497            time,
1498            metrics,
1499            producer_metrics,
1500            log_context,
1501        }
1502    }
1503
1504    /// Create the producer's [`Metrics`] registry and [`KafkaProducerMetrics`].
1505    ///
1506    /// Translated from the metrics-setup block of Java's `KafkaProducer`
1507    /// constructor (`KafkaProducer#KafkaProducer`): a [`MetricConfig`] carrying
1508    /// `metrics.num.samples`, `metrics.sample.window.ms`,
1509    /// `metrics.recording.level` and a single `client-id` tag. Reporters and the
1510    /// JMX metrics context are N/A in Rust (consumer Phase M7 precedent); the
1511    /// registry is reporter-less but fully functional.
1512    fn create_metrics(config: &ProducerConfig) -> (Arc<Metrics>, KafkaProducerMetrics) {
1513        const CLIENT_ID_METRIC_TAG: &str = "client-id";
1514
1515        let mut tags = std::collections::BTreeMap::new();
1516        tags.insert(CLIENT_ID_METRIC_TAG.to_string(), config.client_id.clone());
1517
1518        let recording_level = RecordingLevel::for_name(&config.metrics_recording_level).unwrap_or(RecordingLevel::Info);
1519        let metric_config = MetricConfig::new()
1520            .set_samples(config.metrics_num_samples)
1521            .set_time_window_ms(config.metrics_sample_window_ms)
1522            .set_record_level(recording_level)
1523            .set_tags(tags);
1524
1525        let metrics = Arc::new(Metrics::with_default_config(Arc::new(metric_config)));
1526        let producer_metrics = KafkaProducerMetrics::new(Arc::clone(&metrics));
1527        (metrics, producer_metrics)
1528    }
1529
1530    /// Builds the compression codec from `compression.type` and the level of
1531    /// that codec: Java's `configureCompression`.
1532    ///
1533    /// # Errors
1534    ///
1535    /// The builder's error for a level outside the codec's range, which the
1536    /// `ConfigDef` validators already rule out for a configuration built by
1537    /// `ProducerConfig::new`.
1538    fn configure_compression(config: &ProducerConfig) -> Result<Compression, Error> {
1539        Ok(match config.compression_type {
1540            CompressionType::Gzip => Compression::gzip().level(config.compression_gzip_level)?.build(),
1541            CompressionType::Lz4 => Compression::lz4().level(config.compression_lz4_level)?.build(),
1542            CompressionType::Zstd => Compression::zstd().level(config.compression_zstd_level)?.build(),
1543            compression_type => Compression::of(compression_type).build(),
1544        })
1545    }
1546
1547    /// Returns `linger.ms` as the `i32` the accumulator works with.
1548    ///
1549    /// `linger.ms` is a long-typed config, so values above `i32::MAX` are clamped
1550    /// to `i32::MAX` rather than cast with `as i32`, which keeps only the low 32
1551    /// bits and can yield zero or a negative linger, making every batch ready
1552    /// immediately.
1553    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#lingerMs")]
1554    fn linger_ms(config: &ProducerConfig) -> i32 {
1555        config.linger_ms.min(i32::MAX as i64) as i32
1556    }
1557
1558    /// Validate and optionally adjust `delivery.timeout.ms` against
1559    /// `linger.ms + request.timeout.ms`.
1560    ///
1561    /// Translated from `KafkaProducer.configureDeliveryTimeout()`
1562    /// (`KafkaProducer#configureDeliveryTimeout`). Java's check has two arms and both are
1563    /// reproduced: an *explicitly supplied* inconsistent `delivery.timeout.ms` is
1564    /// a `ConfigException`, while an inconsistency that comes only from the
1565    /// default is clamped up to `linger.ms + request.timeout.ms` and warned about,
1566    /// for backward compatibility. `ProducerConfig::user_configured` stands for
1567    /// Java's `config.originals().containsKey(..)`.
1568    ///
1569    /// # Errors
1570    ///
1571    /// Returns [`Error::Config`] — Java's `ConfigException`, which is inside the
1572    /// `KafkaException` hierarchy — if the user explicitly set a
1573    /// `delivery.timeout.ms` smaller than `linger.ms + request.timeout.ms`.
1574    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#configureDeliveryTimeout")]
1575    fn configure_delivery_timeout(config: &ProducerConfig, log_context: &LogContext) -> Result<i32, Error> {
1576        let mut delivery_timeout_ms = config.delivery_timeout_ms;
1577        let linger_ms = Self::linger_ms(config);
1578        let request_timeout_ms = config.request_timeout_ms;
1579        let linger_and_request_timeout_ms = (linger_ms as i64 + request_timeout_ms as i64).min(i32::MAX as i64) as i32;
1580
1581        if delivery_timeout_ms < linger_and_request_timeout_ms {
1582            if config.user_configured(ProducerConfig::DELIVERY_TIMEOUT_MS_CONFIG) {
1583                // Java `KafkaProducer#configureDeliveryTimeout`: throw if the user explicitly set an inconsistent value.
1584                // The class is `ConfigException`, inside the `KafkaException`
1585                // hierarchy (`KafkaProducerTest.testDeliveryTimeoutAndLingerMsConfig`
1586                // asserts `KafkaException.class`); `illegal_argument` put it outside,
1587                // where `is_kafka_error()` answers `false`.
1588                return Err(Error::config_message(format!(
1589                    "{} should be equal to or larger than {} + {}",
1590                    ProducerConfig::DELIVERY_TIMEOUT_MS_CONFIG,
1591                    ProducerConfig::LINGER_MS_CONFIG,
1592                    ProducerConfig::REQUEST_TIMEOUT_MS_CONFIG,
1593                )));
1594            }
1595            // Java `KafkaProducer#configureDeliveryTimeout`: override the default for backward compatibility.
1596            delivery_timeout_ms = linger_and_request_timeout_ms;
1597            kafka_warn!(
1598                log_context,
1599                "{} should be equal to or larger than {} + {}. Setting it to {}.",
1600                ProducerConfig::DELIVERY_TIMEOUT_MS_CONFIG,
1601                ProducerConfig::LINGER_MS_CONFIG,
1602                ProducerConfig::REQUEST_TIMEOUT_MS_CONFIG,
1603                delivery_timeout_ms
1604            );
1605        }
1606        Ok(delivery_timeout_ms)
1607    }
1608
1609    /// Returns the client ID for this producer.
1610    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#getClientId")]
1611    pub(crate) fn client_id(&self) -> &str {
1612        &self.client_id
1613    }
1614
1615    /// Returns the current time in milliseconds from `time`.
1616    fn now_ms(&self) -> i64 {
1617        self.time.milliseconds()
1618    }
1619
1620    /// `max.block.ms` as a [`Duration`], for the four transactional methods that
1621    /// bound their wait with it (`result.await(maxBlockTimeMs, MILLISECONDS)`).
1622    ///
1623    /// Clamped at zero: a negative `max.block.ms` cannot be configured, and
1624    /// `Duration` has no negative representation.
1625    fn max_block_timeout(&self) -> Duration {
1626        Duration::from_millis(self.max_block_ms.max(0) as u64)
1627    }
1628
1629    /// Needs to be called before any other methods when the `transactional.id`
1630    /// is set in the configuration. This method does the following:
1631    ///
1632    /// 1. Ensures any transactions initiated by previous instances of the producer
1633    ///    with the same `transactional.id` are completed. If the previous instance
1634    ///    had failed with a transaction in progress, it will be aborted. If the
1635    ///    last transaction had begun completion, but not yet finished, this method
1636    ///    awaits its completion.
1637    /// 2. Gets the internal producer id and epoch, used in all future
1638    ///    transactional messages issued by the producer.
1639    ///
1640    /// Note that this method will return a timeout error if the transactional
1641    /// state cannot be initialized before expiration of `max.block.ms`.
1642    /// Additionally, it will return an interrupt error if interrupted. It is safe
1643    /// to retry in either case, but once the transactional state has been
1644    /// successfully initialized, this method should no longer be used.
1645    ///
1646    /// # Errors
1647    ///
1648    /// - `Error::is_local_illegal_state_error` if no `transactional.id` has been configured, or
1649    ///   the producer has been closed
1650    /// - `Error::is_unsupported_version_error` fatal error indicating the broker does not support
1651    ///   transactions (i.e. if its version is lower than 0.11.0.0)
1652    /// - `Error::is_authorization_error` error indicating that the configured transactional.id is
1653    ///   not authorized, or the idempotent producer id is unavailable. See the error for more
1654    ///   details. User may retry this function call after fixing the permission.
1655    /// - `Error::is_kafka_error` if the producer has encountered a previous fatal error or for any
1656    ///   other unexpected error
1657    /// - `Error::is_timeout_error` if the time taken for initialize the transaction has surpassed
1658    ///   `max.block.ms`.
1659    /// - `Error::is_interrupt_error` if the task is interrupted while blocked
1660    ///
1661    /// # Translation notes
1662    ///
1663    /// A Rust task is never interrupted: dropping the future cancels the wait.
1664    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#initTransactions")]
1665    pub fn init_transactions(&self) -> impl Future<Output = Result<(), Error>> + Send {
1666        async move {
1667            let transaction_manager = self.transaction_manager_or_error()?;
1668            self.ensure_not_closed()?;
1669            // Java: `long now = time.nanoseconds()` (`KafkaProducer`), fed to
1670            // `producerMetrics.recordInit(time.nanoseconds() - now)` (`KafkaProducer`) after a
1671            // successful await. Timed with the injected `time`, as Java times with its
1672            // injected `Time` — the same clock `flush` and `waitOnMetadata` read.
1673            let start = self.time.nanoseconds();
1674            let result = {
1675                // `pending_requests` before the manager, per the field docs.
1676                let mut pending_requests = self.pending_requests.lock().unwrap();
1677                transaction_manager
1678                    .lock()
1679                    .unwrap()
1680                    .initialize_transactions(false, &mut pending_requests)?
1681            };
1682            self.wakeup.notify_one();
1683            result
1684                .await_result_timeout(self.max_block_timeout(), Self::INIT_TXN_TIMEOUT_MSG)
1685                .await?;
1686            // Java records only after a successful await (`KafkaProducer#initTransactions`), so the `?` above must
1687            // stay ahead of both this and `maybeUpdateTransactionV2Enabled`.
1688            self.producer_metrics.record_init(self.time.nanoseconds() - start);
1689            transaction_manager.lock().unwrap().maybe_update_transaction_v2_enabled(true);
1690            Ok(())
1691        }
1692    }
1693
1694    /// Should be called before the start of each new transaction. Note that prior
1695    /// to the first invocation of this method, you must invoke
1696    /// [`Self::init_transactions`] exactly one time.
1697    ///
1698    /// # Errors
1699    ///
1700    /// - `Error::is_local_illegal_state_error` if no `transactional.id` has been configured or if
1701    ///   `Self::init_transactions` has not yet been invoked, or the producer has been closed
1702    /// - `Error::is_producer_fenced_error` if another producer with the same transactional.id is
1703    ///   active
1704    /// - `Error::is_invalid_producer_epoch_error` if the producer has attempted to produce with an
1705    ///   old epoch to the partition leader. See the error for more details
1706    /// - `Error::is_unsupported_version_error` fatal error indicating the broker does not support
1707    ///   transactions (i.e. if its version is lower than 0.11.0.0)
1708    /// - `Error::is_authorization_error` fatal error indicating that the configured
1709    ///   `transactional.id` is not authorized. See the error for more details
1710    /// - `Error::is_kafka_error` if the producer has encountered a previous fatal error or for any
1711    ///   other unexpected error
1712    ///
1713    /// # Translation notes
1714    ///
1715    /// Synchronous, as Java's `beginTransaction` never blocks.
1716    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#beginTransaction")]
1717    pub fn begin_transaction(&self) -> Result<(), Error> {
1718        let transaction_manager = self.transaction_manager_or_error()?;
1719        self.ensure_not_closed()?;
1720        // Java: `long now = time.nanoseconds()` (`KafkaProducer`) around the
1721        // synchronous state transition, recorded on success (`KafkaProducer`). Java times this
1722        // even though there is no wait; kept faithful.
1723        let start = self.time.nanoseconds();
1724        transaction_manager.lock().unwrap().begin_transaction()?;
1725        self.producer_metrics.record_begin_txn(self.time.nanoseconds() - start);
1726        Ok(())
1727    }
1728
1729    /// Sends a list of specified offsets to the consumer group coordinator, and
1730    /// also marks those offsets as part of the current transaction. These offsets
1731    /// will be considered committed only if the transaction is committed
1732    /// successfully. The committed offset should be the next message your
1733    /// application will consume, i.e. `next_record_to_be_processed.offset()` (or
1734    /// `ConsumerRecords::next_offsets`). You should also add the leader epoch as
1735    /// commit metadata, which can be obtained from `ConsumerRecord::leader_epoch`
1736    /// or `ConsumerRecords::next_offsets`.
1737    ///
1738    /// This method should be used when you need to batch consumed and produced
1739    /// messages together, typically in a consume-transform-produce pattern. Thus,
1740    /// the specified `group_metadata` should be extracted from the used consumer
1741    /// via `KafkaConsumer::group_metadata` to leverage consumer group metadata.
1742    /// This will provide stronger fencing than just supplying the consumer group
1743    /// id, however note that the full set of consumer group metadata returned by
1744    /// `KafkaConsumer::group_metadata` requires the brokers to be on version 2.5
1745    /// or newer to understand.
1746    ///
1747    /// This method is a blocking call that waits until the request has been
1748    /// received and acknowledged by the consumer group coordinator; but the
1749    /// offsets are not considered as committed until the transaction itself is
1750    /// successfully committed later (via the [`Self::commit_transaction`] call).
1751    ///
1752    /// Note, that the consumer should have `enable.auto.commit=false` and should
1753    /// also not commit offsets manually (via sync or async commits). This method
1754    /// will return a timeout error if the producer cannot resolve the metadata
1755    /// for the topics in `offsets` and send the offsets before expiration of
1756    /// `max.block.ms`. Additionally, it will return an interrupt error if
1757    /// interrupted.
1758    ///
1759    /// # Arguments
1760    ///
1761    /// - `offsets` - the offsets to commit, per partition; an empty map does nothing
1762    /// - `group_metadata` - the consumer group metadata, from `KafkaConsumer::group_metadata`
1763    ///
1764    /// # Errors
1765    ///
1766    /// - `Error::is_local_illegal_argument_error` if `group_metadata` has a generation id greater
1767    ///   than zero but an unknown member id
1768    /// - `Error::is_local_illegal_state_error` if no transactional.id has been configured or no
1769    ///   transaction has been started, or the producer has been closed.
1770    /// - `Error::is_producer_fenced_error` fatal error indicating another producer with the same
1771    ///   transactional.id is active
1772    /// - `Error::is_unsupported_version_error` fatal error indicating the broker does not support
1773    ///   transactions (i.e. if its version is lower than 0.11.0.0) or the broker doesn't support
1774    ///   the latest version of transactional API with all consumer group metadata (i.e. if its
1775    ///   version is lower than 2.5.0).
1776    /// - `Error::is_unsupported_for_message_format_error` fatal error indicating the message format
1777    ///   used for the offsets topic on the broker does not support transactions
1778    /// - `Error::is_authorization_error` fatal error indicating that the configured
1779    ///   transactional.id is not authorized, or the consumer group id is not authorized.
1780    /// - `Error::is_consumer_commit_failed_error` if the commit failed and cannot be retried (e.g.
1781    ///   if the consumer has been kicked out of the group). Users should handle this by aborting
1782    ///   the transaction.
1783    /// - `Error::is_fenced_instance_id_error` if this producer instance gets fenced by broker due
1784    ///   to a mis-configured consumer instance id within group metadata.
1785    /// - `Error::is_invalid_producer_epoch_error` if the producer has attempted to produce with an
1786    ///   old epoch to the partition leader. See the error for more details
1787    /// - `Error::is_kafka_error` if the producer has encountered a previous fatal or abortable
1788    ///   error, or for any other unexpected error
1789    /// - `Error::is_timeout_error` if the combined time taken for resolving topic metadata and
1790    ///   sending the offsets has surpassed `max.block.ms`.
1791    /// - `Error::is_interrupt_error` if the task is interrupted while blocked
1792    /// - `Error::is_bootstrap_resolution_error` if DNS resolution of the bootstrap servers fails
1793    ///   within `bootstrap.resolve.timeout.ms`
1794    ///
1795    /// # Translation notes
1796    ///
1797    /// A Rust task is never interrupted: dropping the future cancels the wait.
1798    /// `offsets` is taken by value, as the request carries it to the
1799    /// coordinator; `group_metadata` is borrowed and copied.
1800    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#sendOffsetsToTransaction")]
1801    pub fn send_offsets_to_transaction(
1802        &self,
1803        offsets: HashMap<TopicPartition, OffsetAndMetadata>,
1804        group_metadata: &dyn ConsumerGroupMetadata,
1805    ) -> impl Future<Output = Result<(), Error>> + Send {
1806        async move {
1807            Self::throw_if_invalid_group_metadata(group_metadata)?;
1808            let transaction_manager = self.transaction_manager_or_error()?;
1809            self.ensure_not_closed()?;
1810
1811            // `KafkaProducer#sendOffsetsToTransaction`: an empty map is a no-op, and in particular does not consult the
1812            // transaction state at all. It is also left unmetered — Java starts the timer
1813            // inside the `if (!offsets.isEmpty())` block, after this check.
1814            if offsets.is_empty() {
1815                return Ok(());
1816            }
1817
1818            // Java: `long start = time.nanoseconds()` inside the non-empty branch
1819            // (`KafkaProducer`), before the KIP-1319 metadata wait, so that wait
1820            // is part of the recorded duration; recorded after a successful await (`KafkaProducer`).
1821            let start = self.time.nanoseconds();
1822
1823            // KIP-1319 (6208dfc014): make sure the metadata cache knows these topics'
1824            // ids before the manager builds the request. No manager guard is held
1825            // across this await (`producer-transactions.md` §4).
1826            let topics: HashSet<&str> = offsets.keys().map(TopicPartition::topic).collect();
1827            let wait_ms = self.await_topic_metadata(&topics).await?;
1828            let remaining_ms = (self.max_block_ms - wait_ms).max(0);
1829            let result = {
1830                // `pending_requests` before the manager, per the field docs.
1831                let mut pending_requests = self.pending_requests.lock().unwrap();
1832                transaction_manager.lock().unwrap().send_offsets_to_transaction(
1833                    offsets,
1834                    ConsumerGroupMetadataImpl::copy_of(group_metadata),
1835                    &mut pending_requests,
1836                )?
1837            };
1838            self.wakeup.notify_one();
1839            result
1840                .await_result_timeout(Duration::from_millis(remaining_ms as u64), Self::SEND_OFFSETS_TIMEOUT_MSG)
1841                .await?;
1842            self.producer_metrics.record_send_offsets(self.time.nanoseconds() - start);
1843            Ok(())
1844        }
1845    }
1846
1847    /// Request a partial metadata refresh for the given topics and await the next
1848    /// metadata update (up to `max.block.ms`). Returns the elapsed wait time so the
1849    /// caller can subtract it from its own `max.block.ms` budget.
1850    ///
1851    /// Translated from `KafkaProducer.awaitTopicMetadata(Set<String>)`
1852    /// (6208dfc014, with KIP-909's bootstrap check). Even when no refresh is
1853    /// needed, a permanent bootstrap failure is surfaced so every API call sees
1854    /// it; on the refresh branch `await_update` surfaces it through
1855    /// `maybe_return_fatal_error`, as Java's `awaitUpdate` does through
1856    /// `maybeThrowFatalException`.
1857    ///
1858    /// # Errors
1859    ///
1860    /// [`Error::Timeout`] (`"Failed to update metadata after <max.block.ms> ms."`)
1861    /// if no update arrives in time, [`Error::BootstrapResolution`] once bootstrap
1862    /// resolution has failed for good, or the fatal metadata error /
1863    /// closed-metadata error `await_update` reports.
1864    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#awaitTopicMetadata")]
1865    async fn await_topic_metadata(&self, topics: &HashSet<&str>) -> Result<i64, Error> {
1866        let start_nanos = self.time.nanoseconds();
1867        let Some(version) = self.metadata.add_with_topics(topics.iter().copied(), self.now_ms()) else {
1868            // Even when no metadata refresh is needed for these topics, a permanent bootstrap
1869            // failure must still be surfaced so every API call sees the error.
1870            self.metadata.maybe_return_bootstrap_fatal_error()?;
1871            return Ok(0);
1872        };
1873        self.wakeup.notify_one();
1874        self.metadata.await_update(version, self.max_block_ms).await?;
1875        let elapsed_nanos = self.time.nanoseconds() - start_nanos;
1876        self.producer_metrics.record_metadata_wait(elapsed_nanos);
1877        Ok(elapsed_nanos / 1_000_000)
1878    }
1879
1880    /// Commits the ongoing transaction. This method will flush any unsent records
1881    /// before actually committing the transaction.
1882    ///
1883    /// Further, if any of the [`send`](Producer::send) calls which were part of
1884    /// the transaction hit irrecoverable errors, this method will return the last
1885    /// received error immediately and the transaction will not be committed. So
1886    /// all `send` calls in a transaction must succeed in order for this method to
1887    /// succeed.
1888    ///
1889    /// If the transaction is committed successfully and this method returns
1890    /// without an error, it is guaranteed that all callbacks for records in the
1891    /// transaction will have been invoked and completed. Note that panics raised
1892    /// by callbacks are ignored; the producer proceeds to commit the transaction
1893    /// in any case.
1894    ///
1895    /// Note that this method will return a timeout error if the transaction
1896    /// cannot be committed before expiration of `max.block.ms`, but this does not
1897    /// mean the request did not actually reach the broker. In fact, it only
1898    /// indicates that we cannot get the acknowledgement response in time, so
1899    /// it's up to the application's logic to decide how to handle timeouts.
1900    /// Additionally, it will return an interrupt error if interrupted. It is safe
1901    /// to retry in either case, but it is not possible to attempt a different
1902    /// operation (such as [`Self::abort_transaction`]) since the commit may
1903    /// already be in the progress of completing. If not retrying, the only
1904    /// option is to close the producer.
1905    ///
1906    /// # Errors
1907    ///
1908    /// - `Error::is_local_illegal_state_error` if no transactional.id has been configured or no
1909    ///   transaction has been started, or the producer has been closed
1910    /// - `Error::is_producer_fenced_error` fatal error indicating another producer with the same
1911    ///   transactional.id is active
1912    /// - `Error::is_unsupported_version_error` fatal error indicating the broker does not support
1913    ///   transactions (i.e. if its version is lower than 0.11.0.0)
1914    /// - `Error::is_authorization_error` fatal error indicating that the configured
1915    ///   transactional.id is not authorized. See the error for more details
1916    /// - `Error::is_invalid_producer_epoch_error` if the producer has attempted to produce with an
1917    ///   old epoch to the partition leader. See the error for more details
1918    /// - `Error::is_kafka_error` if the producer has encountered a previous fatal or abortable
1919    ///   error, or for any other unexpected error
1920    /// - `Error::is_timeout_error` if the time taken for committing the transaction has surpassed
1921    ///   `max.block.ms`.
1922    /// - `Error::is_interrupt_error` if the task is interrupted while blocked
1923    ///
1924    /// # Translation notes
1925    ///
1926    /// A Rust task is never interrupted: dropping the future cancels the wait.
1927    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#commitTransaction")]
1928    pub fn commit_transaction(&self) -> impl Future<Output = Result<(), Error>> + Send {
1929        async move {
1930            let transaction_manager = self.transaction_manager_or_error()?;
1931            self.ensure_not_closed()?;
1932            // Java: `long commitStart = time.nanoseconds()` (`KafkaProducer`),
1933            // recorded after a successful await (`KafkaProducer`).
1934            let start = self.time.nanoseconds();
1935            let result = {
1936                // `pending_requests` before the manager, per the field docs.
1937                let mut pending_requests = self.pending_requests.lock().unwrap();
1938                transaction_manager.lock().unwrap().begin_commit(&mut pending_requests)?
1939            };
1940            self.wakeup.notify_one();
1941            result
1942                .await_result_timeout(self.max_block_timeout(), Self::COMMIT_TXN_TIMEOUT_MSG)
1943                .await?;
1944            self.producer_metrics.record_commit_txn(self.time.nanoseconds() - start);
1945            Ok(())
1946        }
1947    }
1948
1949    /// Aborts the ongoing transaction. Any unflushed produce messages will be
1950    /// aborted when this call is made. This call will return an error
1951    /// immediately if any prior [`send`](Producer::send) calls failed with a
1952    /// producer-fenced error or an authorization error.
1953    ///
1954    /// Note that this method will return a timeout error if the transaction
1955    /// cannot be aborted before expiration of `max.block.ms`, but this does not
1956    /// mean the request did not actually reach the broker. In fact, it only
1957    /// indicates that we cannot get the acknowledgement response in time, so
1958    /// it's up to the application's logic to decide how to handle timeouts.
1959    /// Additionally, it will return an interrupt error if interrupted. It is safe
1960    /// to retry in either case, but it is not possible to attempt a different
1961    /// operation (such as [`Self::commit_transaction`]) since the abort may
1962    /// already be in the progress of completing. If not retrying, the only
1963    /// option is to close the producer.
1964    ///
1965    /// # Errors
1966    ///
1967    /// - `Error::is_local_illegal_state_error` if no transactional.id has been configured or no
1968    ///   transaction has been started, or the producer has been closed
1969    /// - `Error::is_producer_fenced_error` fatal error indicating another producer with the same
1970    ///   transactional.id is active
1971    /// - `Error::is_invalid_producer_epoch_error` if the producer has attempted to produce with an
1972    ///   old epoch to the partition leader. See the error for more details
1973    /// - `Error::is_unsupported_version_error` fatal error indicating the broker does not support
1974    ///   transactions (i.e. if its version is lower than 0.11.0.0)
1975    /// - `Error::is_authorization_error` fatal error indicating that the configured
1976    ///   transactional.id is not authorized. See the error for more details
1977    /// - `Error::is_kafka_error` if the producer has encountered a previous fatal error or for any
1978    ///   other unexpected error
1979    /// - `Error::is_timeout_error` if the time taken for aborting the transaction has surpassed
1980    ///   `max.block.ms`.
1981    /// - `Error::is_interrupt_error` if the task is interrupted while blocked
1982    ///
1983    /// # Translation notes
1984    ///
1985    /// A Rust task is never interrupted: dropping the future cancels the wait.
1986    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#abortTransaction")]
1987    pub fn abort_transaction(&self) -> impl Future<Output = Result<(), Error>> + Send {
1988        async move {
1989            let transaction_manager = self.transaction_manager_or_error()?;
1990            self.ensure_not_closed()?;
1991            kafka_info!(self.log_context, "Aborting incomplete transaction");
1992            // Java: `long abortStart = time.nanoseconds()` (`KafkaProducer`),
1993            // taken after the log line, recorded after a successful await (`KafkaProducer`).
1994            let start = self.time.nanoseconds();
1995            let result = {
1996                // `pending_requests` before the manager, per the field docs.
1997                let mut pending_requests = self.pending_requests.lock().unwrap();
1998                // `Caller::App`: this runs on the application task (rules §1).
1999                transaction_manager
2000                    .lock()
2001                    .unwrap()
2002                    .begin_abort(&mut pending_requests, Caller::App)?
2003            };
2004            self.wakeup.notify_one();
2005            result
2006                .await_result_timeout(self.max_block_timeout(), Self::ABORT_TXN_TIMEOUT_MSG)
2007                .await?;
2008            self.producer_metrics.record_abort_txn(self.time.nanoseconds() - start);
2009            Ok(())
2010        }
2011    }
2012
2013    /// The shared [`TransactionManager`], or the error Java's
2014    /// `throwIfNoTransactionManager()` throws.
2015    ///
2016    /// Java checks `transactionManager == null` only, so an *idempotent* producer
2017    /// passes this check and is rejected one level down by the manager's own
2018    /// `ensureTransactional()` with a different message. That split is preserved:
2019    /// this must not also test `is_transactional()`.
2020    fn transaction_manager_or_error(&self) -> Result<Arc<Mutex<TransactionManager>>, Error> {
2021        match &self.transaction_manager {
2022            Some(transaction_manager) => Ok(Arc::clone(transaction_manager)),
2023            None => Err(Error::local_illegal_state(format!(
2024                "Cannot use transactional methods without enabling transactions by setting the {} configuration property",
2025                ProducerConfig::TRANSACTIONAL_ID_CONFIG
2026            ))),
2027        }
2028    }
2029
2030    /// Validates the consumer group metadata handed to
2031    /// [`Self::send_offsets_to_transaction`].
2032    ///
2033    /// Translated from `KafkaProducer.throwIfInvalidGroupMetadata`
2034    /// (`KafkaProducer#throwIfInvalidGroupMetadata`). Java's first arm rejects a `null`
2035    /// argument; a `ConsumerGroupMetadata` value cannot be null in Rust, so the
2036    /// type system enforces that arm and only the second is translated.
2037    ///
2038    /// # Errors
2039    ///
2040    /// [`Error::LocalIllegalArgument`] when the generation id is greater than zero
2041    /// but the member id is unknown.
2042    fn throw_if_invalid_group_metadata(group_metadata: &dyn ConsumerGroupMetadata) -> Result<(), Error> {
2043        if group_metadata.generation_id() > 0 && group_metadata.member_id() == TxnOffsetCommitRequest::UNKNOWN_MEMBER_ID
2044        {
2045            return Err(Error::local_illegal_argument(format!(
2046                "Passed in group metadata {} has generationId > 0 but the member.id is unknown",
2047                group_metadata
2048            )));
2049        }
2050        Ok(())
2051    }
2052
2053    /// Verify that this producer instance has not been closed.
2054    ///
2055    /// Corresponds to Java's `throwIfProducerClosed()`.
2056    fn ensure_not_closed(&self) -> Result<(), Error> {
2057        if !self.running.load(Ordering::Acquire) {
2058            return Err(Error::local_illegal_state(
2059                "Cannot perform operation after producer has been closed",
2060            ));
2061        }
2062        Ok(())
2063    }
2064
2065    /// The time a send may spend blocked before its record reaches the
2066    /// accumulator: `max.block.ms` on the blocking path, nothing at all on the
2067    /// non-blocking one.
2068    fn max_block_ms_for(&self, wait: SendWait) -> i64 {
2069        match wait {
2070            SendWait::MaxBlock => self.max_block_ms,
2071            SendWait::Never => 0,
2072        }
2073    }
2074
2075    /// Implementation of asynchronously send a record to a topic.
2076    ///
2077    /// Translated from `KafkaProducer.doSend()`.
2078    ///
2079    /// For `ApiException`-type errors (record-too-large, invalid topic, an
2080    /// `ApiException` raised by a serializer, etc.), the callback is invoked with
2081    /// the error and a completed-with-error future is returned
2082    /// (`Ok(failed_future)`). This matches Java's contract where `send()` always
2083    /// returns a `Future` for API errors and always invokes the callback.
2084    ///
2085    /// Everything else is propagated as `Err(...)`: the generic runtime errors
2086    /// (`LocalIllegalState` when the producer is closed, or `LocalIllegalArgument`
2087    /// when a custom partitioner returns a negative partition —
2088    /// `KafkaProducer#partition`, which escapes `doSend` through
2089    /// `catch (Exception e)` and is rethrown out of `send()`) and the
2090    /// `KafkaException`s that are not `ApiException`s. `SerializationException` is
2091    /// one of the latter — it extends `KafkaException` directly — so a
2092    /// serialization failure is returned as `Err`, not as a failed future.
2093    ///
2094    /// `wait` decides what happens at the two points where Java blocks for up to
2095    /// `max.block.ms` before the record reaches the accumulator — `waitOnMetadata`
2096    /// and `BufferPool.allocate`: with [`SendWait::MaxBlock`] this is Java's
2097    /// `doSend`; with [`SendWait::Never`] a wait at either point is reported as
2098    /// [`TrySendOutcome::WouldBlock`] instead, with the callback handed back
2099    /// un-invoked (see [`try_send`](Self::try_send)). The blocking callers never
2100    /// see `WouldBlock`.
2101    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#doSend")]
2102    async fn do_send(
2103        &self,
2104        record: ProducerRecord<K, V>,
2105        callback: Option<Box<dyn Callback>>,
2106        wait: SendWait,
2107    ) -> Result<TrySendOutcome, Error> {
2108        self.ensure_not_closed()?;
2109
2110        // First make sure the metadata for the topic is available
2111        let now_ms = self.now_ms();
2112        let max_block_ms = self.max_block_ms_for(wait);
2113        let cluster_and_wait_time = match self
2114            .wait_on_metadata(record.topic(), record.partition(), now_ms, max_block_ms)
2115            .await
2116        {
2117            Ok(cwt) => cwt,
2118            Err(e) => {
2119                // `KafkaProducer#doSend` relabels a closed-producer race first, then the
2120                // outer catches dispatch on the resulting class. The relabel comes
2121                // before the would-block check: a producer closed while the send
2122                // was in flight is an `Err` on both paths, not something to retry.
2123                let e = self.relabel_if_closed_while_sending(e);
2124                // The non-blocking path's first would-block point. With a zero
2125                // budget, `wait_on_metadata` fails with a `TimeoutException` exactly
2126                // when it would have had to wait — the metadata request it issued
2127                // stays in flight, so a later attempt can succeed. Every other error
2128                // (`InvalidTopicException`, the topic's own error, a fatal metadata
2129                // error) is a real failure and dispatches as it would on `send`.
2130                if wait == SendWait::Never && matches!(e, Error::Timeout(_)) {
2131                    return Ok(TrySendOutcome::WouldBlock(callback));
2132                }
2133                if e.is_api_error() {
2134                    // Java's `AppendCallbacks.topicPartition()` (`KafkaProducer`):
2135                    // no partition is computed yet, so the record's own, if it names one.
2136                    let partition = record.partition().unwrap_or(RecordMetadata::UNKNOWN_PARTITION);
2137                    return self
2138                        .handle_api_error(e, record.topic(), partition, callback)
2139                        .map(TrySendOutcome::Accepted);
2140                }
2141                return Err(e);
2142            },
2143        };
2144        let now_ms = now_ms + cluster_and_wait_time.waited_on_metadata_ms;
2145        let remaining_wait_ms = 0i64.max(max_block_ms - cluster_and_wait_time.waited_on_metadata_ms);
2146        let cluster = cluster_and_wait_time.cluster;
2147
2148        // Destructure the record to take ownership of key/value for serialization.
2149        // The headers stay owned and mutable until both serializers have run:
2150        // Java hands each serializer `record.headers()`, which it may add to
2151        // (`KafkaProducer`).
2152        let (record_topic, partition_opt, timestamp_opt, mut record_headers, key, value) = record.into_parts();
2153
2154        // Java's only catch around either serializer call is
2155        // `catch (ClassCastException cce)` (`KafkaProducer#doSend`),
2156        // which relabels a key/value whose runtime class does not match the
2157        // configured serializer. That cannot happen here — the serializer is
2158        // statically typed on `K` / `V` — so there is nothing to convert, and the
2159        // serializer's own error propagates with its class intact. `doSend`'s outer
2160        // catches then dispatch on that class exactly as Java does: an
2161        // `ApiException` (a schema-registry `TimeoutException`, say) fires the
2162        // callback and yields a failed future, anything else is returned
2163        // as `Err`. Rewriting every serializer error as
2164        // `SerializationException` flipped `is_retriable_error()` for the first
2165        // case and skipped its callback entirely.
2166        //
2167        // The serializers borrow `key` / `value`, so a serializer whose input
2168        // already is the bytes (`ByteArraySerializer`, `StringSerializer`) hands
2169        // back a borrow of the record's own buffer, and the accumulator's append
2170        // is the only copy (CLAUDE.md §14). Each serializer future lives inside
2171        // this method's own future, so awaiting one through `dyn` allocates
2172        // nothing (`InlineFuture`). The serialized bytes borrow only the record
2173        // and the serializer, not the headers, so the key's bytes are held while
2174        // the value serializer gets the same headers.
2175        let serialized_key = match DynSerializer::serialize_with_headers(
2176            &*self.key_serializer,
2177            &record_topic,
2178            &mut record_headers,
2179            key.as_ref(),
2180        )
2181        .await
2182        {
2183            Ok(bytes) => bytes,
2184            Err(e) if e.is_api_error() => {
2185                // `AppendCallbacks.topicPartition()`, as for the metadata wait.
2186                let partition = partition_opt.unwrap_or(RecordMetadata::UNKNOWN_PARTITION);
2187                return self
2188                    .handle_api_error(e, &record_topic, partition, callback)
2189                    .map(TrySendOutcome::Accepted);
2190            },
2191            Err(e) => return Err(e),
2192        };
2193
2194        let serialized_value = match DynSerializer::serialize_with_headers(
2195            &*self.value_serializer,
2196            &record_topic,
2197            &mut record_headers,
2198            value.as_ref(),
2199        )
2200        .await
2201        {
2202            Ok(bytes) => bytes,
2203            Err(e) if e.is_api_error() => {
2204                // `AppendCallbacks.topicPartition()`, as for the metadata wait.
2205                let partition = partition_opt.unwrap_or(RecordMetadata::UNKNOWN_PARTITION);
2206                return self
2207                    .handle_api_error(e, &record_topic, partition, callback)
2208                    .map(TrySendOutcome::Accepted);
2209            },
2210            Err(e) => return Err(e),
2211        };
2212
2213        // A custom partitioner is handed the *typed* key/value
2214        // (`KafkaProducer#partition` passes `record.key()` / `record.value()`),
2215        // so with one the partition is computed here, where the typed references
2216        // are still at hand. It is passed to `do_send_bytes` as an explicit
2217        // `Some(..)`, which short-circuits that method's own partition step so the
2218        // partitioner runs EXACTLY once, even a stateful one such as
2219        // `RoundRobinPartitioner`. Without one, `do_send_bytes` runs the built-in
2220        // key-hash partitioning on the serialized key.
2221        let partition = if self.partitioner.is_some() {
2222            Some(self.compute_partition(
2223                &record_topic,
2224                partition_opt,
2225                key.as_ref(),
2226                serialized_key.as_deref(),
2227                value.as_ref(),
2228                serialized_value.as_deref(),
2229                &cluster,
2230            )?)
2231        } else {
2232            partition_opt
2233        };
2234
2235        // `KafkaProducer`: once both serializers have run, the
2236        // headers are read-only and are sent as they are.
2237        Self::set_read_only(&mut record_headers);
2238        let headers = record_headers.to_array();
2239
2240        self.do_send_bytes(
2241            &record_topic,
2242            partition,
2243            timestamp_opt,
2244            serialized_key.as_deref(),
2245            serialized_value.as_deref(),
2246            headers,
2247            callback,
2248            now_ms,
2249            remaining_wait_ms,
2250            &cluster,
2251            wait,
2252        )
2253        .await
2254    }
2255
2256    /// The non-blocking form of `send`: the same path, except that it never waits
2257    /// on metadata or on buffer memory. It still awaits the key and value
2258    /// serializers, which complete at once when they are built-in but may wait
2259    /// for an asynchronous implementation; the C FFI therefore runs it on the
2260    /// calling thread only for a producer whose serializers are built-in.
2261    ///
2262    /// Java's `send` blocks for up to `max.block.ms` at two points before the
2263    /// record is appended — `waitOnMetadata` (`KafkaProducer`) and
2264    /// `BufferPool.allocate` inside `RecordAccumulator.append`. Where `send`
2265    /// would wait at either, this returns [`TrySendOutcome::WouldBlock`] at once,
2266    /// having done nothing to the record: no batch was touched, no future was
2267    /// created and `callback` was **not** invoked (it is handed back in the
2268    /// outcome). Everything else is exactly `send`: an appended record is
2269    /// [`TrySendOutcome::Accepted`] with its future, and a record `send` would
2270    /// have rejected with an `ApiException` — too large, say — is `Accepted` too,
2271    /// with the callback already fired and the future already failed, because
2272    /// the producer *did* resolve it. The `Err` cases are `send`'s `Err` cases: a
2273    /// closed producer, a bad partition, and the failures outside the
2274    /// `ApiException` hierarchy, a `SerializationException` among them.
2275    ///
2276    /// Asking for metadata is a side effect, not a wait, so an unknown topic is
2277    /// `WouldBlock` **and** a metadata request has been issued; the next attempt,
2278    /// once the response is in, is `Accepted`.
2279    ///
2280    /// There is no Java counterpart (DoD #7), which is why it is crate-internal:
2281    /// the public surface stays Java's `send`. The C FFI's non-blocking send uses
2282    /// it as its fast path — append on the caller's own thread when the producer
2283    /// is ready and its serializers are built-in, and fall back to the
2284    /// submission queue, rather than stall the caller, when it is not.
2285    pub(crate) async fn try_send(
2286        &self,
2287        record: ProducerRecord<K, V>,
2288        callback: Option<Box<dyn Callback>>,
2289    ) -> Result<TrySendOutcome, Error> {
2290        self.do_send(record, callback, SendWait::Never).await
2291    }
2292
2293    /// Common send path for already-serialized key/value bytes.
2294    ///
2295    /// Both [`do_send`](Self::do_send) (after serialization) and
2296    /// [`send`](KafkaProducer::<Vec<u8>, Vec<u8>>::send) (zero-copy borrowed path)
2297    /// delegate here for partition calculation, size validation, and accumulator
2298    /// append.
2299    ///
2300    /// `wait` decides what happens when the append has to wait for buffer memory:
2301    /// with [`SendWait::MaxBlock`] the `BufferExhaustedException` of an expired
2302    /// `remaining_wait_ms` is dispatched like any other `ApiException`; with
2303    /// [`SendWait::Never`] (`remaining_wait_ms` is then zero) it is reported as
2304    /// [`TrySendOutcome::WouldBlock`] instead, with the callback handed back
2305    /// un-invoked. The blocking callers therefore never see `WouldBlock`.
2306    #[expect(clippy::too_many_arguments)]
2307    async fn do_send_bytes(
2308        &self,
2309        topic: &str,
2310        partition: Option<i32>,
2311        timestamp: Option<i64>,
2312        key: Option<&[u8]>,
2313        value: Option<&[u8]>,
2314        headers: &[RecordHeader],
2315        callback: Option<Box<dyn Callback>>,
2316        now_ms: i64,
2317        remaining_wait_ms: i64,
2318        cluster: &Cluster,
2319        wait: SendWait,
2320    ) -> Result<TrySendOutcome, Error> {
2321        // `compute_partition` with `None` typed key/value: this method has only the
2322        // serialized bytes. When called from `do_send`'s custom-partitioner branch the
2323        // partition arrives pre-computed as `Some(..)`, so this short-circuits and the
2324        // partitioner is not consulted twice. On the borrowed `send(&[u8], &[u8])` path
2325        // the typed key/value ARE the bytes, so passing them as `serialized_*` (and
2326        // `None` typed) is the faithful call there.
2327        let partition = self.compute_partition(topic, partition, None, key, None, value, cluster)?;
2328
2329        // An `i64`: unlike a Java `byte[]`, a key or value slice can be longer than
2330        // `i32::MAX`, and such a record must still be rejected below.
2331        let serialized_size = AbstractRecords::estimate_size_in_bytes_upper_bound(
2332            RecordBatch::CURRENT_MAGIC_VALUE,
2333            self.compression_type,
2334            key,
2335            value,
2336            headers,
2337        );
2338        if let Err(err) = self.ensure_valid_record_size(serialized_size) {
2339            return self
2340                .handle_api_error(err, topic, partition, callback)
2341                .map(TrySendOutcome::Accepted);
2342        }
2343
2344        let timestamp = timestamp.unwrap_or(now_ms);
2345
2346        // Java's `accumulator.append(..)` dispatches virtually to `ChunkedRecordAccumulator.append`
2347        // for the incremental strategy (`KafkaProducer`); here the strategy picks the
2348        // accumulator explicitly (see `chunked_accumulator`). Each arm awaits its own concrete
2349        // future, so the dispatch costs a branch, not a boxed future (CLAUDE.md §13).
2350        let appended = match &self.chunked_accumulator {
2351            Some(chunked_accumulator) => {
2352                chunked_accumulator
2353                    .append(
2354                        topic,
2355                        partition,
2356                        timestamp,
2357                        key,
2358                        value,
2359                        headers,
2360                        callback,
2361                        remaining_wait_ms,
2362                        now_ms,
2363                        cluster,
2364                    )
2365                    .await
2366            },
2367            None => {
2368                self.accumulator
2369                    .append(
2370                        topic,
2371                        partition,
2372                        timestamp,
2373                        key,
2374                        value,
2375                        headers,
2376                        callback,
2377                        remaining_wait_ms,
2378                        now_ms,
2379                        cluster,
2380                    )
2381                    .await
2382            },
2383        };
2384        match appended {
2385            Ok(result) => {
2386                // `append` only ever returns an appended result, whose future and partition are
2387                // set (`RecordAppendResult::appended`); the other outcomes are signals between
2388                // the accumulator's own steps.
2389                let (Some(future), Some(topic_partition)) = (result.future, result.topic_partition) else {
2390                    return Err(Error::local_illegal_state("append returned a result that was not appended"));
2391                };
2392                // Add the partition to the transaction (if in progress) after it has
2393                // been successfully appended to the accumulator. We cannot do it
2394                // before because the partition may be unknown. Note that the `Sender`
2395                // will refuse to dequeue batches from the accumulator until they have
2396                // been added to the transaction (`KafkaProducer`).
2397                //
2398                // `topic_partition` is what Java reads back as
2399                // `appendCallbacks.topicPartition()`. It is borrowed, not rebuilt:
2400                // constructing it here from `topic: &str` would allocate a `String` and
2401                // an `Arc<str>` and copy the topic name twice on **every** record, which
2402                // CLAUDE.md §13 forbids on the send path. The accumulator interns one
2403                // `Arc<str>` per topic and hands the `TopicPartition` back, so this costs
2404                // nothing. (Critic 44 issue 1.)
2405                if let Some(transaction_manager) = &self.transaction_manager {
2406                    // `Caller::App`: this runs on the application task.
2407                    //
2408                    // The guard is bound to its own statement so it is released at the
2409                    // `;`. It MUST NOT stay alive into the error handling below:
2410                    // `handle_api_error` re-locks this same non-reentrant
2411                    // `std::sync::Mutex` through `maybe_transition_to_error_state`, and an
2412                    // `if let` scrutinee's temporaries live for the whole success arm —
2413                    // edition 2024 only shortens them across the `else`. Written inline,
2414                    // this self-deadlocks the application task.
2415                    let add_partition = transaction_manager.lock().unwrap().maybe_add_partition(&topic_partition);
2416                    if let Err(error) = add_partition {
2417                        // `maybeAddPartition` throws across two of `doSend`'s catch
2418                        // blocks, so the error class decides how the failure surfaces:
2419                        //
2420                        // - `ProducerFencedException` / `InvalidProducerEpochException`
2421                        //   (`TransactionManager#maybeFailWithError`)
2422                        //   are `ApiException`s: `catch (ApiException e)` records the
2423                        //   error state and returns a failed future.
2424                        // - `IllegalStateException` — a send that is out of order with
2425                        //   the transactional API (`TransactionManager#maybeAddPartition`),
2426                        //   or a previous operation that timed out
2427                        //   (`TransactionManager#throwIfPendingState`), or a previous
2428                        //   invalid transition (`TransactionManager#maybeFailWithError`) —
2429                        //   is not even a `KafkaException`, so it reaches
2430                        //   `catch (Exception e)` and is rethrown out of `send()`.
2431                        // - The bare `KafkaException` of `maybeFailWithError`'s last line is not an
2432                        //   `ApiException` either, so `catch (KafkaException e)`
2433                        //   rethrows it as well. Neither rethrowing block calls
2434                        //   `maybeTransitionToErrorState`.
2435                        //
2436                        // `is_api_error()` alone is the whole test. It used to be
2437                        // `&& error.error() != Errors::UnknownServerError`, because a bare
2438                        // `KafkaException` was then spelled
2439                        // `Error::with_message(Errors::UnknownServerError, ..)`, which
2440                        // resolves the code to `UnknownServerException` — an
2441                        // `ApiException`. Every producer site now builds a bare
2442                        // `KafkaException` as `Error::kafka_message(..)` / `Error::kafka_message_source(..)`
2443                        // (the `Error::KafkaError` variant), for which `is_api_error()`
2444                        // answers `false` directly, so the code-based workaround is gone.
2445                        if error.is_api_error() {
2446                            let partition = topic_partition.partition();
2447                            // `None`, not a callback: unlike the pre-append failure paths
2448                            // (`ensure_valid_record_size`, `wait_on_metadata`), the record
2449                            // has ALREADY been appended here, and `append` above took
2450                            // ownership of `callback` and registered it with the record's
2451                            // future. The callback is therefore both (a) moved — no longer
2452                            // available at this point — and (b) already carried by that
2453                            // future, which fires it exactly once when the batch is later
2454                            // completed/aborted. Firing it here too would double-invoke it,
2455                            // breaking the exactly-once-per-record contract (CLAUDE.md §11.5).
2456                            // (Java's `doSend` catch fires the raw `callback` at
2457                            // `KafkaProducer#doSend`, but keeps it as a reference separate
2458                            // from the `appendCallbacks` it registered, so Java can fire it
2459                            // twice on this path; Rust's single-owner model fires it once.)
2460                            return self
2461                                .handle_api_error(error, topic, partition, None)
2462                                .map(TrySendOutcome::Accepted);
2463                        }
2464                        return Err(error);
2465                    }
2466                }
2467
2468                if result.batch_is_full || result.new_batch_created {
2469                    kafka_trace!(
2470                        self.log_context,
2471                        "Waking up the sender since topic {} is either full or getting a new batch",
2472                        topic
2473                    );
2474                    self.wakeup.notify_one();
2475                }
2476                Ok(TrySendOutcome::Accepted(KafkaFuture::new(future)))
2477            },
2478            // The non-blocking path's second would-block point. `remaining_wait_ms`
2479            // is zero here, so the pool gave up at once with `BufferExhaustedException`
2480            // (`BufferPool.allocate`, or `try_allocate_chunks` on the incremental
2481            // strategy) — on `send` with `max.block.ms = 0` that is an `ApiException`
2482            // and takes the arm below; `try_send` instead hands the record (and its
2483            // still un-invoked callback) back so the caller can retry it with `send`
2484            // when it can afford to block. The pool is untouched: a failed allocation
2485            // releases whatever it accumulated before returning.
2486            Err(failure) if wait == SendWait::Never && matches!(failure.error, Error::ProducerBufferExhausted(_)) => {
2487                Ok(TrySendOutcome::WouldBlock(failure.callback))
2488            },
2489            // Java's `catch (ApiException e)` (`KafkaProducer`) fires the
2490            // user `Callback` with a null-metadata `RecordMetadata(tp, -1, -1,
2491            // NO_TIMESTAMP, -1, -1)` *and* returns a failed future. `append` gives the
2492            // callback back (`AppendFailure::callback`) precisely so this arm can honour
2493            // that obligation exactly once (CLAUDE.md §11.5) — the four sibling arms
2494            // covering the same Java block all route through `handle_api_error` too.
2495            //
2496            // The callback is given the partition `append` chose, as Java reports
2497            // `appendCallbacks.topicPartition()`, not the
2498            // `partition` passed in, which is `UNKNOWN_PARTITION` when the built-in
2499            // partitioner chooses.
2500            Err(failure) if failure.error.is_api_error() => self
2501                .handle_api_error(failure.error, topic, failure.partition, failure.callback)
2502                .map(TrySendOutcome::Accepted),
2503            // Java's `catch (KafkaException e)` / `catch (Exception e)`
2504            // rethrow, and neither invokes the callback. `failure.callback` is dropped
2505            // here, exactly as Java drops its `appendCallbacks` reference when `send()`
2506            // throws.
2507            Err(failure) => Err(failure.error),
2508        }
2509    }
2510
2511    /// `transactionManager.maybeTransitionToErrorState(e)`, the tail of
2512    /// `KafkaProducer.doSend`'s `catch (ApiException e)` block
2513    /// (`KafkaProducer#doSend`).
2514    ///
2515    /// [`Caller::App`](crate::producer::internals::Caller::App): `doSend` runs on the
2516    /// application task.
2517    ///
2518    /// Locks the [`TransactionManager`]. Java's monitor is reentrant and
2519    /// `std::sync::Mutex` is not, so no caller — directly or through
2520    /// [`handle_api_error`](Self::handle_api_error) — may already hold that
2521    /// lock, including in a still-live `if let` / `match` scrutinee temporary.
2522    fn maybe_transition_to_error_state(&self, error: &Error) {
2523        if let Some(transaction_manager) = &self.transaction_manager {
2524            // Java lets an invalid transition propagate out of `doSend`. That cannot
2525            // happen on the idempotent path — the only transition
2526            // `maybeTransitionToErrorState` performs is to `FATAL_ERROR`, which is
2527            // always valid — and swallowing it here would hide a Phase-5 regression,
2528            // so it is logged rather than dropped.
2529            if let Err(transition_error) = transaction_manager
2530                .lock()
2531                .unwrap()
2532                .maybe_transition_to_error_state(error, Caller::App)
2533            {
2534                kafka_warn!(
2535                    self.log_context,
2536                    "Failed to record a send error in the transaction manager: {}",
2537                    transition_error
2538                );
2539            }
2540        }
2541    }
2542
2543    /// `doSend`'s **inner** `catch (KafkaException e)` around `waitOnMetadata`
2544    /// (`KafkaProducer#doSend`):
2545    ///
2546    /// ```java
2547    /// } catch (KafkaException e) {
2548    ///     if (metadata.isClosed())
2549    ///         throw new KafkaException("Producer closed while send in progress", e);
2550    ///     throw e;
2551    /// }
2552    /// ```
2553    ///
2554    /// Its whole job is to relabel a `close()` racing an in-flight send, so the
2555    /// caller can tell it apart from a metadata timeout. Anything else — and
2556    /// anything outside the `KafkaException` hierarchy — passes through untouched,
2557    /// to be dispatched by the outer `catch (ApiException e)` /
2558    /// `catch (KafkaException e)` split at the call site.
2559    fn relabel_if_closed_while_sending(&self, error: Error) -> Error {
2560        if error.is_kafka_error() && self.metadata.is_closed() {
2561            // A bare `KafkaException`, matching Java: not an `ApiException`, so the
2562            // caller returns it as `Err` rather than as a failed future.
2563            return Error::kafka_message_source("Producer closed while send in progress", error);
2564        }
2565        error
2566    }
2567
2568    /// Handle an `ApiException`-type error by invoking the callback (if any)
2569    /// and returning a completed-with-error future.
2570    ///
2571    /// This matches Java's `catch (ApiException e)` block in `doSend()`.
2572    fn handle_api_error(
2573        &self,
2574        error: Error,
2575        topic: &str,
2576        partition: i32,
2577        callback: Option<Box<dyn Callback>>,
2578    ) -> Result<KafkaFuture<RecordMetadata>, Error> {
2579        kafka_debug!(self.log_context, "Error occurred during message send: {}", error);
2580        self.maybe_transition_to_error_state(&error);
2581        if let Some(cb) = callback {
2582            let tp = TopicPartition::new(topic.to_string(), partition);
2583            let null_metadata = RecordMetadata::new(tp, -1, -1, RecordBatch::NO_TIMESTAMP, -1, -1);
2584            cb.on_completion(Some(&null_metadata), Some(&error));
2585        }
2586        let tp = TopicPartition::new(topic.to_string(), partition);
2587        Ok(KafkaFuture::new(Arc::new(FutureRecordMetadata::failed(tp, error))))
2588    }
2589
2590    /// Wait for cluster metadata including partitions for the given topic to be available.
2591    ///
2592    /// Translated from `KafkaProducer.waitOnMetadata()`.
2593    ///
2594    /// # Arguments
2595    /// * `topic` - The topic we want metadata for
2596    /// * `partition` - A specific partition expected to exist in metadata, or `None`
2597    /// * `now_ms` - The current time in ms
2598    /// * `max_wait_ms` - The maximum time in ms for waiting on the metadata
2599    ///
2600    /// # Returns
2601    /// The cluster containing topic metadata and the amount of time we waited in ms.
2602    ///
2603    /// # Errors
2604    /// Returns `Err` if:
2605    /// - The topic is invalid ([`InvalidTopic`](Error::InvalidTopic))
2606    /// - Metadata could not be refreshed within `max_wait_ms` ([`Timeout`](Error::Timeout))
2607    /// - The producer is closed
2608    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#waitOnMetadata")]
2609    async fn wait_on_metadata(
2610        &self,
2611        topic: &str,
2612        partition: Option<i32>,
2613        now_ms: i64,
2614        max_wait_ms: i64,
2615    ) -> Result<ClusterAndWaitTime, Error> {
2616        let cluster = self.metadata.fetch();
2617
2618        if cluster.invalid_topics().contains(topic) {
2619            // Java: `throw new InvalidTopicException(topic)` — `topic` is a
2620            // `String`, so this binds to the `(String message)` constructor:
2621            // the message is the topic name and `invalidTopics()` is empty.
2622            return Err(Error::invalid_topics_message(
2623                std::collections::HashSet::new(),
2624                topic.to_string(),
2625            ));
2626        }
2627
2628        // Add topic to metadata topic list if it is not there already and reset expiry
2629        self.metadata.add(topic, now_ms);
2630
2631        let partitions_count = cluster.partition_count_for_topic(topic);
2632        // Return cached metadata if we have it, and if the record's partition is either
2633        // undefined or within the known partition range
2634        if let Some(count) = partitions_count
2635            && (partition.is_none() || partition.unwrap() < count as i32)
2636        {
2637            return Ok(ClusterAndWaitTime { cluster, waited_on_metadata_ms: 0 });
2638        }
2639
2640        let mut remaining_wait_ms = max_wait_ms;
2641        let mut elapsed: i64 = 0;
2642        let mut partitions_count = partitions_count;
2643
2644        // Java `waitOnMetadata`: `long nowNanos = time.nanoseconds()` right
2645        // before the refresh loop (after the early cached-metadata return), and
2646        // `producerMetrics.recordMetadataWait(time.nanoseconds() - nowNanos)`
2647        // once the loop succeeds. A timeout inside the loop throws before the
2648        // record, so only successful waits are recorded — preserved here.
2649        let now_nanos = self.time.nanoseconds();
2650
2651        // Issue metadata requests until we have metadata for the topic and the
2652        // requested partition, or until max_wait_ms is exceeded.
2653        loop {
2654            if let Some(p) = partition {
2655                kafka_trace!(
2656                    self.log_context,
2657                    "Requesting metadata update for partition {} of topic {}.",
2658                    p,
2659                    topic
2660                );
2661            } else {
2662                kafka_trace!(self.log_context, "Requesting metadata update for topic {}.", topic);
2663            }
2664            self.metadata.add(topic, now_ms + elapsed);
2665            let version = self.metadata.request_update_for_topic(topic);
2666            self.wakeup.notify_one();
2667
2668            match self.metadata.await_update(version, remaining_wait_ms).await {
2669                Ok(()) => {},
2670                // A fatal metadata error (an authentication failure, say) must
2671                // reach the caller as itself. Java's `awaitUpdate` throws it from
2672                // the wait predicate via `maybeThrowFatalException()`, so it never
2673                // becomes a `TimeoutException`; only an actual deadline expiry does.
2674                // Re-wrapping everything as a timeout made a dead producer look
2675                // retriable, so the application retried bad credentials forever.
2676                Err(e) if !e.is_timeout_error() => return Err(e),
2677                Err(_) => {
2678                    // Rethrow with the original `max_wait_ms` to keep the message
2679                    // free of the shrinking `remaining_wait_ms` (`KafkaProducer#waitOnMetadata`).
2680                    let error_message = self.get_error_message(partitions_count, topic, partition, max_wait_ms);
2681                    if let Some(code) = self.metadata.get_error(topic) {
2682                        // `new TimeoutException(errorMessage, metadata.getError(topic).exception())`:
2683                        // the broker error is the timeout's *cause*, so
2684                        // `Error::source()` can be walked back to it. Flattening it
2685                        // into the message left `source()` empty.
2686                        return Err(Error::Timeout(TimeoutError::with_source(error_message, Error::new(code))));
2687                    }
2688                    return Err(Error::timeout(error_message));
2689                },
2690            }
2691
2692            let cluster = self.metadata.fetch();
2693            elapsed = self.now_ms() - now_ms;
2694            if elapsed >= max_wait_ms {
2695                let error_message = self.get_error_message(partitions_count, topic, partition, max_wait_ms);
2696                // `KafkaProducer#waitOnMetadata` attaches the topic's error as the cause here too,
2697                // but only when it is retriable — a non-retriable one is about to be
2698                // raised as itself by `maybe_return_error_for_topic` on the next
2699                // iteration, so pinning it under a timeout would mislabel it.
2700                if let Some(code) = self.metadata.get_error(topic) {
2701                    let underlying = Error::new(code);
2702                    if underlying.is_retriable_error() {
2703                        return Err(Error::Timeout(TimeoutError::with_source(error_message, underlying)));
2704                    }
2705                }
2706                return Err(Error::timeout(error_message));
2707            }
2708            self.metadata.maybe_return_error_for_topic(topic)?;
2709            remaining_wait_ms = max_wait_ms - elapsed;
2710            partitions_count = cluster.partition_count_for_topic(topic);
2711
2712            let done = match partitions_count {
2713                None => false,
2714                Some(count) => partition.is_none() || partition.unwrap() < count as i32,
2715            };
2716            if done {
2717                self.producer_metrics.record_metadata_wait(self.time.nanoseconds() - now_nanos);
2718                return Ok(ClusterAndWaitTime { cluster, waited_on_metadata_ms: elapsed });
2719            }
2720        }
2721    }
2722
2723    /// Format the error message for a metadata wait timeout.
2724    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#getErrorMessage")]
2725    fn get_error_message(
2726        &self,
2727        partitions_count: Option<usize>,
2728        topic: &str,
2729        partition: Option<i32>,
2730        max_wait_ms: i64,
2731    ) -> String {
2732        match partitions_count {
2733            None => format!("Topic {} not present in metadata after {} ms.", topic, max_wait_ms),
2734            Some(count) => format!(
2735                "Partition {} of topic {} with partition count {} is not present in metadata after {} ms.",
2736                partition.unwrap_or(-1),
2737                topic,
2738                count,
2739                max_wait_ms
2740            ),
2741        }
2742    }
2743
2744    /// Validate that the record size isn't too large.
2745    ///
2746    /// Translated from `KafkaProducer.ensureValidRecordSize()`.
2747    ///
2748    /// Takes an `i64` so that a size above `i32::MAX`, which
2749    /// `AbstractRecords::estimate_size_in_bytes_upper_bound` can return, is rejected
2750    /// and reported exactly.
2751    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#ensureValidRecordSize")]
2752    fn ensure_valid_record_size(&self, size: i64) -> Result<(), Error> {
2753        if size > self.max_request_size as i64 {
2754            return Err(Error::record_too_large(format!(
2755                "The message is {} bytes when serialized which is larger than {}, which is the value of the {} configuration.",
2756                size,
2757                self.max_request_size,
2758                ProducerConfig::MAX_REQUEST_SIZE_CONFIG
2759            )));
2760        }
2761        if size > self.total_memory_size {
2762            return Err(Error::record_too_large(format!(
2763                "The message is {} bytes when serialized which is larger than the total memory buffer you have configured with the {} configuration.",
2764                size,
2765                ProducerConfig::BUFFER_MEMORY_CONFIG
2766            )));
2767        }
2768        Ok(())
2769    }
2770
2771    /// Resolve the key-based partition when no explicit partition was supplied,
2772    /// or [`UNKNOWN_PARTITION`](RecordMetadata::UNKNOWN_PARTITION) to defer to
2773    /// the keyless (sticky, KIP-794) path.
2774    ///
2775    /// Mirrors the key branch of Java's `KafkaProducer.partition()`: the key is
2776    /// hashed only when it is present, keys are not being ignored
2777    /// (`partitioner.ignore.keys=false`), the configured [`KeyHasher`] actually
2778    /// hashes this key ([`KeyHasher::hashes_key`] — CRC-32 leaves an *empty* key
2779    /// to the keyless path, matching librdkafka `consistent_random`; murmur2
2780    /// hashes it, matching the Java client), and the topic has known partitions.
2781    ///
2782    /// Both send call sites ([`do_send_bytes`](Self::do_send_bytes) and
2783    /// [`partition`](Self::partition)) route their key branch through this one
2784    /// helper so the hashing gate cannot diverge between them.
2785    fn partition_for_key_or_unknown(&self, key: Option<&[u8]>, topic: &str, cluster: &Cluster) -> i32 {
2786        if let Some(k) = key
2787            && !self.partitioner_ignore_keys
2788            && self.key_hasher.hashes_key(k)
2789        {
2790            let num_partitions = cluster.partitions_for_topic(topic).len() as i32;
2791            if num_partitions > 0 {
2792                return BuiltInPartitioner::partition_for_key(k, num_partitions, self.key_hasher);
2793            }
2794        }
2795        RecordMetadata::UNKNOWN_PARTITION
2796    }
2797
2798    /// Compute the partition for a record, mirroring Java's `KafkaProducer.partition()`
2799    /// (`KafkaProducer#partition`).
2800    ///
2801    /// Precedence is exactly Java's:
2802    /// 1. an explicit `partition` on the record wins outright;
2803    /// 2. otherwise, if a custom [`Partitioner`] is configured, it decides — and a
2804    ///    negative result is rejected with the same `IllegalArgumentException` message
2805    ///    Java throws. In Java that exception escapes `doSend`'s `catch (Exception e)`
2806    ///    and is rethrown out of `send()`; here it is the `Err` variant, propagated by
2807    ///    the caller with `?`.
2808    /// 3. otherwise fall back to the built-in key-hash / `UNKNOWN_PARTITION` path
2809    ///    ([`partition_for_key_or_unknown`](Self::partition_for_key_or_unknown)).
2810    ///
2811    /// The typed `key` / `value` are handed straight to the custom partitioner (Java
2812    /// passes it `record.key()` / `record.value()` alongside the serialized bytes); the
2813    /// built-in fallback in step 3 uses only `serialized_key`.
2814    #[expect(clippy::too_many_arguments)]
2815    fn compute_partition(
2816        &self,
2817        topic: &str,
2818        partition: Option<i32>,
2819        key: Option<&K>,
2820        serialized_key: Option<&[u8]>,
2821        value: Option<&V>,
2822        serialized_value: Option<&[u8]>,
2823        cluster: &Cluster,
2824    ) -> Result<i32, Error> {
2825        if let Some(p) = partition {
2826            return Ok(p);
2827        }
2828
2829        if let Some(partitioner) = &self.partitioner {
2830            let custom_partition = partitioner.partition(topic, key, serialized_key, value, serialized_value, cluster);
2831            if custom_partition < 0 {
2832                return Err(Error::local_illegal_argument(format!(
2833                    "The partitioner generated an invalid partition number: {}. Partition number should always be non-negative.",
2834                    custom_partition
2835                )));
2836            }
2837            return Ok(custom_partition);
2838        }
2839
2840        Ok(self.partition_for_key_or_unknown(serialized_key, topic, cluster))
2841    }
2842
2843    /// Compute partition for the given record.
2844    ///
2845    /// A thin test wrapper over [`compute_partition`](Self::compute_partition), which is
2846    /// the faithful translation of `KafkaProducer.partition()`. Every caller is a test
2847    /// that uses a producer WITHOUT a custom partitioner, so the negative-partition `Err`
2848    /// path is unreachable and unwrapping it here is safe.
2849    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#partition")]
2850    fn partition(
2851        &self,
2852        record: &ProducerRecord<K, V>,
2853        serialized_key: Option<&[u8]>,
2854        serialized_value: Option<&[u8]>,
2855        cluster: &Cluster,
2856    ) -> i32 {
2857        self.compute_partition(
2858            record.topic(),
2859            record.partition(),
2860            record.key(),
2861            serialized_key,
2862            record.value(),
2863            serialized_value,
2864            cluster,
2865        )
2866        .expect("partition() test helper is only used without a custom partitioner")
2867    }
2868
2869    /// Initiate a graceful close of the sender.
2870    ///
2871    /// Closes the accumulator first to guarantee that no more appends are
2872    /// accepted after breaking from the sender loop. Otherwise, we may miss
2873    /// some callbacks when shutting down.
2874    ///
2875    /// Translated from `Sender.initiateClose()`.
2876    fn initiate_close(&self) {
2877        // Ensure accumulator is closed first to guarantee that no more appends
2878        // are accepted after breaking from the sender loop.
2879        self.accumulator.close();
2880        self.running.store(false, Ordering::Release);
2881        self.wakeup.notify_one();
2882    }
2883
2884    /// Whether no close has started on this producer.
2885    ///
2886    /// Every close clears `running` in [`Self::initiate_close`].
2887    pub(crate) fn is_running(&self) -> bool {
2888        self.running.load(Ordering::Acquire)
2889    }
2890
2891    /// Force-close the sender, aborting all pending batches.
2892    fn force_close(&self) {
2893        self.force_close.store(true, Ordering::Release);
2894        self.initiate_close();
2895    }
2896
2897    /// Await the sender task handle with a timeout.
2898    ///
2899    /// Takes the `JoinHandle` from the mutex and awaits it with the given
2900    /// timeout. Returns `true` if the sender task completed within the
2901    /// timeout, `false` if it is still running.
2902    ///
2903    /// If there is no sender handle (e.g. in tests that don't spawn a sender),
2904    /// returns `true` immediately.
2905    ///
2906    /// Corresponds to Java's `ioThread.join(closeTimer.remainingMs())`.
2907    ///
2908    /// On expiry the handle is **put back**, because Java's `close` force-closes and
2909    /// then joins unconditionally (`KafkaProducer#close`) — dropping it here
2910    /// would leave [`Self::await_sender_handle_indefinitely`] with nothing to join and
2911    /// `close` would return while the Sender task was still running, which CLAUDE.md
2912    /// §11.4 forbids. `JoinHandle` is `Unpin`, so `&mut` is enough to await it without
2913    /// giving it away.
2914    async fn await_sender_handle(&self, timeout: Duration) -> bool {
2915        let handle = self.sender_handle.lock().unwrap().take();
2916        match handle {
2917            None => true,
2918            Some(mut join_handle) => {
2919                let completed = tokio::time::timeout(timeout, &mut join_handle).await.is_ok();
2920                if !completed {
2921                    *self.sender_handle.lock().unwrap() = Some(join_handle);
2922                }
2923                completed
2924            },
2925        }
2926    }
2927
2928    /// Await the sender task handle indefinitely.
2929    ///
2930    /// Called after force-close to ensure the sender task has exited.
2931    /// Corresponds to Java's `ioThread.join()` (no timeout).
2932    async fn await_sender_handle_indefinitely(&self) {
2933        let handle = self.sender_handle.lock().unwrap().take();
2934        if let Some(join_handle) = handle {
2935            let _ = join_handle.await;
2936        }
2937    }
2938}
2939
2940impl KafkaProducer<Vec<u8>, Vec<u8>> {
2941    /// Asynchronously send a record to a topic and invoke the provided callback
2942    /// when the send has been acknowledged. See
2943    /// [`Producer::send_with_callback`] for details.
2944    ///
2945    /// # Arguments
2946    ///
2947    /// - `record` - The record to send. If the topic or the partition specified in it cannot be
2948    ///   found in metadata within `max.block.ms`, the returned future will time out when retrieved.
2949    /// - `callback` - A user-supplied callback to execute when the record has been acknowledged by
2950    ///   the server (`None` indicates no callback)
2951    ///
2952    /// # Returns
2953    ///
2954    /// A future which will eventually contain the response information.
2955    ///
2956    /// # Errors
2957    ///
2958    /// - `Error::is_local_illegal_state_error` if a transactional.id has been configured and no
2959    ///   transaction has been started, or when send is invoked after producer has been closed.
2960    /// - `Error::is_interrupt_error` If the task is interrupted while blocked
2961    /// - `Error::is_serialization_error` If the key or value are not valid objects given the
2962    ///   configured serializers
2963    /// - `Error::is_bootstrap_resolution_error` if DNS resolution of the bootstrap servers fails
2964    ///   within `bootstrap.resolve.timeout.ms`
2965    /// - `Error::is_kafka_error` If a Kafka related error occurs that does not belong to the public
2966    ///   API errors.
2967    ///
2968    /// # Translation notes
2969    ///
2970    /// The byte-slice key and value bypass serialization, so no serialization
2971    /// error is returned: they are written straight into the batch buffer, the
2972    /// zero-copy path for callers that already hold `&[u8]` data (the C FFI).
2973    /// A Rust task is never interrupted: dropping the future cancels the wait.
2974    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#send")]
2975    pub fn send(
2976        &self,
2977        record: ProducerRecord<&[u8], &[u8]>,
2978        callback: Option<Box<dyn Callback>>,
2979    ) -> impl Future<Output = Result<KafkaFuture<RecordMetadata>, Error>> + Send {
2980        async move {
2981            self.ensure_not_closed()?;
2982
2983            let now_ms = self.now_ms();
2984            let cluster_and_wait_time = match self
2985                .wait_on_metadata(record.topic(), record.partition(), now_ms, self.max_block_ms)
2986                .await
2987            {
2988                Ok(cwt) => cwt,
2989                Err(e) => {
2990                    // `KafkaProducer#doSend`'s relabel, as in `do_send`.
2991                    let e = self.relabel_if_closed_while_sending(e);
2992                    if e.is_api_error() {
2993                        // Java's `AppendCallbacks.topicPartition()` (`KafkaProducer`):
2994                        // no partition is computed yet, so the record's own, if it names one.
2995                        let partition = record.partition().unwrap_or(RecordMetadata::UNKNOWN_PARTITION);
2996                        return self.handle_api_error(e, record.topic(), partition, callback);
2997                    }
2998                    return Err(e);
2999                },
3000            };
3001            let now_ms = now_ms + cluster_and_wait_time.waited_on_metadata_ms;
3002            let remaining_wait_ms = 0i64.max(self.max_block_ms - cluster_and_wait_time.waited_on_metadata_ms);
3003            let cluster = cluster_and_wait_time.cluster;
3004
3005            let (record_topic, partition, timestamp, mut record_headers, key, value) = record.into_parts();
3006
3007            // Java's `doSend` makes the headers read-only and appends
3008            // `record.headers().toArray()` (`KafkaProducer#doSend`,
3009            // `KafkaProducer#doSend`); the header bytes are written straight into the
3010            // batch buffer, as on the typed path.
3011            Self::set_read_only(&mut record_headers);
3012            self.do_send_bytes(
3013                &record_topic,
3014                partition,
3015                timestamp,
3016                key,
3017                value,
3018                record_headers.to_array(),
3019                callback,
3020                now_ms,
3021                remaining_wait_ms,
3022                &cluster,
3023                SendWait::MaxBlock,
3024            )
3025            .await?
3026            .into_accepted_future()
3027        }
3028    }
3029}
3030
3031impl<K, V> Producer<K, V> for KafkaProducer<K, V>
3032where
3033    K: Send + Sync,
3034    V: Send + Sync,
3035{
3036    // Needs to be called before any other method when the `transactional.id` is
3037    // set in the configuration.
3038    async fn init_transactions(&self) -> Result<(), Error> {
3039        KafkaProducer::init_transactions(self).await
3040    }
3041
3042    // Should be called before the start of each new transaction.
3043    fn begin_transaction(&self) -> Result<(), Error> {
3044        KafkaProducer::begin_transaction(self)
3045    }
3046
3047    // Sends a list of specified offsets to the consumer group coordinator, and
3048    // also marks those offsets as part of the current transaction.
3049    async fn send_offsets_to_transaction(
3050        &self,
3051        offsets: HashMap<TopicPartition, OffsetAndMetadata>,
3052        group_metadata: &dyn ConsumerGroupMetadata,
3053    ) -> Result<(), Error> {
3054        KafkaProducer::send_offsets_to_transaction(self, offsets, group_metadata).await
3055    }
3056
3057    // Commits the ongoing transaction.
3058    async fn commit_transaction(&self) -> Result<(), Error> {
3059        KafkaProducer::commit_transaction(self).await
3060    }
3061
3062    // Aborts the ongoing transaction.
3063    async fn abort_transaction(&self) -> Result<(), Error> {
3064        KafkaProducer::abort_transaction(self).await
3065    }
3066
3067    // Asynchronously send a record to a topic.
3068    //
3069    // See [`send_with_callback`](Producer::send_with_callback) for details.
3070    async fn send(&self, record: ProducerRecord<K, V>) -> Result<KafkaFuture<RecordMetadata>, Error> {
3071        self.do_send(record, None, SendWait::MaxBlock).await?.into_accepted_future()
3072    }
3073
3074    // Asynchronously send a record to a topic and invoke the provided callback
3075    // when the send has been acknowledged.
3076    async fn send_with_callback(
3077        &self,
3078        record: ProducerRecord<K, V>,
3079        callback: Option<Box<dyn Callback>>,
3080    ) -> Result<KafkaFuture<RecordMetadata>, Error> {
3081        self.do_send(record, callback, SendWait::MaxBlock).await?.into_accepted_future()
3082    }
3083
3084    // Invoking this method makes all buffered records immediately available to
3085    // send and awaits the completion of the requests associated with these
3086    // records.
3087    //
3088    // Translated from `KafkaProducer.flush()`.
3089    async fn flush(&self) -> Result<(), Error> {
3090        kafka_trace!(self.log_context, "Flushing accumulated records in producer.");
3091        // Java: `long start = time.nanoseconds()` then a try/finally recording
3092        // `producerMetrics.recordFlush(time.nanoseconds() - start)`
3093        // (`KafkaProducer`). `await_flush_completion` returns
3094        // `()` (no error channel), so the finally reduces to recording after
3095        // the await.
3096        let start = self.time.nanoseconds();
3097        self.accumulator.begin_flush();
3098        self.wakeup.notify_one();
3099        self.accumulator.await_flush_completion().await;
3100        self.producer_metrics.record_flush(self.time.nanoseconds() - start);
3101        Ok(())
3102    }
3103
3104    // Get the full set of producer metrics maintained by this producer.
3105    //
3106    // Translated from `KafkaProducer.metrics()` — a live view of the registry.
3107    fn metrics(&self) -> MetricsView {
3108        self.metrics.metrics()
3109    }
3110
3111    // Get the partition metadata for the given topic.
3112    async fn partitions_for(&self, topic: &str) -> Result<Vec<PartitionInfo>, Error> {
3113        let now_ms = self.now_ms();
3114        let cluster_and_wait_time = self.wait_on_metadata(topic, None, now_ms, self.max_block_ms).await?;
3115        Ok(cluster_and_wait_time.cluster.partitions_for_topic(topic).to_vec())
3116    }
3117
3118    // Close this producer. This method awaits until all previously sent requests
3119    // complete.
3120    async fn close(&self) -> Result<(), Error> {
3121        self.close_with_timeout(Duration::from_millis(i64::MAX as u64)).await
3122    }
3123
3124    // Close this producer, waiting up to the given timeout for pending requests
3125    // to complete.
3126    //
3127    // Translated from `KafkaProducer.close(Duration timeout)`.
3128    //
3129    // If `timeout > 0`: initiates a graceful close and awaits the sender task up
3130    // to the remaining time. If the sender task is still alive after the timeout,
3131    // it is force-closed and awaited indefinitely.
3132    //
3133    // If `timeout == 0`: force-closes immediately without draining.
3134    //
3135    // A timeout whose length in milliseconds exceeds `i64::MAX`, such as
3136    // `Duration::MAX`, is clamped to `i64::MAX` ms, the value passed by `close()`.
3137    // The producer then closes gracefully and waits for the sender task to finish.
3138    //
3139    // Note: Rust's `Duration` is unsigned, so the negative-timeout check from
3140    // Java is omitted (impossible to construct a negative `Duration`).
3141    async fn close_with_timeout(&self, timeout: Duration) -> Result<(), Error> {
3142        // Java: `long timeoutMs = timeout.toMillis()`. The millisecond count is
3143        // clamped rather than cast with `as i64`, which truncates it to the low 64
3144        // bits and can produce a negative or zero value that neither branch below
3145        // handles as intended. Java's `toMillis()` throws `ArithmeticException` on
3146        // overflow; clamping is preferred here because `Duration::MAX` is the
3147        // conventional way to express an unbounded timeout in Rust.
3148        // `AsyncKafkaConsumer` applies the same clamp on close. The `Duration` is
3149        // normalized to the same whole-millisecond value, so the sender wait below
3150        // uses the timeout that is logged and checked, as Java joins the I/O thread
3151        // for `timeoutMs`.
3152        let timeout = Duration::from_millis(timeout.as_millis().min(i64::MAX as u128) as u64);
3153        let timeout_ms = timeout.as_millis() as i64;
3154        kafka_info!(
3155            self.log_context,
3156            "Closing the Kafka producer with timeoutMillis = {} ms.",
3157            timeout_ms
3158        );
3159
3160        // Track whether the sender is still alive after the graceful close attempt.
3161        let mut sender_still_alive = false;
3162
3163        if timeout_ms > 0 {
3164            // Try to close gracefully: close accumulator, set running=false, wake sender.
3165            self.initiate_close();
3166
3167            // Await the sender task with the remaining timeout.
3168            sender_still_alive = !self.await_sender_handle(timeout).await;
3169        }
3170
3171        if timeout_ms == 0 || sender_still_alive {
3172            // Force close if timeout is 0 or sender is still alive after timeout
3173            kafka_info!(
3174                self.log_context,
3175                "Proceeding to force close the producer since pending requests could not be \
3176                 completed within timeout {} ms.",
3177                timeout_ms
3178            );
3179            self.force_close();
3180
3181            // Await the sender task indefinitely after force close.
3182            self.await_sender_handle_indefinitely().await;
3183        }
3184
3185        // Java `close`'s `Utils.closeQuietly(...)` chain, in order:
3186        // `producerMetrics` → `metrics` → `keySerializer` → `valueSerializer` →
3187        // `partitioner` (`KafkaProducer#close`).
3188        self.producer_metrics.close();
3189        self.metrics.close();
3190        let first_error = self.close_serializers().await;
3191        if let Some(partitioner) = &self.partitioner {
3192            partitioner.close();
3193        }
3194
3195        // `KafkaProducer`: the first failure `closeQuietly`
3196        // recorded is raised once everything is closed.
3197        if let Some(error) = first_error {
3198            return Err(Error::kafka_message_source("Failed to close kafka producer", error));
3199        }
3200        kafka_debug!(self.log_context, "Kafka producer has been closed");
3201        Ok(())
3202    }
3203}
3204
3205impl<K, V> KafkaProducer<K, V> {
3206    /// Makes the record's headers read-only once both serializers have run, so
3207    /// nothing changes them after the producer reads them
3208    /// (`KafkaProducer#setReadOnly`; the headers are always `RecordHeaders`
3209    /// here, so Java's `instanceof` test always holds).
3210    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#setReadOnly")]
3211    fn set_read_only(headers: &mut RecordHeaders) {
3212        headers.set_read_only();
3213    }
3214
3215    /// `Utils.closeQuietly(keySerializerPlugin, "producer keySerializer", firstException)`
3216    /// then the same for the value serializer (`KafkaProducer`),
3217    /// through [`close_serializers_quietly`](Self::close_serializers_quietly).
3218    /// Only the first call closes them.
3219    async fn close_serializers(&self) -> Option<Error> {
3220        if self.serializers_closed.swap(true, Ordering::AcqRel) {
3221            return None;
3222        }
3223        Self::close_serializers_quietly(&*self.key_serializer, &*self.value_serializer, &self.log_context).await
3224    }
3225
3226    /// `Utils.closeQuietly(keySerializerPlugin, "producer keySerializer", firstException)`
3227    /// then `Utils.closeQuietly(valueSerializerPlugin, "producer valueSerializer", firstException)`
3228    /// (`KafkaProducer#close`): both are closed, each failure (an `Err` from
3229    /// `close`, or a caught panic) is logged as `closeQuietly` logs one
3230    /// (`Utils`), and the first one is returned. Shared by `close` and a failed
3231    /// constructor (`CloseSerializersOnError`).
3232    async fn close_serializers_quietly(
3233        key_serializer: &dyn DynSerializer<K>,
3234        value_serializer: &dyn DynSerializer<V>,
3235        log_context: &LogContext,
3236    ) -> Option<Error> {
3237        let mut first_error = None;
3238        let key = AssertUnwindSafe(key_serializer.close()).catch_unwind().await;
3239        Self::record_serializer_close_failure(&mut first_error, log_context, "producer keySerializer", key);
3240        let value = AssertUnwindSafe(value_serializer.close()).catch_unwind().await;
3241        Self::record_serializer_close_failure(&mut first_error, log_context, "producer valueSerializer", value);
3242        first_error
3243    }
3244
3245    /// Records a serializer's `close` failure, an `Err` or a caught panic, as
3246    /// `closeQuietly` records a throw: logged, and kept in `first_error` when it
3247    /// is the first (`Utils`).
3248    fn record_serializer_close_failure(
3249        first_error: &mut Option<Error>,
3250        log_context: &LogContext,
3251        name: &str,
3252        outcome: Result<Result<(), Error>, Box<dyn std::any::Any + Send>>,
3253    ) {
3254        let error = match outcome {
3255            Ok(Ok(())) => return,
3256            Ok(Err(error)) => error,
3257            Err(payload) => Error::kafka_message(panic_payload_message(&*payload).to_string()),
3258        };
3259        kafka_error!(log_context, "Failed to close {}: {}", name, error);
3260        first_error.get_or_insert(error);
3261    }
3262}
3263
3264/// Closes the producer's serializers when construction fails, as the `catch
3265/// (Throwable t)` of Java's constructor does (`KafkaProducer#KafkaProducer`):
3266/// it calls `close(Duration.ofMillis(0), true)`, whose `closeQuietly` of the
3267/// key and value serializers (`KafkaProducer#close`) closes them and only logs a
3268/// failure, since `swallowException` is `true`.
3269///
3270/// Rust-only (DoD #7): an RAII stand-in for that `catch`, the producer's twin
3271/// of the consumer's `CloseDeserializersOnError`. `new_inner` returns through
3272/// many `?`s; holding the serializers in this guard covers every one of them,
3273/// and any step added later. It is disarmed when the serializers are handed to
3274/// the producer. `new` is synchronous and a serializer's `close` is async, so
3275/// the close runs through [`Utils::close_quietly_from_sync`].
3276///
3277/// Java wraps the serializers after resolving the partitioner
3278/// (`KafkaProducer#KafkaProducer`), so a partitioner that cannot be
3279/// instantiated leaves a caller-supplied serializer unclosed there. Here the
3280/// producer owns the serializers from the start of `new_inner`, so the caller
3281/// can no longer close them: every failure closes them.
3282struct CloseSerializersOnError<K: 'static, V: 'static> {
3283    serializers: Option<KeyAndValueSerializers<K, V>>,
3284    log_context: LogContext,
3285}
3286
3287/// The key and the value serializer held by [`CloseSerializersOnError`].
3288type KeyAndValueSerializers<K, V> = (Box<dyn DynSerializer<K>>, Box<dyn DynSerializer<V>>);
3289
3290impl<K: 'static, V: 'static> CloseSerializersOnError<K, V> {
3291    fn new(
3292        key_serializer: Box<dyn DynSerializer<K>>,
3293        value_serializer: Box<dyn DynSerializer<V>>,
3294        log_context: LogContext,
3295    ) -> Self {
3296        Self { serializers: Some((key_serializer, value_serializer)), log_context }
3297    }
3298
3299    /// Construction succeeded: hands the serializers on, so nothing closes
3300    /// them here.
3301    fn disarm(mut self) -> KeyAndValueSerializers<K, V> {
3302        self.serializers.take().expect("the guard is disarmed once")
3303    }
3304}
3305
3306impl<K: 'static, V: 'static> Drop for CloseSerializersOnError<K, V> {
3307    fn drop(&mut self) {
3308        let Some((key_serializer, value_serializer)) = self.serializers.take() else {
3309            return;
3310        };
3311        let log_context = self.log_context.clone();
3312        Utils::close_quietly_from_sync(
3313            Box::pin(async move {
3314                // `swallowException` is `true`: the failures are only logged.
3315                let _ = KafkaProducer::<K, V>::close_serializers_quietly(
3316                    &*key_serializer,
3317                    &*value_serializer,
3318                    &log_context,
3319                )
3320                .await;
3321            }),
3322            "producer serializers",
3323        );
3324    }
3325}
3326
3327/// Starts a graceful close of a producer that was not closed, so its sender task
3328/// still sends the records it holds and then stops.
3329///
3330/// Java has no counterpart: its I/O thread is a daemon (built in the `KafkaProducer` constructor),
3331/// which keeps sending the records of a producer nothing references any more. The
3332/// sender task here does the same for as long as its runtime runs.
3333impl<K, V> Drop for KafkaProducer<K, V> {
3334    fn drop(&mut self) {
3335        if self.is_running() {
3336            kafka_warn!(
3337                self.log_context,
3338                "KafkaProducer was not closed before being dropped. Call close() to avoid resource leaks."
3339            );
3340            self.initiate_close();
3341        }
3342    }
3343}
3344
3345#[cfg(test)]
3346mod tests {
3347    use crate::common::utils::MockTime;
3348    use std::collections::HashMap;
3349
3350    use super::*;
3351    use crate::MockClient;
3352    use crate::common::Node;
3353    use crate::common::compress::Compression;
3354    use crate::common::internals::ClusterResourceListeners;
3355    use crate::common::protocol::Errors;
3356    use crate::common::requests::ConcreteResponse;
3357    use crate::common::serialization::Serializer;
3358    use crate::common::serialization::StringSerializer;
3359    use crate::common::utils::Utils;
3360    use crate::producer::MockPartitioner;
3361    use crate::producer::ProducerConfig;
3362    use crate::producer::ProducerRecordOptionsBuilder;
3363    use crate::producer::RoundRobinPartitioner;
3364    use crate::producer::internals::BufferPool;
3365    use crate::producer::internals::{PartitionerConfig, RecordAccumulator};
3366    use std::borrow::Cow;
3367
3368    const TOPIC: &str = "test-topic";
3369
3370    /// Java's `"some.id"`, the `transactional.id` most transactional
3371    /// `KafkaProducerTest` methods configure.
3372    const TRANSACTIONAL_ID: &str = "some.id";
3373
3374    /// Java's `initProducerIdResponse(1L, (short) 5, ..)` pair
3375    /// (`KafkaProducerTest#initProducerIdResponse`).
3376    const PRODUCER_ID: i64 = 1;
3377    const EPOCH: i16 = 5;
3378
3379    fn default_time() -> Arc<dyn Time> {
3380        Arc::new(SystemTime)
3381    }
3382
3383    fn create_metadata_with_topic(topic: &str, num_partitions: i32) -> Arc<ProducerMetadata> {
3384        use crate::MetadataResponseData;
3385        use crate::common::protocol::ApiKeys;
3386        use crate::common::protocol::Errors;
3387        use crate::common::requests::MetadataResponse;
3388        use crate::metadata_response_data::{MetadataResponseBroker, MetadataResponsePartition, MetadataResponseTopic};
3389
3390        let metadata = Arc::new(ProducerMetadata::new(
3391            100,
3392            1000,
3393            300_000,
3394            300_000,
3395            ClusterResourceListeners::new(),
3396        ));
3397
3398        let mut data = MetadataResponseData::new();
3399        data.set_controller_id(0);
3400        data.set_cluster_id(Some("test-cluster".to_string()));
3401
3402        let mut broker = MetadataResponseBroker::new();
3403        broker.set_node_id(0);
3404        broker.set_host("localhost".to_string());
3405        broker.set_port(9092);
3406        data.set_brokers(vec![broker]);
3407
3408        let mut topic_resp = MetadataResponseTopic::new();
3409        topic_resp.set_name(Some(topic.to_string()));
3410        topic_resp.set_error_code(Errors::None.code());
3411        topic_resp.set_is_internal(false);
3412
3413        let mut partitions = Vec::new();
3414        for i in 0..num_partitions {
3415            let mut p = MetadataResponsePartition::new();
3416            p.set_partition_index(i);
3417            p.set_leader_id(0);
3418            p.set_leader_epoch(0);
3419            p.set_replica_nodes(vec![0]);
3420            p.set_isr_nodes(vec![0]);
3421            p.set_error_code(Errors::None.code());
3422            partitions.push(p);
3423        }
3424        topic_resp.set_partitions(partitions);
3425        data.set_topics(vec![topic_resp]);
3426
3427        let response = MetadataResponse::with_version(data, ApiKeys::METADATA.latest_version());
3428
3429        metadata.add(topic, 0);
3430        metadata.update_with_current_request_version(&response, false, 0);
3431
3432        metadata
3433    }
3434
3435    fn create_accumulator() -> Arc<RecordAccumulator> {
3436        Arc::new(RecordAccumulator::new_for_test(
3437            16384,
3438            Compression::none().build(),
3439            5,
3440            100,
3441            1000,
3442            120_000,
3443            PartitionerConfig::new(true, 0, false, "").unwrap(),
3444            default_time(),
3445            Arc::new(BufferPool::new_for_test(32 * 1024 * 1024, 16384)),
3446            None,
3447        ))
3448    }
3449
3450    fn create_producer(
3451        metadata: Arc<ProducerMetadata>,
3452        accumulator: Arc<RecordAccumulator>,
3453    ) -> KafkaProducer<String, String> {
3454        create_producer_with_config(ProducerConfig::defaults(), metadata, accumulator)
3455    }
3456
3457    fn create_producer_with_config(
3458        config: ProducerConfig,
3459        metadata: Arc<ProducerMetadata>,
3460        accumulator: Arc<RecordAccumulator>,
3461    ) -> KafkaProducer<String, String> {
3462        let running = Arc::new(AtomicBool::new(true));
3463        let force_close = Arc::new(AtomicBool::new(false));
3464        let wakeup = Arc::new(Notify::new());
3465
3466        KafkaProducer::with_options(
3467            KafkaProducerOptionsBuilder::new()
3468                .set_config(&config)
3469                .set_key_serializer(Box::new(StringSerializer))
3470                .set_value_serializer(Box::new(StringSerializer))
3471                .set_metadata(metadata)
3472                .set_accumulator(accumulator)
3473                .set_running(running)
3474                .set_force_close(force_close)
3475                .set_wakeup(wakeup)
3476                .set_time(default_time())
3477                .set_pending_requests(Arc::new(Mutex::new(PendingRequests::new())))
3478                .build()
3479                .expect("KafkaProducerOptionsBuilder::build: every mandatory parameter is set above"),
3480        )
3481    }
3482
3483    /// A serializer that always fails with a caller-chosen error, so `do_send`'s
3484    /// dispatch on the serializer's error *class* is observable.
3485    struct FailingSerializer(Error);
3486
3487    impl Serializer<String> for FailingSerializer {
3488        async fn serialize<'a>(
3489            &'a self,
3490            _topic: &'a str,
3491            _data: Option<&'a String>,
3492        ) -> Result<Option<Cow<'a, [u8]>>, Error> {
3493            Err(self.0.clone())
3494        }
3495    }
3496
3497    fn create_producer_with_key_serializer(
3498        metadata: Arc<ProducerMetadata>,
3499        accumulator: Arc<RecordAccumulator>,
3500        key_serializer: Box<dyn DynSerializer<String>>,
3501    ) -> KafkaProducer<String, String> {
3502        KafkaProducer::with_options(
3503            KafkaProducerOptionsBuilder::new()
3504                .set_config(&ProducerConfig::defaults())
3505                .set_key_serializer(key_serializer)
3506                .set_value_serializer(Box::new(StringSerializer))
3507                .set_metadata(metadata)
3508                .set_accumulator(accumulator)
3509                .set_time(default_time())
3510                .set_pending_requests(Arc::new(Mutex::new(PendingRequests::new())))
3511                .build()
3512                .expect("KafkaProducerOptionsBuilder::build: every mandatory parameter is set above"),
3513        )
3514    }
3515
3516    /// Java's only catch around the serializer call is
3517    /// `catch (ClassCastException cce)` (`KafkaProducer#doSend`), which
3518    /// cannot happen in Rust — the serializer is statically typed on `K`. Everything
3519    /// else the serializer throws keeps its class and is dispatched by `doSend`'s
3520    /// outer catches: an `ApiException` at `KafkaProducer#doSend` fires the callback and returns a
3521    /// failed future.
3522    ///
3523    /// Rewriting every serializer error as `SerializationException` flipped
3524    /// `is_retriable_error()` from `true` to `false` for exactly this case (a
3525    /// schema-registry `TimeoutException`), skipped the callback entirely because
3526    /// `Serialization` is not an `ApiException`, and flattened the cause into a
3527    /// display string.
3528    #[tokio::test]
3529    async fn a_retriable_serializer_error_keeps_its_class_and_fires_the_callback() {
3530        let metadata = create_metadata_with_topic(TOPIC, 1);
3531        let accumulator = create_accumulator();
3532        let producer = create_producer_with_key_serializer(
3533            metadata,
3534            accumulator,
3535            Box::new(FailingSerializer(Error::timeout("schema registry lookup timed out"))),
3536        );
3537
3538        let callback_error: Arc<Mutex<Option<Error>>> = Arc::new(Mutex::new(None));
3539        let sink = Arc::clone(&callback_error);
3540        let callback: Box<dyn Callback> = Box::new(move |_metadata: Option<&RecordMetadata>, error: Option<&Error>| {
3541            *sink.lock().unwrap() = error.cloned();
3542        });
3543
3544        let record = ProducerRecord::with_key(TOPIC.to_string(), Some("k".to_string()), Some("v".to_string()));
3545        // An `ApiException` yields `Ok(failed_future)`, not `Err` — Java's
3546        // `return new FutureFailure(e)`.
3547        let future = producer
3548            .send_with_callback(record, Some(callback))
3549            .await
3550            .expect("an API-error serializer failure must come back as a failed future, not Err");
3551
3552        let error = future.get().await.expect_err("the future must be failed");
3553        assert!(error.is_timeout_error(), "the serializer's class must survive: {error:?}");
3554        assert!(error.is_retriable_error(), "a schema-registry timeout is retriable: {error:?}");
3555        assert_eq!(error.message(), "schema registry lookup timed out");
3556
3557        let invoked = callback_error.lock().unwrap().take().expect("the callback must be invoked");
3558        assert!(invoked.is_timeout_error(), "got {invoked:?}");
3559    }
3560
3561    /// The callback partition of a pre-append API error: Java's `catch (ApiException e)`
3562    /// (`KafkaProducer`) builds its null metadata from
3563    /// `AppendCallbacks.topicPartition()` (`KafkaProducer`), which, before a partition is
3564    /// computed, names the record's own partition when the record names one, and
3565    /// `UNKNOWN_PARTITION` only otherwise. Covers a serializer failure on both
3566    /// serialization paths (built-in and custom partitioner) and the metadata wait on both
3567    /// `send` paths (typed and borrowed bytes), each with and without a record partition.
3568    #[tokio::test]
3569    async fn a_pre_append_api_error_reports_the_record_partition() {
3570        use crate::common::serialization::ByteArraySerializer;
3571
3572        fn recording_callback(slot: &Arc<Mutex<Option<(String, i32)>>>) -> Box<dyn Callback> {
3573            let slot = Arc::clone(slot);
3574            Box::new(move |metadata: Option<&RecordMetadata>, error: Option<&Error>| {
3575                assert!(error.is_some(), "the callback is handed the error");
3576                let metadata = metadata.expect("Java passes a non-null RecordMetadata here");
3577                *slot.lock().unwrap() = Some((metadata.topic().to_string(), metadata.partition()));
3578            })
3579        }
3580
3581        // A serializer API error, on the built-in partitioner's zero-copy path.
3582        for (partition, expected) in [(Some(2), 2), (None, RecordMetadata::UNKNOWN_PARTITION)] {
3583            let producer = create_producer_with_key_serializer(
3584                create_metadata_with_topic(TOPIC, 3),
3585                create_accumulator(),
3586                Box::new(FailingSerializer(Error::timeout("schema registry lookup timed out"))),
3587            );
3588            let seen = Arc::new(Mutex::new(None));
3589            let record =
3590                ProducerRecord::with_partition_key(TOPIC.to_string(), partition, Some("k".to_string()), None).unwrap();
3591            let future = producer
3592                .send_with_callback(record, Some(recording_callback(&seen)))
3593                .await
3594                .expect("an API error comes back as a failed future");
3595            assert!(future.get().await.unwrap_err().is_timeout_error());
3596            assert_eq!(seen.lock().unwrap().take(), Some((TOPIC.to_string(), expected)));
3597        }
3598
3599        // The same on the custom partitioner's borrowing path.
3600        for (partition, expected) in [(Some(2), 2), (None, RecordMetadata::UNKNOWN_PARTITION)] {
3601            let producer: KafkaProducer<String, String> = KafkaProducer::with_options(
3602                KafkaProducerOptionsBuilder::new()
3603                    .set_config(&ProducerConfig::defaults())
3604                    .set_key_serializer(Box::new(FailingSerializer(Error::timeout("schema registry lookup timed out"))))
3605                    .set_value_serializer(Box::new(StringSerializer))
3606                    .set_metadata(create_metadata_with_topic(TOPIC, 3))
3607                    .set_accumulator(create_accumulator())
3608                    .set_time(default_time())
3609                    .set_pending_requests(Arc::new(Mutex::new(PendingRequests::new())))
3610                    .set_partitioner(Some(Box::new(RoundRobinPartitioner::new())))
3611                    .build()
3612                    .expect("KafkaProducerOptionsBuilder::build: every mandatory parameter is set above"),
3613            );
3614            let seen = Arc::new(Mutex::new(None));
3615            let record =
3616                ProducerRecord::with_partition_key(TOPIC.to_string(), partition, Some("k".to_string()), None).unwrap();
3617            let future = producer
3618                .send_with_callback(record, Some(recording_callback(&seen)))
3619                .await
3620                .expect("an API error comes back as a failed future");
3621            assert!(future.get().await.unwrap_err().is_timeout_error());
3622            assert_eq!(seen.lock().unwrap().take(), Some((TOPIC.to_string(), expected)));
3623        }
3624
3625        // A metadata-wait timeout: the record names a partition the topic does not have,
3626        // and `max.block.ms = 0` fails the wait at once.
3627        let config = || ProducerConfig { max_block_ms: 0, ..ProducerConfig::defaults() };
3628        let producer =
3629            create_producer_with_config(config(), create_metadata_with_topic(TOPIC, 1), create_accumulator());
3630        let seen = Arc::new(Mutex::new(None));
3631        let record =
3632            ProducerRecord::with_partition_key(TOPIC.to_string(), Some(5), None, Some("v".to_string())).unwrap();
3633        let future = producer
3634            .send_with_callback(record, Some(recording_callback(&seen)))
3635            .await
3636            .expect("a metadata timeout comes back as a failed future");
3637        assert!(future.get().await.unwrap_err().is_timeout_error());
3638        assert_eq!(seen.lock().unwrap().take(), Some((TOPIC.to_string(), 5)));
3639
3640        // The same on the borrowed byte-slice `send`.
3641        let producer: KafkaProducer<Vec<u8>, Vec<u8>> = KafkaProducer::with_options(
3642            KafkaProducerOptionsBuilder::new()
3643                .set_config(&config())
3644                .set_key_serializer(Box::new(ByteArraySerializer))
3645                .set_value_serializer(Box::new(ByteArraySerializer))
3646                .set_metadata(create_metadata_with_topic(TOPIC, 1))
3647                .set_accumulator(create_accumulator())
3648                .set_time(default_time())
3649                .set_pending_requests(Arc::new(Mutex::new(PendingRequests::new())))
3650                .build()
3651                .expect("KafkaProducerOptionsBuilder::build: every mandatory parameter is set above"),
3652        );
3653        let seen = Arc::new(Mutex::new(None));
3654        let record =
3655            ProducerRecord::with_partition_key(TOPIC.to_string(), Some(5), None, Some(b"v".as_slice())).unwrap();
3656        let future = producer
3657            .send(record, Some(recording_callback(&seen)))
3658            .await
3659            .expect("a metadata timeout comes back as a failed future");
3660        assert!(future.get().await.unwrap_err().is_timeout_error());
3661        assert_eq!(seen.lock().unwrap().take(), Some((TOPIC.to_string(), 5)));
3662    }
3663
3664    /// The other half of the same dispatch: a `SerializationException` — which
3665    /// extends `KafkaException` directly and is NOT an `ApiException` — reaches
3666    /// `catch (KafkaException e)` at `KafkaProducer` and is rethrown, so
3667    /// `send()` returns `Err` rather than a failed future.
3668    #[tokio::test]
3669    async fn a_serialization_error_from_the_serializer_is_returned_as_err() {
3670        let metadata = create_metadata_with_topic(TOPIC, 1);
3671        let accumulator = create_accumulator();
3672        let producer = create_producer_with_key_serializer(
3673            metadata,
3674            accumulator,
3675            Box::new(FailingSerializer(Error::serialization("not a valid string"))),
3676        );
3677
3678        let record = ProducerRecord::with_key(TOPIC.to_string(), Some("k".to_string()), Some("v".to_string()));
3679        let error = producer
3680            .send(record)
3681            .await
3682            .expect_err("a serialization error is not an API error, so send() returns Err");
3683        assert!(matches!(error, Error::Serialization(_)), "got {error:?}");
3684        assert!(error.is_kafka_error(), "a serialization error is a Kafka error: {error:?}");
3685        assert!(!error.is_api_error(), "but it is not an API error: {error:?}");
3686        assert_eq!(error.message(), "not a valid string");
3687    }
3688
3689    /// `waitOnMetadata`'s first throw attaches the topic's metadata error as the
3690    /// `TimeoutException`'s **cause** (`KafkaProducer#waitOnMetadata`,
3691    /// `new TimeoutException(errorMessage, metadata.getError(topic).exception())`).
3692    ///
3693    /// The message used to be built by interpolating the underlying error into it,
3694    /// which left `Error::source()` empty — a caller (or a log line) could no longer
3695    /// walk from the timeout to the broker error that produced it.
3696    #[tokio::test]
3697    async fn wait_on_metadata_carries_the_topic_error_as_the_timeout_cause() {
3698        use crate::MetadataResponseData;
3699        use crate::common::protocol::ApiKeys;
3700        use crate::common::protocol::Errors;
3701        use crate::common::requests::MetadataResponse;
3702        use crate::metadata_response_data::{MetadataResponseBroker, MetadataResponseTopic};
3703
3704        let metadata = create_metadata_with_topic(TOPIC, 1);
3705        let accumulator = create_accumulator();
3706        let producer = create_producer(Arc::clone(&metadata), accumulator);
3707
3708        // A topic the broker reports a retriable error for and no partitions, so the
3709        // wait times out with `get_error(topic)` populated.
3710        const MISSING: &str = "topic-with-an-error";
3711        let mut data = MetadataResponseData::new();
3712        data.set_controller_id(0);
3713        let mut broker = MetadataResponseBroker::new();
3714        broker.set_node_id(0);
3715        broker.set_host("localhost".to_string());
3716        broker.set_port(9092);
3717        data.set_brokers(vec![broker]);
3718        let mut topic_resp = MetadataResponseTopic::new();
3719        topic_resp.set_name(Some(MISSING.to_string()));
3720        topic_resp.set_error_code(Errors::LeaderNotAvailable.code());
3721        topic_resp.set_is_internal(false);
3722        topic_resp.set_partitions(Vec::new());
3723        data.set_topics(vec![topic_resp]);
3724        let response = MetadataResponse::with_version(data, ApiKeys::METADATA.latest_version());
3725        metadata.add(MISSING, 0);
3726        metadata.update_with_current_request_version(&response, false, 0);
3727        assert_eq!(
3728            metadata.get_error(MISSING),
3729            Some(Errors::LeaderNotAvailable),
3730            "the fixture must actually record a topic error"
3731        );
3732
3733        let now_ms = producer.now_ms();
3734        let error = producer
3735            .wait_on_metadata(MISSING, None, now_ms, 50)
3736            .await
3737            .expect_err("the topic has no metadata, so the wait must time out");
3738
3739        assert!(error.is_timeout_error(), "got {error:?}");
3740        assert!(
3741            error.message().contains("not present in metadata"),
3742            "the message keeps Java's text, without the cause interpolated: {}",
3743            error.message()
3744        );
3745        assert_eq!(
3746            error.source().expect("the metadata error must be the timeout's cause").error(),
3747            Errors::LeaderNotAvailable
3748        );
3749    }
3750
3751    /// Translated from `KafkaProducerTest.testMetricConfigRecordingLevel`.
3752    ///
3753    /// The Java test constructs a producer with the default config and asserts
3754    /// `producer.metrics.config().recordLevel() == INFO`, then with
3755    /// `metrics.recording.level=DEBUG` and asserts `DEBUG`.
3756    #[test]
3757    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testMetricConfigRecordingLevel")]
3758    fn test_metric_config_recording_level() {
3759        let default_producer = create_producer_with_config(
3760            ProducerConfig::defaults(),
3761            create_metadata_with_topic(TOPIC, 1),
3762            create_accumulator(),
3763        );
3764        assert_eq!(default_producer.metrics.config().record_level(), RecordingLevel::Info);
3765
3766        let debug_config =
3767            ProducerConfig { metrics_recording_level: "DEBUG".to_string(), ..ProducerConfig::defaults() };
3768        let debug_producer =
3769            create_producer_with_config(debug_config, create_metadata_with_topic(TOPIC, 1), create_accumulator());
3770        assert_eq!(debug_producer.metrics.config().record_level(), RecordingLevel::Debug);
3771    }
3772
3773    /// The producer registers the `producer-metrics` latency sensors and
3774    /// exposes them through `metrics()`; `flush()` records `flush-time-ns-total`.
3775    #[tokio::test]
3776    async fn test_flush_records_producer_metrics() {
3777        let producer = create_producer_with_config(
3778            ProducerConfig::defaults(),
3779            create_metadata_with_topic(TOPIC, 1),
3780            create_accumulator(),
3781        );
3782
3783        // The metrics snapshot exposes the producer-metrics latency sensors.
3784        let flush_name = producer.metrics.metric_name("flush-time-ns-total", "producer-metrics");
3785        let snapshot = producer.metrics();
3786        assert!(
3787            snapshot.contains_key(&flush_name),
3788            "metrics() should expose flush-time-ns-total"
3789        );
3790
3791        // flush() drives a CumulativeSum record; the metric stays present.
3792        producer.flush().await.expect("flush");
3793        assert!(producer.metrics().contains_key(&flush_name));
3794    }
3795
3796    /// Translated from `KafkaProducerTest.testSendToInvalidTopic`.
3797    ///
3798    /// Tests that sending to an invalid topic name returns a failed future with
3799    /// an InvalidTopic error. Matches Java behavior where InvalidTopicException
3800    /// (an ApiException) is caught and returned via a FutureFailure.
3801    #[tokio::test]
3802    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testSendToInvalidTopic")]
3803    async fn test_send_to_invalid_topic() {
3804        use crate::MetadataResponseData;
3805        use crate::common::protocol::ApiKeys;
3806        use crate::common::protocol::Errors;
3807        use crate::common::requests::MetadataResponse;
3808        use crate::metadata_response_data::{MetadataResponseBroker, MetadataResponseTopic};
3809
3810        let metadata = Arc::new(ProducerMetadata::new(
3811            100,
3812            1000,
3813            300_000,
3814            300_000,
3815            ClusterResourceListeners::new(),
3816        ));
3817
3818        // Create a metadata response with an invalid topic
3819        let mut data = MetadataResponseData::new();
3820        data.set_controller_id(0);
3821
3822        let mut broker = MetadataResponseBroker::new();
3823        broker.set_node_id(0);
3824        broker.set_host("localhost".to_string());
3825        broker.set_port(9092);
3826        data.set_brokers(vec![broker]);
3827
3828        let mut topic_resp = MetadataResponseTopic::new();
3829        topic_resp.set_name(Some("".to_string()));
3830        topic_resp.set_error_code(Errors::InvalidTopicError.code());
3831        topic_resp.set_is_internal(false);
3832        data.set_topics(vec![topic_resp]);
3833
3834        let response = MetadataResponse::with_version(data, ApiKeys::METADATA.latest_version());
3835        metadata.add("", 0);
3836        metadata.update_with_current_request_version(&response, false, 0);
3837
3838        let accumulator = create_accumulator();
3839        let producer = create_producer(metadata, accumulator);
3840
3841        let record = ProducerRecord::new("".to_string(), Some("test".to_string()));
3842        let result = producer.send(record).await;
3843        // Java returns a FutureFailure for ApiExceptions like InvalidTopicException
3844        assert!(result.is_ok(), "send() should return Ok with a failed future for InvalidTopic");
3845        let future = result.unwrap();
3846        assert!(future.is_done(), "Failed future should be immediately done");
3847        let err = future.get().await.unwrap_err();
3848        assert!(
3849            matches!(err, Error::InvalidTopic(_)),
3850            "Expected InvalidTopic error, got: {:?}",
3851            err
3852        );
3853        // `waitOnMetadata` throws `new InvalidTopicException(topic)` — the
3854        // `(String message)` constructor, so `getMessage()` is the topic name
3855        // (empty here) and `invalidTopics()` is empty, NOT `{topic}`.
3856        assert_eq!(err.message(), "");
3857        match &err {
3858            Error::InvalidTopic(e) => assert!(
3859                e.invalid_topics().is_empty(),
3860                "the (String) constructor leaves invalidTopics empty"
3861            ),
3862            _ => unreachable!(),
3863        }
3864    }
3865
3866    /// Translated from `KafkaProducerTest.closeShouldBeIdempotent`.
3867    ///
3868    /// Tests that calling close multiple times is safe.
3869    #[tokio::test]
3870    async fn test_close_should_be_idempotent() {
3871        let metadata = create_metadata_with_topic(TOPIC, 1);
3872        let accumulator = create_accumulator();
3873        let producer = create_producer(metadata, accumulator);
3874
3875        producer.close().await.unwrap();
3876        producer.close().await.unwrap();
3877    }
3878
3879    /// Translated from `KafkaProducerTest.closeWithNegativeTimestampShouldThrow`.
3880    ///
3881    /// Tests that close with zero timeout works correctly.
3882    /// In Java this tests negative Duration; in Rust `Duration` is unsigned
3883    /// so we test `Duration::ZERO` instead.
3884    #[tokio::test]
3885    async fn test_close_with_zero_timeout() {
3886        let metadata = create_metadata_with_topic(TOPIC, 1);
3887        let accumulator = create_accumulator();
3888        let producer = create_producer(metadata, accumulator);
3889
3890        let result = producer.close_with_timeout(Duration::ZERO).await;
3891        assert!(result.is_ok());
3892    }
3893
3894    /// Translated from `KafkaProducerTest.testPartitionsForWithNullTopic`.
3895    ///
3896    /// Tests that partitions_for returns the correct number of partitions.
3897    #[tokio::test]
3898    async fn test_partitions_for_returns_partitions() {
3899        let metadata = create_metadata_with_topic(TOPIC, 3);
3900        let accumulator = create_accumulator();
3901        let producer = create_producer(metadata, accumulator);
3902
3903        let partitions = producer.partitions_for(TOPIC).await.unwrap();
3904        assert_eq!(3, partitions.len());
3905    }
3906
3907    /// Tests that sending after close returns an error.
3908    ///
3909    /// Translated from `KafkaProducerTest.testTransactionalMethodThrowsWhenSenderClosed`
3910    /// (non-transactional part).
3911    #[tokio::test]
3912    async fn test_send_after_close_returns_error() {
3913        let metadata = create_metadata_with_topic(TOPIC, 1);
3914        let accumulator = create_accumulator();
3915        let producer = create_producer(metadata, accumulator);
3916
3917        producer.close().await.unwrap();
3918
3919        let record = ProducerRecord::new(TOPIC.to_string(), Some("test".to_string()));
3920        let result = producer.send(record).await;
3921        assert!(result.is_err());
3922        match result.unwrap_err() {
3923            Error::LocalIllegalState(msg) => {
3924                assert!(msg.message().contains("after producer has been closed"));
3925            },
3926            other => panic!("Expected IllegalState error, got: {:?}", other),
3927        }
3928    }
3929
3930    /// Tests that a record too large for max_request_size returns a failed future.
3931    ///
3932    /// Translated from `KafkaProducerTest.testInterceptorPartitionSetOnTooLargeRecord`
3933    /// (the record-too-large validation part).
3934    ///
3935    /// Matches Java behavior: ApiExceptions like RecordTooLargeException are returned
3936    /// via a completed-with-error future, not propagated as Err from send().
3937    #[tokio::test]
3938    async fn test_ensure_valid_record_size_rejects_too_large() {
3939        let config = ProducerConfig { max_request_size: 10, ..ProducerConfig::defaults() };
3940        let metadata = create_metadata_with_topic(TOPIC, 1);
3941        let accumulator = create_accumulator();
3942
3943        let producer = create_producer_with_config(config, metadata, accumulator);
3944
3945        // Create a record that's larger than 10 bytes
3946        let large_value = "a".repeat(100);
3947        let record = ProducerRecord::new(TOPIC.to_string(), Some(large_value));
3948        let result = producer.send(record).await;
3949        // Java returns a FutureFailure, not an exception from send()
3950        assert!(
3951            result.is_ok(),
3952            "send() should return Ok with a failed future for RecordTooLarge"
3953        );
3954        let future = result.unwrap();
3955        assert!(future.is_done(), "Failed future should be immediately done");
3956        let err = future.get().await.unwrap_err();
3957        match err {
3958            Error::RecordTooLarge(msg) => {
3959                assert!(
3960                    msg.message().contains(ProducerConfig::MAX_REQUEST_SIZE_CONFIG),
3961                    "Error message should mention the config key: {}",
3962                    msg
3963                );
3964            },
3965            other => panic!("Expected RecordTooLarge error, got: {:?}", other),
3966        }
3967    }
3968
3969    /// Tests that records larger than total buffer memory return a failed future.
3970    ///
3971    /// Matches Java behavior: ApiExceptions like RecordTooLargeException are returned
3972    /// via a completed-with-error future, not propagated as Err from send().
3973    #[tokio::test]
3974    async fn test_ensure_valid_record_size_rejects_larger_than_buffer_memory() {
3975        let config = ProducerConfig { buffer_memory: 10, ..ProducerConfig::defaults() };
3976        let metadata = create_metadata_with_topic(TOPIC, 1);
3977        let accumulator = create_accumulator();
3978
3979        let producer = create_producer_with_config(config, metadata, accumulator);
3980
3981        let large_value = "a".repeat(100);
3982        let record = ProducerRecord::new(TOPIC.to_string(), Some(large_value));
3983        let result = producer.send(record).await;
3984        assert!(
3985            result.is_ok(),
3986            "send() should return Ok with a failed future for RecordTooLarge"
3987        );
3988        let future = result.unwrap();
3989        assert!(future.is_done(), "Failed future should be immediately done");
3990        let err = future.get().await.unwrap_err();
3991        match err {
3992            Error::RecordTooLarge(msg) => {
3993                assert!(
3994                    msg.message().contains(ProducerConfig::BUFFER_MEMORY_CONFIG),
3995                    "Error message should mention the config key: {}",
3996                    msg
3997                );
3998            },
3999            other => panic!("Expected RecordTooLarge error, got: {:?}", other),
4000        }
4001    }
4002
4003    /// A size above `i32::MAX` is rejected with the `max.request.size` message.
4004    #[tokio::test]
4005    async fn test_ensure_valid_record_size_rejects_size_beyond_i32_max() {
4006        let metadata = create_metadata_with_topic(TOPIC, 1);
4007        let producer = create_producer_with_config(ProducerConfig::defaults(), metadata, create_accumulator());
4008
4009        let size = (1i64 << 32) + 98;
4010        let err = producer.ensure_valid_record_size(size).expect_err("too large");
4011        assert!(matches!(err, Error::RecordTooLarge(_)), "Expected RecordTooLarge, got: {err:?}");
4012        assert_eq!(
4013            err.message(),
4014            "The message is 4294967394 bytes when serialized which is larger than 1048576, which is the value of \
4015             the max.request.size configuration."
4016        );
4017    }
4018
4019    /// With `max.request.size` at `i32::MAX`, a size of exactly `i32::MAX` is valid and
4020    /// `i32::MAX + 1` is not: the size does not wrap to a negative `i32` and pass.
4021    #[tokio::test]
4022    async fn test_ensure_valid_record_size_at_i32_max() {
4023        let config = ProducerConfig {
4024            max_request_size: i32::MAX,
4025            buffer_memory: i64::MAX,
4026            ..ProducerConfig::defaults()
4027        };
4028        let producer = create_producer_with_config(config, create_metadata_with_topic(TOPIC, 1), create_accumulator());
4029
4030        producer
4031            .ensure_valid_record_size(i32::MAX as i64)
4032            .expect("exactly max.request.size is valid");
4033        let err = producer
4034            .ensure_valid_record_size(i32::MAX as i64 + 1)
4035            .expect_err("one byte too large");
4036        assert!(matches!(err, Error::RecordTooLarge(_)), "Expected RecordTooLarge, got: {err:?}");
4037        assert_eq!(
4038            err.message(),
4039            "The message is 2147483648 bytes when serialized which is larger than 2147483647, which is the value of \
4040             the max.request.size configuration."
4041        );
4042    }
4043
4044    /// A size the estimate saturated at `i64::MAX` is rejected and reported as is.
4045    #[tokio::test]
4046    async fn test_ensure_valid_record_size_rejects_saturated_size() {
4047        let producer = create_producer_with_config(
4048            ProducerConfig::defaults(),
4049            create_metadata_with_topic(TOPIC, 1),
4050            create_accumulator(),
4051        );
4052
4053        let err = producer.ensure_valid_record_size(i64::MAX).expect_err("too large");
4054        assert!(matches!(err, Error::RecordTooLarge(_)), "Expected RecordTooLarge, got: {err:?}");
4055        assert_eq!(
4056            err.message(),
4057            "The message is 9223372036854775807 bytes when serialized which is larger than 1048576, which is the \
4058             value of the max.request.size configuration."
4059        );
4060    }
4061
4062    /// `doSend`'s `catch (ApiException e)` block invokes the user `Callback` exactly
4063    /// once with a null-metadata `RecordMetadata(tp, -1, -1, NO_TIMESTAMP, -1, -1)`
4064    /// and the error, then returns a failed future
4065    /// (`KafkaProducer`).
4066    ///
4067    /// The arm covering a failure of `accumulator.append` itself used to drop the
4068    /// callback: it is *moved* into `append`, and `append` returned it only on the
4069    /// success path. An application that registers callbacks and never awaits the
4070    /// future was therefore never told the record had been dropped.
4071    ///
4072    /// `buffer.memory` sized for exactly one batch plus `max.block.ms = 0` makes the
4073    /// second `append` fail deterministically inside `BufferPool::allocate` with
4074    /// `BufferExhaustedException`, which is an `ApiException`
4075    /// (`BufferExhaustedException extends TimeoutException`) — so this is the
4076    /// `catch (ApiException e)` arm.
4077    #[tokio::test]
4078    async fn test_api_error_from_append_fires_the_callback_exactly_once() {
4079        let producer = create_producer_with_room_for_one_batch(0);
4080
4081        // The first record consumes the pool's only batch.
4082        producer
4083            .send_with_callback(
4084                ProducerRecord::with_partition_key(TOPIC.to_string(), Some(0), None, Some("first".to_string()))
4085                    .unwrap(),
4086                None,
4087            )
4088            .await
4089            .expect("the first send has memory");
4090
4091        // The second record targets partition 1, so its append has to allocate; it
4092        // finds the pool empty and `max.block.ms = 0`, and fails with
4093        // `BufferExhaustedException`.
4094        let invocations: Arc<Mutex<Vec<(i64, i32, String)>>> = Arc::new(Mutex::new(Vec::new()));
4095        let recorder = Arc::clone(&invocations);
4096        let callback: Box<dyn Callback> = Box::new(move |metadata: Option<&RecordMetadata>, error: Option<&Error>| {
4097            let metadata = metadata.expect("Java passes a non-null RecordMetadata here");
4098            let error = error.expect("Java passes the exception here");
4099            recorder
4100                .lock()
4101                .unwrap()
4102                .push((metadata.offset(), metadata.partition(), error.message().to_string()));
4103        });
4104
4105        let future = producer
4106            .send_with_callback(
4107                ProducerRecord::with_partition_key(TOPIC.to_string(), Some(1), None, Some("second".to_string()))
4108                    .unwrap(),
4109                Some(callback),
4110            )
4111            .await
4112            .expect("an ApiException becomes a failed future, not an Err");
4113
4114        // The callback fired exactly once. `on_completion` consumes its `Box<dyn Callback>`, so
4115        // "at most once" is a type-level guarantee; this pins "at least once".
4116        // The guard is cloned out and dropped before the `await` below (CLAUDE.md §11.6).
4117        let invocations = invocations.lock().unwrap().clone();
4118        assert_eq!(invocations.len(), 1, "the callback must fire exactly once, got {invocations:?}");
4119        let (offset, partition, message) = &invocations[0];
4120        // Java's `nullMetadata`: `new RecordMetadata(tp, -1, -1, NO_TIMESTAMP, -1, -1)`.
4121        assert_eq!(*offset, -1, "the null metadata carries offset -1");
4122        assert_eq!(*partition, 1, "the null metadata names the record's topic-partition");
4123        assert!(
4124            message.contains("Failed to allocate"),
4125            "the callback receives the BufferExhausted error: {message}"
4126        );
4127
4128        // And the future is failed with the same error, as Java's `FutureFailure` is.
4129        let error = future.get().await.expect_err("the future must be failed");
4130        assert!(
4131            matches!(error, Error::ProducerBufferExhausted(_)),
4132            "expected ProducerBufferExhausted, got {error:?}"
4133        );
4134        assert!(
4135            error.is_api_error(),
4136            "BufferExhaustedException extends TimeoutException, an ApiException"
4137        );
4138    }
4139
4140    /// The sibling of the test above on the *rethrowing* path: Java's
4141    /// `catch (KafkaException e)` (`KafkaProducer`) does NOT invoke the
4142    /// callback — it rethrows out of `send()`. `RecordAccumulator.tryAppend`'s
4143    /// `KafkaException("Producer closed while send in progress")` is a bare
4144    /// `KafkaException`, so it lands there.
4145    #[tokio::test]
4146    async fn test_non_api_error_from_append_does_not_fire_the_callback() {
4147        let metadata = create_metadata_with_topic(TOPIC, 1);
4148        let accumulator = create_accumulator();
4149        accumulator.close();
4150        let producer = create_producer_with_config(ProducerConfig::defaults(), metadata, accumulator);
4151
4152        let invoked = Arc::new(AtomicBool::new(false));
4153        let flag = Arc::clone(&invoked);
4154        let callback: Box<dyn Callback> =
4155            Box::new(move |_: Option<&RecordMetadata>, _: Option<&Error>| flag.store(true, Ordering::SeqCst));
4156
4157        let error = producer
4158            .send_with_callback(
4159                ProducerRecord::with_partition_key(TOPIC.to_string(), Some(0), None, Some("v".to_string())).unwrap(),
4160                Some(callback),
4161            )
4162            .await
4163            .expect_err("a bare KafkaException is rethrown out of send()");
4164        assert_eq!(error.message(), "Producer closed while send in progress");
4165        assert!(!error.is_api_error(), "a bare KafkaException is not an ApiException");
4166        assert!(
4167            !invoked.load(Ordering::SeqCst),
4168            "Java's catch (KafkaException e) block does not invoke the callback"
4169        );
4170    }
4171
4172    /// The fixture of `test_api_error_from_append_fires_the_callback_exactly_once`:
4173    /// `buffer.memory` sized for exactly one batch, and two partitions, so the
4174    /// second record needs a *new* batch rather than appending to the first
4175    /// record's still-roomy one — its append has to allocate, and finds the pool
4176    /// empty.
4177    fn create_producer_with_room_for_one_batch(max_block_ms: i64) -> KafkaProducer<String, String> {
4178        const BATCH_SIZE: usize = 16384;
4179        let config = ProducerConfig {
4180            batch_size: BATCH_SIZE as i32,
4181            buffer_memory: BATCH_SIZE as i64,
4182            max_block_ms,
4183            linger_ms: 0,
4184            ..ProducerConfig::defaults()
4185        };
4186        let metadata = create_metadata_with_topic(TOPIC, 2);
4187        let accumulator = Arc::new(RecordAccumulator::new(
4188            BATCH_SIZE as i32,
4189            Compression::none().build(),
4190            0,
4191            100,
4192            1000,
4193            120_000,
4194            PartitionerConfig::new(true, 0, false, "").unwrap(),
4195            Arc::new(Metrics::new()),
4196            KafkaProducer::<String, String>::PRODUCER_METRIC_GROUP_NAME,
4197            Arc::new(crate::common::utils::SystemTime),
4198            // Room for exactly one batch.
4199            Arc::new(BufferPool::new_for_test(BATCH_SIZE as i64, BATCH_SIZE)),
4200            None,
4201        ));
4202        create_producer_with_config(config, metadata, accumulator)
4203    }
4204
4205    fn string_record(partition: Option<i32>, value: &str) -> ProducerRecord<String, String> {
4206        ProducerRecord::with_partition_key(TOPIC.to_string(), partition, None, Some(value.to_string())).unwrap()
4207    }
4208
4209    /// A callback that records every invocation, plus the recorder to read back.
4210    #[expect(clippy::type_complexity)]
4211    fn recording_callback() -> (Box<dyn Callback>, Arc<Mutex<Vec<(i32, String)>>>) {
4212        let invocations: Arc<Mutex<Vec<(i32, String)>>> = Arc::new(Mutex::new(Vec::new()));
4213        let recorder = Arc::clone(&invocations);
4214        let callback: Box<dyn Callback> = Box::new(crate::producer::callback::new(move |metadata, error| {
4215            let metadata = metadata.expect("Java passes a non-null RecordMetadata here");
4216            let error = error.expect("Java passes the exception here");
4217            recorder
4218                .lock()
4219                .unwrap()
4220                .push((metadata.partition(), error.message().to_string()));
4221        }));
4222        (callback, invocations)
4223    }
4224
4225    /// `try_send` on a producer that has metadata and memory is `send`: the record
4226    /// is appended and its future returned, and the callback waits for the
4227    /// broker's answer rather than firing now.
4228    #[tokio::test]
4229    async fn try_send_accepts_a_record_the_producer_can_take_at_once() {
4230        let producer = create_producer_with_room_for_one_batch(0);
4231        let (callback, invocations) = recording_callback();
4232
4233        let outcome = producer
4234            .try_send(string_record(Some(0), "first"), Some(callback))
4235            .await
4236            .expect("a ready producer accepts the record");
4237
4238        let future = match outcome {
4239            TrySendOutcome::Accepted(future) => future,
4240            other => panic!("expected Accepted, got {other:?}"),
4241        };
4242        assert!(!future.is_done(), "the record is in a batch awaiting the sender, not resolved");
4243        assert!(invocations.lock().unwrap().is_empty(), "nothing has completed, so no callback");
4244    }
4245
4246    /// The buffer-memory would-block point. `send` with `max.block.ms = 0` fails the
4247    /// same record with `BufferExhaustedException` — callback fired, failed future
4248    /// (`test_api_error_from_append_fires_the_callback_exactly_once`). `try_send`
4249    /// instead does nothing to it: no callback, no future, the pool's memory
4250    /// untouched, and the callback handed back intact for the retry.
4251    #[tokio::test]
4252    async fn try_send_reports_would_block_on_buffer_exhaustion_without_firing_the_callback() {
4253        let producer = create_producer_with_room_for_one_batch(0);
4254
4255        // The first record consumes the pool's only batch.
4256        producer
4257            .try_send(string_record(Some(0), "first"), None)
4258            .await
4259            .expect("the first send has memory");
4260        let memory_before = producer.accumulator.buffer_pool_available_memory();
4261
4262        // The second record targets partition 1, so its append has to allocate.
4263        let (callback, invocations) = recording_callback();
4264        let outcome = producer
4265            .try_send(string_record(Some(1), "second"), Some(callback))
4266            .await
4267            .expect("would-block is an outcome, not an error");
4268
4269        let returned = match outcome {
4270            TrySendOutcome::WouldBlock(callback) => callback,
4271            other => panic!("expected WouldBlock, got {other:?}"),
4272        };
4273        assert!(
4274            invocations.lock().unwrap().is_empty(),
4275            "a would-block must not invoke the callback"
4276        );
4277        assert_eq!(
4278            producer.accumulator.buffer_pool_available_memory(),
4279            memory_before,
4280            "a would-block must leave the pool as it found it"
4281        );
4282
4283        // The very callback that was passed in comes back, still live: firing it
4284        // ourselves proves the producer neither invoked nor dropped it.
4285        let returned = returned.expect("the caller's callback is handed back");
4286        let tp = TopicPartition::new(TOPIC.to_string(), 1);
4287        let null_metadata = RecordMetadata::new(tp, -1, -1, RecordBatch::NO_TIMESTAMP, -1, -1);
4288        returned.on_completion(Some(&null_metadata), Some(&Error::timeout("retry")));
4289        assert_eq!(invocations.lock().unwrap().as_slice(), &[(1, "retry".to_string())]);
4290    }
4291
4292    /// The same fixture through the blocking `send`: `SendWait::MaxBlock` dispatches
4293    /// `BufferExhaustedException` as the `ApiException` it is — callback fired once
4294    /// with the null metadata, failed future — so `send` keeps Java's contract
4295    /// while sharing its body with `try_send`.
4296    #[tokio::test]
4297    async fn send_still_fails_the_record_where_try_send_would_block() {
4298        let producer = create_producer_with_room_for_one_batch(0);
4299        producer
4300            .send_with_callback(string_record(Some(0), "first"), None)
4301            .await
4302            .expect("the first send has memory");
4303
4304        let (callback, invocations) = recording_callback();
4305        let future = producer
4306            .send_with_callback(string_record(Some(1), "second"), Some(callback))
4307            .await
4308            .expect("an ApiException becomes a failed future, not an Err");
4309
4310        let invocations = invocations.lock().unwrap().clone();
4311        assert_eq!(invocations.len(), 1, "the callback must fire exactly once, got {invocations:?}");
4312        assert_eq!(invocations[0].0, 1, "the null metadata names the record's partition");
4313        assert!(invocations[0].1.contains("Failed to allocate"), "got: {}", invocations[0].1);
4314        let error = future.get().await.expect_err("the future must be failed");
4315        assert!(matches!(error, Error::ProducerBufferExhausted(_)), "got {error:?}");
4316    }
4317
4318    /// The metadata would-block point. An unknown topic makes `send` wait for a
4319    /// metadata response; `try_send` reports `WouldBlock` with the callback
4320    /// un-invoked — and has still asked for the metadata, so the attempt is not
4321    /// wasted: the topic is now tracked and an update is requested.
4322    #[tokio::test]
4323    async fn try_send_reports_would_block_for_an_unknown_topic_and_requests_its_metadata() {
4324        const UNKNOWN_TOPIC: &str = "not-in-metadata-yet";
4325        let metadata = create_metadata_with_topic(TOPIC, 1);
4326        let producer =
4327            create_producer_with_config(ProducerConfig::defaults(), Arc::clone(&metadata), create_accumulator());
4328        assert!(!metadata.contains_topic(UNKNOWN_TOPIC));
4329
4330        let (callback, invocations) = recording_callback();
4331        let record =
4332            ProducerRecord::with_partition_key(UNKNOWN_TOPIC.to_string(), None, None, Some("v".to_string())).unwrap();
4333        let outcome = producer
4334            .try_send(record, Some(callback))
4335            .await
4336            .expect("would-block is an outcome");
4337
4338        assert!(matches!(outcome, TrySendOutcome::WouldBlock(Some(_))), "got {outcome:?}");
4339        assert!(
4340            invocations.lock().unwrap().is_empty(),
4341            "a would-block must not invoke the callback"
4342        );
4343        assert!(
4344            metadata.contains_topic(UNKNOWN_TOPIC),
4345            "the metadata request is the side effect worth keeping"
4346        );
4347        assert!(
4348            metadata.metadata_arc().update_requested(),
4349            "an update was requested for the topic"
4350        );
4351    }
4352
4353    /// Only a *wait* is a would-block. A record the producer rejects outright — too
4354    /// large for `max.request.size` — is resolved exactly as `send` resolves it:
4355    /// `Accepted`, with the callback already fired and the future already failed.
4356    #[tokio::test]
4357    async fn try_send_resolves_a_rejected_record_as_accepted_with_a_failed_future() {
4358        let config = ProducerConfig { max_request_size: 64, ..ProducerConfig::defaults() };
4359        let producer = create_producer_with_config(config, create_metadata_with_topic(TOPIC, 1), create_accumulator());
4360
4361        let (callback, invocations) = recording_callback();
4362        let outcome = producer
4363            .try_send(string_record(Some(0), &"x".repeat(1024)), Some(callback))
4364            .await
4365            .expect("an ApiException becomes a failed future, not an Err");
4366
4367        let future = match outcome {
4368            TrySendOutcome::Accepted(future) => future,
4369            other => panic!("expected Accepted, got {other:?}"),
4370        };
4371        let invocations = invocations.lock().unwrap().clone();
4372        assert_eq!(invocations.len(), 1, "the callback fires exactly once, got {invocations:?}");
4373        assert!(invocations[0].1.contains("larger than 64"), "got: {}", invocations[0].1);
4374        let error = future.get().await.expect_err("the future must be failed");
4375        assert!(matches!(error, Error::RecordTooLarge(_)), "got {error:?}");
4376    }
4377
4378    /// `throwIfProducerClosed()` comes before either wait point, so a closed
4379    /// producer is an `Err` from `try_send` just as from `send` — never a
4380    /// `WouldBlock` that would have the caller queue the record for a producer
4381    /// that will not take it.
4382    #[tokio::test]
4383    async fn try_send_on_a_closed_producer_is_an_error_not_a_would_block() {
4384        let producer = create_producer_with_config(
4385            ProducerConfig::defaults(),
4386            create_metadata_with_topic(TOPIC, 1),
4387            create_accumulator(),
4388        );
4389        producer.close().await.unwrap();
4390
4391        let (callback, invocations) = recording_callback();
4392        let error = producer
4393            .try_send(string_record(Some(0), "v"), Some(callback))
4394            .await
4395            .expect_err("a closed producer rejects the send");
4396
4397        assert!(matches!(error, Error::LocalIllegalState(_)), "got {error:?}");
4398        assert_eq!(error.message(), "Cannot perform operation after producer has been closed");
4399        assert!(
4400            invocations.lock().unwrap().is_empty(),
4401            "Java's rethrow path never invokes the callback"
4402        );
4403    }
4404
4405    /// Tests that the partition() method returns the explicit partition when set.
4406    #[test]
4407    fn test_partition_returns_explicit_partition() {
4408        let metadata = create_metadata_with_topic(TOPIC, 3);
4409        let accumulator = create_accumulator();
4410        let producer = create_producer(metadata.clone(), accumulator);
4411
4412        let record = ProducerRecord::with_partition_key(
4413            TOPIC.to_string(),
4414            Some(2),
4415            Some("key".to_string()),
4416            Some("value".to_string()),
4417        )
4418        .unwrap();
4419        let cluster = metadata.fetch();
4420        let partition = producer.partition(&record, Some(b"key"), Some(b"value"), &cluster);
4421        assert_eq!(2, partition);
4422    }
4423
4424    /// Tests that the partition() method uses key hashing when no partition is set.
4425    #[test]
4426    fn test_partition_uses_key_hash() {
4427        let metadata = create_metadata_with_topic(TOPIC, 3);
4428        let accumulator = create_accumulator();
4429        let producer = create_producer(metadata.clone(), accumulator);
4430
4431        let record = ProducerRecord::with_key(TOPIC.to_string(), Some("key".to_string()), Some("value".to_string()));
4432        let cluster = metadata.fetch();
4433        let partition = producer.partition(&record, Some(b"key"), Some(b"value"), &cluster);
4434        // Should be deterministic based on key hash
4435        assert!((0..3).contains(&partition));
4436
4437        // Same key should give the same partition
4438        let partition2 = producer.partition(&record, Some(b"key"), Some(b"value"), &cluster);
4439        assert_eq!(partition, partition2);
4440    }
4441
4442    /// Tests that the partition() method returns UNKNOWN_PARTITION when no key and no partition.
4443    #[test]
4444    fn test_partition_returns_unknown_when_no_key() {
4445        let metadata = create_metadata_with_topic(TOPIC, 3);
4446        let accumulator = create_accumulator();
4447        let producer = create_producer(metadata.clone(), accumulator);
4448
4449        let record: ProducerRecord<String, String> = ProducerRecord::new(TOPIC.to_string(), Some("value".to_string()));
4450        let cluster = metadata.fetch();
4451        let partition = producer.partition(&record, None, Some(b"value"), &cluster);
4452        assert_eq!(RecordMetadata::UNKNOWN_PARTITION, partition);
4453    }
4454
4455    /// Tests that the partition() method ignores keys when partitioner_ignore_keys is true.
4456    #[test]
4457    fn test_partition_ignores_keys_when_configured() {
4458        let config = ProducerConfig { partitioner_ignore_keys: true, ..ProducerConfig::defaults() };
4459        let metadata = create_metadata_with_topic(TOPIC, 3);
4460        let accumulator = create_accumulator();
4461
4462        let producer = create_producer_with_config(config, metadata.clone(), accumulator);
4463
4464        let record = ProducerRecord::with_key(TOPIC.to_string(), Some("key".to_string()), Some("value".to_string()));
4465        let cluster = metadata.fetch();
4466        let partition = producer.partition(&record, Some(b"key"), Some(b"value"), &cluster);
4467        assert_eq!(RecordMetadata::UNKNOWN_PARTITION, partition);
4468    }
4469
4470    /// Builds a `KafkaProducer` whose `partitioner.type` selects
4471    /// `Murmur2RandomPartitioner` (the `KeyHasher::Murmur2` hash).
4472    fn create_murmur2_producer(
4473        metadata: Arc<ProducerMetadata>,
4474        accumulator: Arc<RecordAccumulator>,
4475    ) -> KafkaProducer<String, String> {
4476        let mut props = HashMap::from([("bootstrap.servers".to_string(), "localhost:9092".to_string())]);
4477        props.insert("partitioner.type".to_string(), "Murmur2RandomPartitioner".to_string());
4478        let config = ProducerConfig::new(&props).expect("murmur2 partitioner config is valid");
4479        create_producer_with_config(config, metadata, accumulator)
4480    }
4481
4482    /// The default (unset `partitioner.type`) keyed partition path uses the
4483    /// IEEE CRC-32 (librdkafka `consistent_random`) hash taken UNSIGNED modulo
4484    /// the partition count — NOT murmur2 / `Utils.toPositive`. This is the
4485    /// deliberate, user-approved deviation from Java parity.
4486    #[test]
4487    fn test_partition_default_hasher_is_crc32() {
4488        let metadata = create_metadata_with_topic(TOPIC, 7);
4489        let accumulator = create_accumulator();
4490        let producer = create_producer(metadata.clone(), accumulator);
4491        assert_eq!(KeyHasher::Crc32, producer.key_hasher);
4492
4493        let record = ProducerRecord::with_key(TOPIC.to_string(), Some("key".to_string()), Some("value".to_string()));
4494        let cluster = metadata.fetch();
4495        let partition = producer.partition(&record, Some(b"key"), Some(b"value"), &cluster);
4496        assert_eq!((crc32fast::hash(b"key") % 7) as i32, partition);
4497        // Both send call sites route through the same helper, so it agrees.
4498        assert_eq!(partition, producer.partition_for_key_or_unknown(Some(b"key"), TOPIC, &cluster));
4499    }
4500
4501    /// Under the CRC-32 default, an EMPTY (but non-null) key defers to the
4502    /// keyless sticky (KIP-794) path — `UNKNOWN_PARTITION` — matching
4503    /// librdkafka `consistent_random` (which treats `keylen == 0` as no key).
4504    /// Asserted through both send call sites' shared helper.
4505    #[test]
4506    fn test_partition_empty_key_crc32_defers_to_sticky() {
4507        let metadata = create_metadata_with_topic(TOPIC, 3);
4508        let accumulator = create_accumulator();
4509        let producer = create_producer(metadata.clone(), accumulator);
4510
4511        let record = ProducerRecord::with_key(TOPIC.to_string(), Some(String::new()), Some("value".to_string()));
4512        let cluster = metadata.fetch();
4513        // via KafkaProducer.partition()
4514        assert_eq!(
4515            RecordMetadata::UNKNOWN_PARTITION,
4516            producer.partition(&record, Some(b""), Some(b"value"), &cluster)
4517        );
4518        // via the helper do_send_bytes uses
4519        assert_eq!(
4520            RecordMetadata::UNKNOWN_PARTITION,
4521            producer.partition_for_key_or_unknown(Some(b""), TOPIC, &cluster)
4522        );
4523    }
4524
4525    /// `Murmur2RandomPartitioner` reproduces the exact Java-client formula
4526    /// `Utils.toPositive(Utils.murmur2(key)) % numPartitions` for a non-empty
4527    /// key.
4528    #[test]
4529    fn test_partition_murmur2_matches_java_formula() {
4530        let metadata = create_metadata_with_topic(TOPIC, 7);
4531        let accumulator = create_accumulator();
4532        let producer = create_murmur2_producer(metadata.clone(), accumulator);
4533        assert_eq!(KeyHasher::Murmur2, producer.key_hasher);
4534
4535        let record = ProducerRecord::with_key(TOPIC.to_string(), Some("key".to_string()), Some("value".to_string()));
4536        let cluster = metadata.fetch();
4537        let partition = producer.partition(&record, Some(b"key"), Some(b"value"), &cluster);
4538        assert_eq!(Utils::to_positive(Utils::murmur2(b"key")) % 7, partition);
4539        assert_eq!(partition, producer.partition_for_key_or_unknown(Some(b"key"), TOPIC, &cluster));
4540    }
4541
4542    /// murmur2 hashes an EMPTY (non-null) key rather than deferring to the
4543    /// sticky path — exact Java-client parity, and the point of contrast with
4544    /// the CRC-32 default (`test_partition_empty_key_crc32_defers_to_sticky`).
4545    #[test]
4546    fn test_partition_empty_key_murmur2_is_hashed() {
4547        let metadata = create_metadata_with_topic(TOPIC, 3);
4548        let accumulator = create_accumulator();
4549        let producer = create_murmur2_producer(metadata.clone(), accumulator);
4550
4551        let cluster = metadata.fetch();
4552        let partition = producer.partition_for_key_or_unknown(Some(b""), TOPIC, &cluster);
4553        assert_eq!(Utils::to_positive(Utils::murmur2(b"")) % 3, partition);
4554        assert_ne!(RecordMetadata::UNKNOWN_PARTITION, partition);
4555    }
4556
4557    /// An explicit record partition wins over key hashing regardless of the
4558    /// selected hasher (here: murmur2). Complements
4559    /// `test_partition_returns_explicit_partition` (which covers the default).
4560    #[test]
4561    fn test_partition_explicit_partition_wins_under_murmur2() {
4562        let metadata = create_metadata_with_topic(TOPIC, 3);
4563        let accumulator = create_accumulator();
4564        let producer = create_murmur2_producer(metadata.clone(), accumulator);
4565
4566        let record = ProducerRecord::with_partition_key(
4567            TOPIC.to_string(),
4568            Some(2),
4569            Some("key".to_string()),
4570            Some("value".to_string()),
4571        )
4572        .unwrap();
4573        let cluster = metadata.fetch();
4574        assert_eq!(2, producer.partition(&record, Some(b"key"), Some(b"value"), &cluster));
4575    }
4576
4577    /// `partitioner.ignore.keys=true` forces `UNKNOWN_PARTITION` even when a
4578    /// non-empty key would otherwise hash, under murmur2 as well as the default.
4579    #[test]
4580    fn test_partition_ignore_keys_overrides_murmur2() {
4581        let mut props = HashMap::from([("bootstrap.servers".to_string(), "localhost:9092".to_string())]);
4582        props.insert("partitioner.type".to_string(), "Murmur2RandomPartitioner".to_string());
4583        props.insert("partitioner.ignore.keys".to_string(), "true".to_string());
4584        let config = ProducerConfig::new(&props).unwrap();
4585        let metadata = create_metadata_with_topic(TOPIC, 3);
4586        let accumulator = create_accumulator();
4587        let producer = create_producer_with_config(config, metadata.clone(), accumulator);
4588
4589        let cluster = metadata.fetch();
4590        assert_eq!(
4591            RecordMetadata::UNKNOWN_PARTITION,
4592            producer.partition_for_key_or_unknown(Some(b"key"), TOPIC, &cluster)
4593        );
4594    }
4595
4596    // =====================================================================
4597    // PHASE-2 TEST ACCOUNTING — the pluggable-`Partitioner` `KafkaProducerTest` methods
4598    //
4599    // SCOPE CRITERION. A `KafkaProducerTest.java` method is in scope for Phase 2 iff its
4600    // body sets `PARTITIONER_CLASS_CONFIG` or names one of the partitioner nested classes
4601    // (`MockPartitioner` / `PartitionerForClientId` / `BuggyPartitioner` /
4602    // `MonitorablePartitioner`) as its subject. That marker set is exactly five methods:
4603    //
4604    //   - testPartitionerClose (:668) .............. TRANSLATED -> test_partitioner_close
4605    //   - negativePartitionShouldThrow (:2410) ..... TRANSLATED -> negative_partition_should_throw
4606    //   - configurableObjectsShouldSeeGeneratedClientId (:2296)
4607    //         ...... TRANSLATED (partitioner portion) -> configurable_objects_should_see_generated_client_id
4608    //   - shouldCloseProperlyAndThrowIfInterrupted (:689) ....... N/A (see below)
4609    //   - testMonitorablePlugins (:2828) ........................ N/A (see below)
4610    //
4611    // N/A justifications:
4612    //
4613    //   - shouldCloseProperlyAndThrowIfInterrupted asserts that `close()` blocks until an
4614    //     in-flight `send` completes and that INTERRUPTING the closing thread surfaces an
4615    //     `InterruptException`. Thread interruption (`Thread.interrupt()` /
4616    //     `InterruptException`) has no Rust/Tokio analogue — the same gap the producer-
4617    //     transaction rules record for `TransactionalRequestResult`. `MockPartitioner` is
4618    //     incidental here (merely the configured `partitioner.type`), not the subject, so
4619    //     this belongs to the close/interrupt surface, not Phase 2.
4620    //   - testMonitorablePlugins exercises the `Plugin` / `Monitorable` metrics SPI, which
4621    //     the Phase 2 spec places explicitly out of scope. `MonitorablePartitioner` is only
4622    //     the vehicle; there is nothing partitioner-specific to translate.
4623    //
4624    // RUST-ONLY BEHAVIORAL TESTS. Java gets its partitioner round-trip coverage "for free"
4625    // from a real/mock broker plus reflective `partitioner.class` loading. Rust has no
4626    // reflection (a user partitioner is passed as an instance) and these unit tests do not
4627    // stand up a broker, so five behavioral contracts that Java never asserts directly are
4628    // pinned explicitly here:
4629    //
4630    //   - test_round_robin_partitioner_resolution_and_gating — `new` resolves a
4631    //         `partitioner.type=RoundRobinPartitioner` and a resolved partitioner
4632    //         disables adaptive partitioning (`KafkaProducer#KafkaProducer`).
4633    //   - test_round_robin_partitioner_used_once_per_record — a stateful partitioner is
4634    //         consulted EXACTLY once per record on the send path (do_send computes it,
4635    //         do_send_bytes short-circuits on the passed `Some(..)`).
4636    //   - test_partitioner_receives_key_value_and_serialized_bytes — the typed key/value
4637    //         and the serialized bytes are forwarded unchanged.
4638    //   - test_explicit_partition_bypasses_partitioner — an explicit record partition
4639    //         short-circuits `compute_partition` before the partitioner is consulted.
4640    //   - test_set_partitioner_replaces_partitioner_type_property — the partitioner
4641    //         set with `ProducerConfig::set_partitioner` replaces a built-in
4642    //         `partitioner.type` from the properties.
4643    // =====================================================================
4644
4645    // --- test-local partitioners (translated from KafkaProducerTest's nested classes) ---
4646
4647    /// Translated from `KafkaProducerTest.BuggyPartitioner` (`KafkaProducerTest.BuggyPartitioner#close`): always
4648    /// returns an invalid negative partition, exercising `compute_partition`'s
4649    /// non-negative guard. Java's empty `close`/`configure` overrides are the trait's
4650    /// default no-ops, so they are intentionally not overridden.
4651    struct BuggyPartitioner;
4652
4653    impl<K, V> Partitioner<K, V> for BuggyPartitioner {
4654        fn partition(
4655            &self,
4656            _topic: &str,
4657            _key: Option<&K>,
4658            _key_bytes: Option<&[u8]>,
4659            _value: Option<&V>,
4660            _value_bytes: Option<&[u8]>,
4661            _cluster: &Cluster,
4662        ) -> i32 {
4663            -1
4664        }
4665    }
4666
4667    /// Translated from `KafkaProducerTest.PartitionerForClientId`:
4668    /// `partition` returns 0, and `configure` records the resolved `client.id` so a
4669    /// test can assert the partitioner saw the generated id. Java accumulates into a
4670    /// static `CLIENT_IDS` set; here the sink is an injected `Arc<Mutex<Vec<String>>>`
4671    /// so tests running in parallel do not share it.
4672    struct PartitionerForClientId {
4673        client_ids: Arc<Mutex<Vec<String>>>,
4674    }
4675
4676    impl<K, V> Partitioner<K, V> for PartitionerForClientId {
4677        fn configure(&mut self, configs: &HashMap<String, String>) {
4678            if let Some(id) = configs.get(ProducerConfig::CLIENT_ID_CONFIG) {
4679                self.client_ids.lock().unwrap().push(id.clone());
4680            }
4681        }
4682
4683        fn partition(
4684            &self,
4685            _topic: &str,
4686            _key: Option<&K>,
4687            _key_bytes: Option<&[u8]>,
4688            _value: Option<&V>,
4689            _value_bytes: Option<&[u8]>,
4690            _cluster: &Cluster,
4691        ) -> i32 {
4692            0
4693        }
4694    }
4695
4696    /// One recorded `Partitioner::partition` invocation: the borrowed arguments
4697    /// (captured as owned copies) and the partition the delegate returned.
4698    #[derive(Debug, Clone)]
4699    struct RecordedCall {
4700        topic: String,
4701        key: Option<String>,
4702        key_bytes: Option<Vec<u8>>,
4703        value: Option<String>,
4704        value_bytes: Option<Vec<u8>>,
4705        returned: i32,
4706    }
4707
4708    /// A `Partitioner` decorator with no Java counterpart: it delegates to an inner
4709    /// partitioner and records every `partition` call. It exists to probe two
4710    /// contracts Java gets implicitly from a real broker — that a stateful partitioner
4711    /// is consulted EXACTLY once per record, and that the producer forwards the typed
4712    /// key/value and the serialized bytes unchanged. A justified `definition-of-done`
4713    /// §7 addition (a test-only decorator, present only under `#[cfg(test)]`).
4714    struct RecordingPartitioner {
4715        inner: Box<dyn Partitioner<String, String>>,
4716        calls: Arc<Mutex<Vec<RecordedCall>>>,
4717    }
4718
4719    impl Partitioner<String, String> for RecordingPartitioner {
4720        fn partition(
4721            &self,
4722            topic: &str,
4723            key: Option<&String>,
4724            key_bytes: Option<&[u8]>,
4725            value: Option<&String>,
4726            value_bytes: Option<&[u8]>,
4727            cluster: &Cluster,
4728        ) -> i32 {
4729            let returned = self.inner.partition(topic, key, key_bytes, value, value_bytes, cluster);
4730            self.calls.lock().unwrap().push(RecordedCall {
4731                topic: topic.to_string(),
4732                key: key.cloned(),
4733                key_bytes: key_bytes.map(|b| b.to_vec()),
4734                value: value.cloned(),
4735                value_bytes: value_bytes.map(|b| b.to_vec()),
4736                returned,
4737            });
4738            returned
4739        }
4740    }
4741
4742    /// Seam constructor for a producer that owns a custom `partitioner` (the
4743    /// `KafkaProducer::new` last argument). Metadata is pre-seeded (see
4744    /// `create_metadata_with_topic`) and `sender_handle` is `None`, so `send()` and
4745    /// `close()` neither block on metadata nor hang on a sender task — the send-path
4746    /// partitioner tests need a producer that can actually append a record. NOTE the
4747    /// `new()` seam stores the partitioner as-is and does NOT `configure` it or gate
4748    /// adaptive partitioning; the `new`-based tests below cover those.
4749    fn create_producer_with_partitioner(
4750        config: ProducerConfig,
4751        metadata: Arc<ProducerMetadata>,
4752        accumulator: Arc<RecordAccumulator>,
4753        partitioner: Box<dyn Partitioner<String, String>>,
4754    ) -> KafkaProducer<String, String> {
4755        let running = Arc::new(AtomicBool::new(true));
4756        let force_close = Arc::new(AtomicBool::new(false));
4757        let wakeup = Arc::new(Notify::new());
4758
4759        KafkaProducer::with_options(
4760            KafkaProducerOptionsBuilder::new()
4761                .set_config(&config)
4762                .set_key_serializer(Box::new(StringSerializer))
4763                .set_value_serializer(Box::new(StringSerializer))
4764                .set_metadata(metadata)
4765                .set_accumulator(accumulator)
4766                .set_running(running)
4767                .set_force_close(force_close)
4768                .set_wakeup(wakeup)
4769                .set_time(default_time())
4770                .set_pending_requests(Arc::new(Mutex::new(PendingRequests::new())))
4771                .set_partitioner(Some(partitioner))
4772                .build()
4773                .expect("KafkaProducerOptionsBuilder::build: every mandatory parameter is set above"),
4774        )
4775    }
4776
4777    /// Translated from `KafkaProducerTest.testPartitionerClose` (`KafkaProducer`).
4778    ///
4779    /// Java resolves the partitioner reflectively through `partitioner.class`, which
4780    /// invokes the `MockPartitioner` constructor (bumping `INIT_COUNT`) and then
4781    /// `configure`. `MockPartitioner` is a test-only type with no built-in
4782    /// `partitioner.type` name, so here it is supplied as an explicit instance;
4783    /// constructing it via `MockPartitioner::new()` bumps `INIT_COUNT` exactly as the
4784    /// Java ctor does. `close()` then bumps `CLOSE_COUNT` (`KafkaProducer#close`).
4785    ///
4786    /// The counters are process-global statics, so the whole body must hold
4787    /// `lock_counters()` and reset at start AND end (Java's `finally`) to serialize
4788    /// against the other counter-touching test (`mock_partitioner`'s `test_counts_init_and_close`).
4789    /// That serializer is a `std::sync::Mutex` guard (matching the codebase's
4790    /// preference for CPU-bound state, and shared with the D6 test), and it must stay
4791    /// held across the async `close()` — otherwise a parallel counter test could
4792    /// interleave between the INIT and CLOSE assertions. Holding a `std` guard across
4793    /// an `.await` is exactly what `clippy::await_holding_lock` forbids, so this is a
4794    /// plain `#[test]` that drives `close()` on a dedicated current-thread runtime via
4795    /// `block_on`: the guard spans a synchronous `block_on` call rather than an
4796    /// `.await` point (and the close path never locks `COUNTER_GUARD`, so parking the
4797    /// thread inside `block_on` while holding it cannot deadlock).
4798    #[test]
4799    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testPartitionerClose")]
4800    fn test_partitioner_close() {
4801        let _guard = MockPartitioner::lock_counters();
4802        MockPartitioner::reset_counters();
4803
4804        let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap();
4805        runtime.block_on(async {
4806            let producer = create_producer_with_partitioner(
4807                ProducerConfig::defaults(),
4808                create_metadata_with_topic(TOPIC, 1),
4809                create_accumulator(),
4810                Box::new(MockPartitioner::new()),
4811            );
4812            assert_eq!(1, MockPartitioner::init_count().load(Ordering::SeqCst));
4813            assert_eq!(0, MockPartitioner::close_count().load(Ordering::SeqCst));
4814
4815            producer.close().await.expect("seam producer closes cleanly");
4816
4817            assert_eq!(1, MockPartitioner::init_count().load(Ordering::SeqCst));
4818            assert_eq!(1, MockPartitioner::close_count().load(Ordering::SeqCst));
4819        });
4820
4821        // Cleanup since we use mutable process-global statics in MockPartitioner.
4822        MockPartitioner::reset_counters();
4823    }
4824
4825    /// Java's `MockSerializer` (`clients/src/test/java/org/apache/kafka/test/MockSerializer.java`):
4826    /// counts its constructions and closes. Java keeps the counters in statics;
4827    /// here each test shares its own, so parallel tests cannot disturb them.
4828    use std::sync::atomic::AtomicUsize;
4829
4830    struct MockSerializer {
4831        close_count: Arc<AtomicUsize>,
4832        panic_on_close: Option<&'static str>,
4833        fail_on_close: Option<&'static str>,
4834    }
4835
4836    impl MockSerializer {
4837        fn new(init_count: &AtomicUsize, close_count: &Arc<AtomicUsize>) -> Self {
4838            init_count.fetch_add(1, Ordering::SeqCst);
4839            Self { close_count: Arc::clone(close_count), panic_on_close: None, fail_on_close: None }
4840        }
4841    }
4842
4843    impl Serializer<String> for MockSerializer {
4844        async fn serialize<'a>(
4845            &'a self,
4846            _topic: &'a str,
4847            data: Option<&'a String>,
4848        ) -> Result<Option<Cow<'a, [u8]>>, Error> {
4849            Ok(data.map(|data| Cow::Borrowed(data.as_bytes())))
4850        }
4851
4852        async fn close(&self) -> Result<(), Error> {
4853            self.close_count.fetch_add(1, Ordering::SeqCst);
4854            if let Some(message) = self.panic_on_close {
4855                panic!("{message}");
4856            }
4857            match self.fail_on_close {
4858                Some(message) => Err(Error::serialization(message)),
4859                None => Ok(()),
4860            }
4861        }
4862    }
4863
4864    fn create_producer_with_serializers(
4865        key_serializer: Box<dyn DynSerializer<String>>,
4866        value_serializer: Box<dyn DynSerializer<String>>,
4867    ) -> KafkaProducer<String, String> {
4868        KafkaProducer::with_options(
4869            KafkaProducerOptionsBuilder::new()
4870                .set_config(&ProducerConfig::defaults())
4871                .set_key_serializer(key_serializer)
4872                .set_value_serializer(value_serializer)
4873                .set_metadata(create_metadata_with_topic(TOPIC, 1))
4874                .set_accumulator(create_accumulator())
4875                .set_time(default_time())
4876                .set_pending_requests(Arc::new(Mutex::new(PendingRequests::new())))
4877                .build()
4878                .expect("KafkaProducerOptionsBuilder::build: every mandatory parameter is set above"),
4879        )
4880    }
4881
4882    /// Translated from `KafkaProducerTest.testSerializerClose`:
4883    /// `close` closes both serializers (`KafkaProducer#close`). The
4884    /// second `close` (Java's try-with-resources closes once) shows the
4885    /// serializers are closed only once.
4886    #[tokio::test]
4887    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testSerializerClose")]
4888    async fn test_serializer_close() {
4889        let init_count = AtomicUsize::new(0);
4890        let close_count = Arc::new(AtomicUsize::new(0));
4891        let producer = create_producer_with_serializers(
4892            Box::new(MockSerializer::new(&init_count, &close_count)),
4893            Box::new(MockSerializer::new(&init_count, &close_count)),
4894        );
4895        assert_eq!(2, init_count.load(Ordering::SeqCst));
4896        assert_eq!(0, close_count.load(Ordering::SeqCst));
4897
4898        producer.close().await.expect("close");
4899
4900        assert_eq!(2, init_count.load(Ordering::SeqCst));
4901        assert_eq!(2, close_count.load(Ordering::SeqCst));
4902
4903        producer.close().await.expect("a second close");
4904        assert_eq!(2, close_count.load(Ordering::SeqCst));
4905    }
4906
4907    /// `Utils.closeQuietly` keeps closing after a failure and `close` raises
4908    /// the first one as the cause of `KafkaException("Failed to close kafka
4909    /// producer", ..)` (`KafkaProducer#close`): the value serializer
4910    /// is closed although the key serializer's `close` panicked.
4911    #[tokio::test]
4912    async fn test_serializer_close_failure_closes_both_and_reports_the_first() {
4913        let init_count = AtomicUsize::new(0);
4914        let close_count = Arc::new(AtomicUsize::new(0));
4915        let mut key = MockSerializer::new(&init_count, &close_count);
4916        key.panic_on_close = Some("key close failed");
4917        let mut value = MockSerializer::new(&init_count, &close_count);
4918        value.panic_on_close = Some("value close failed");
4919        let producer = create_producer_with_serializers(Box::new(key), Box::new(value));
4920
4921        let error = producer.close().await.expect_err("both closes panicked");
4922        assert!(error.is_kafka_error());
4923        assert_eq!("Failed to close kafka producer", error.message());
4924        let source = std::error::Error::source(&error).expect("the first failure is the source");
4925        assert_eq!("key close failed", source.to_string());
4926        assert_eq!(2, close_count.load(Ordering::SeqCst));
4927
4928        producer.close().await.expect("the serializers are not closed again");
4929        assert_eq!(2, close_count.load(Ordering::SeqCst));
4930    }
4931
4932    /// A serializer `close` returning an error takes the same path as a
4933    /// throwing one in Java: it is recorded by `closeQuietly`, the other
4934    /// serializer is still closed, and `close` returns it as the cause of
4935    /// `Failed to close kafka producer` (`KafkaProducer#close`).
4936    #[tokio::test]
4937    async fn test_serializer_close_error_closes_both_and_reports_the_first() {
4938        let init_count = AtomicUsize::new(0);
4939        let close_count = Arc::new(AtomicUsize::new(0));
4940        let mut key = MockSerializer::new(&init_count, &close_count);
4941        key.fail_on_close = Some("key close failed");
4942        let mut value = MockSerializer::new(&init_count, &close_count);
4943        value.panic_on_close = Some("value close failed");
4944        let producer = create_producer_with_serializers(Box::new(key), Box::new(value));
4945
4946        let error = producer.close().await.expect_err("both closes failed");
4947        assert!(error.is_kafka_error());
4948        assert_eq!("Failed to close kafka producer", error.message());
4949        let source = error.source().expect("the first failure is the source");
4950        assert!(source.is_serialization_error(), "the serializer's own error: {source:?}");
4951        assert_eq!("key close failed", source.message());
4952        assert_eq!(2, close_count.load(Ordering::SeqCst));
4953    }
4954
4955    /// KAFKA-2121: a failed constructor closes the serializers, as Java's
4956    /// `catch (Throwable t)` calls `close(Duration.ofMillis(0), true)`
4957    /// (`KafkaProducer#KafkaProducer`), whose `closeQuietly` closes both
4958    /// serializers (`KafkaProducer#close`). Here the first fallible step fails
4959    /// (`configureDeliveryTimeout`'s `ConfigException`), without a Tokio
4960    /// runtime: a serializer whose `close` does not wait is closed inline.
4961    #[test]
4962    fn test_serializers_closed_when_construction_fails() {
4963        let init_count = AtomicUsize::new(0);
4964        let close_count = Arc::new(AtomicUsize::new(0));
4965        let mut props = HashMap::from([("bootstrap.servers".to_string(), "localhost:9999".to_string())]);
4966        props.insert("delivery.timeout.ms".to_string(), "1".to_string());
4967        let config = ProducerConfig::new(&props).expect("valid config");
4968
4969        let error = KafkaProducer::<String, String>::new(
4970            config,
4971            MockSerializer::new(&init_count, &close_count),
4972            MockSerializer::new(&init_count, &close_count),
4973        )
4974        .expect_err("delivery.timeout.ms below linger.ms + request.timeout.ms fails construction");
4975
4976        assert_eq!("Failed to construct kafka producer", error.message());
4977        assert_eq!(2, init_count.load(Ordering::SeqCst));
4978        assert_eq!(2, close_count.load(Ordering::SeqCst), "both serializers are closed once");
4979    }
4980
4981    /// A serializer `close` that panics on a failed constructor is logged, not
4982    /// propagated (`swallowException` is `true`, `KafkaProducer#close`), and
4983    /// the other serializer is still closed; the constructor's error is
4984    /// unchanged.
4985    #[test]
4986    fn test_serializer_close_failure_on_failed_construction_is_swallowed() {
4987        let init_count = AtomicUsize::new(0);
4988        let close_count = Arc::new(AtomicUsize::new(0));
4989        let mut key = MockSerializer::new(&init_count, &close_count);
4990        key.panic_on_close = Some("key close failed");
4991        let mut props = HashMap::from([("bootstrap.servers".to_string(), "localhost:9999".to_string())]);
4992        props.insert("delivery.timeout.ms".to_string(), "1".to_string());
4993        let config = ProducerConfig::new(&props).expect("valid config");
4994
4995        let error = KafkaProducer::<String, String>::new(config, key, MockSerializer::new(&init_count, &close_count))
4996            .expect_err("the construction fails");
4997
4998        assert_eq!("Failed to construct kafka producer", error.message());
4999        assert_eq!(2, close_count.load(Ordering::SeqCst), "the value serializer is closed too");
5000    }
5001
5002    /// Translated from `KafkaProducerTest.negativePartitionShouldThrow`.
5003    ///
5004    /// A partitioner returning a negative partition makes `send()` fail with
5005    /// `IllegalArgument` (Java: `IllegalArgumentException`). The exact message is
5006    /// asserted (DoD §3): error-message content is part of the behavioral contract.
5007    #[tokio::test]
5008    async fn negative_partition_should_throw() {
5009        let producer = create_producer_with_partitioner(
5010            ProducerConfig::defaults(),
5011            create_metadata_with_topic("topic", 1),
5012            create_accumulator(),
5013            Box::new(BuggyPartitioner),
5014        );
5015
5016        let record = ProducerRecord::with_key("topic".to_string(), Some("key".to_string()), Some("value".to_string()));
5017        let err = producer.send(record).await.expect_err("negative partition must be rejected");
5018        match &err {
5019            Error::LocalIllegalArgument(msg) => {
5020                assert_eq!(
5021                    msg.message(),
5022                    "The partitioner generated an invalid partition number: -1. Partition number should always be non-negative."
5023                );
5024            },
5025            other => panic!("Expected LocalIllegalArgument error, got: {:?}", other),
5026        }
5027    }
5028
5029    /// Translated from `KafkaProducerTest.configurableObjectsShouldSeeGeneratedClientId`
5030    /// (`KafkaProducerTest`), partitioner portion.
5031    ///
5032    /// A `configure`-able partitioner must see the generated `client.id` when the user
5033    /// sets none. Java asserts the same for the two serializers and the interceptor
5034    /// (`CLIENT_IDS.size() == 4`). Java configures only a serializer it builds from
5035    /// `key.serializer` / `value.serializer`, never one passed as an instance
5036    /// (`KafkaProducer#KafkaProducer`), and a Rust serializer is always passed as an
5037    /// instance; no `ProducerInterceptor` exists yet. So only the partitioner is
5038    /// checked here — hence exactly ONE recorded id rather than four. The partitioner is set
5039    /// with `ProducerConfig::set_partitioner`, as Java sets `partitioner.class`,
5040    /// so `new` builds and `configure`s it (the `with_options` seam does not
5041    /// configure).
5042    #[tokio::test]
5043    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#configurableObjectsShouldSeeGeneratedClientId")]
5044    async fn configurable_objects_should_see_generated_client_id() {
5045        let client_ids = Arc::new(Mutex::new(Vec::new()));
5046        let props = guard_props(&[]); // bootstrap only; NO client.id
5047        let config = ProducerConfig::new(&props)
5048            .expect("valid config")
5049            .set_partitioner::<String, String>(Box::new(PartitionerForClientId {
5050                client_ids: Arc::clone(&client_ids),
5051            }));
5052        let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
5053            .expect("constructs with a custom partitioner");
5054
5055        assert!(!producer.client_id().is_empty(), "a client.id is generated when unset");
5056        let recorded = client_ids.lock().unwrap();
5057        assert_eq!(1, recorded.len(), "the partitioner's configure ran exactly once");
5058        assert_eq!(
5059            recorded[0].as_str(),
5060            producer.client_id(),
5061            "the partitioner saw the generated client.id"
5062        );
5063    }
5064
5065    /// `new` gives the producer's metadata the `retry.backoff.ms` / `retry.backoff.max.ms`
5066    /// refresh backoff, as Java's constructor does, not the `reconnect.backoff.*` values
5067    /// that only the `NetworkClient` takes.
5068    ///
5069    /// Not a translation of a Java test. After one failed refresh the backoff is
5070    /// `retry.backoff.ms`, and after repeated failures it reaches `retry.backoff.max.ms`,
5071    /// each within `ExponentialBackoff`'s jitter and never above the maximum. The maximum
5072    /// differs from its default, and neither range overlaps the `reconnect.backoff.*`
5073    /// values, so each bound shows which config is used. The producer is closed first, so
5074    /// the Sender task cannot record a refresh of its own while the metadata is read.
5075    #[tokio::test]
5076    async fn test_metadata_refresh_backoff_follows_retry_backoff() {
5077        const RETRY_BACKOFF_MS: i64 = 100;
5078        const RETRY_BACKOFF_MAX_MS: i64 = 2000;
5079        let jitter = crate::CommonClientConfigs::RETRY_BACKOFF_JITTER;
5080        let props = guard_props(&[
5081            ("retry.backoff.ms", &RETRY_BACKOFF_MS.to_string()),
5082            ("retry.backoff.max.ms", &RETRY_BACKOFF_MAX_MS.to_string()),
5083            ("reconnect.backoff.ms", "50"),
5084            ("reconnect.backoff.max.ms", "50"),
5085        ]);
5086        let config = ProducerConfig::new(&props).expect("valid config");
5087        let producer =
5088            KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer).expect("constructs");
5089        producer.close_with_timeout(Duration::ZERO).await.expect("close");
5090
5091        let now = SystemTime.milliseconds();
5092        producer.metadata.failed_update(now);
5093        let initial = producer.metadata.time_to_allow_update(now);
5094        let initial_range =
5095            (RETRY_BACKOFF_MS as f64 * (1.0 - jitter)) as i64..=(RETRY_BACKOFF_MS as f64 * (1.0 + jitter)) as i64;
5096        assert!(
5097            initial_range.contains(&initial),
5098            "the first refresh backoff must be retry.backoff.ms with jitter, got {initial}"
5099        );
5100
5101        // `RETRY_BACKOFF_EXP_BASE` doubles the backoff per failure, so ten failures take it
5102        // well past the maximum.
5103        for _ in 0..10 {
5104            producer.metadata.failed_update(now);
5105        }
5106        let capped = producer.metadata.time_to_allow_update(now);
5107        let capped_range = (RETRY_BACKOFF_MAX_MS as f64 * (1.0 - jitter)) as i64..=RETRY_BACKOFF_MAX_MS;
5108        assert!(
5109            capped_range.contains(&capped),
5110            "repeated failures must back off up to retry.backoff.max.ms, got {capped}"
5111        );
5112    }
5113
5114    /// `partitioner.type=RoundRobinPartitioner` is resolved by `new` into a live
5115    /// `partitioner`, and a
5116    /// resolved partitioner turns OFF adaptive partitioning in the accumulator (Java
5117    /// `KafkaProducer#KafkaProducer`: "no need ... if we use a custom partitioner"). The
5118    /// control producer (no `partitioner.type`) keeps `partitioner = None` and adaptive
5119    /// partitioning follows the config default.
5120    ///
5121    /// Rust-only behavioral test (Java never inspects these internals); it pins the
5122    /// `new` resolution + accumulator gating D3/D4 added. `#[tokio::test]`:
5123    /// `new` spawns the Sender task, which fails to reach localhost:9999
5124    /// harmlessly and is dropped with the test.
5125    #[tokio::test]
5126    async fn test_round_robin_partitioner_resolution_and_gating() {
5127        {
5128            let name = ProducerConfig::ROUND_ROBIN_PARTITIONER;
5129            let props = guard_props(&[("partitioner.type", name)]);
5130            let config = ProducerConfig::new(&props).expect("valid config");
5131            let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
5132                .expect("RoundRobinPartitioner resolves");
5133            assert!(
5134                producer.partitioner.is_some(),
5135                "partitioner.type={name} must resolve a partitioner"
5136            );
5137            assert!(
5138                !producer.accumulator.enable_adaptive_partitioning_for_test(),
5139                "a resolved partitioner must disable adaptive partitioning (partitioner.type={name})"
5140            );
5141        }
5142
5143        // Control: no partitioner.type -> no partitioner, adaptive follows config.
5144        let props = guard_props(&[]);
5145        let config = ProducerConfig::new(&props).expect("valid config");
5146        let adaptive_default = config.partitioner_adaptive_partitioning_enable;
5147        let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
5148            .expect("default config constructs");
5149        assert!(producer.partitioner.is_none(), "no partitioner.type -> no partitioner");
5150        assert_eq!(
5151            adaptive_default,
5152            producer.accumulator.enable_adaptive_partitioning_for_test(),
5153            "without a partitioner, adaptive partitioning follows partitioner.adaptive.partitioning.enable"
5154        );
5155    }
5156
5157    /// A stateful `RoundRobinPartitioner` used through the producer's send path is
5158    /// consulted EXACTLY once per record and cycles through the topic's available
5159    /// partitions. A recording decorator wraps the RoundRobin; four sends over a
5160    /// 3-partition topic must produce four calls whose first three returned partitions
5161    /// are a permutation of {0,1,2} and whose fourth repeats the first (the counter
5162    /// wraps mod 3). The available-partition ORDER is not contractually fixed, so the
5163    /// assertion is order-robust rather than hard-coded to `[0,1,2,0]`.
5164    #[tokio::test]
5165    async fn test_round_robin_partitioner_used_once_per_record() {
5166        let calls = Arc::new(Mutex::new(Vec::new()));
5167        let partitioner =
5168            RecordingPartitioner { inner: Box::new(RoundRobinPartitioner::new()), calls: Arc::clone(&calls) };
5169        let producer = create_producer_with_partitioner(
5170            ProducerConfig::defaults(),
5171            create_metadata_with_topic("topic", 3),
5172            create_accumulator(),
5173            Box::new(partitioner),
5174        );
5175
5176        for _ in 0..4 {
5177            let record = ProducerRecord::with_key("topic".to_string(), Some("k".to_string()), Some("v".to_string()));
5178            producer.send(record).await.expect("send succeeds");
5179        }
5180
5181        let recorded = calls.lock().unwrap();
5182        assert_eq!(4, recorded.len(), "the partitioner is consulted exactly once per record");
5183        let returned: Vec<i32> = recorded.iter().map(|c| c.returned).collect();
5184        let mut first_cycle = returned[..3].to_vec();
5185        first_cycle.sort_unstable();
5186        assert_eq!(vec![0, 1, 2], first_cycle, "the first cycle visits every partition once");
5187        assert_eq!(returned[0], returned[3], "the fourth record wraps back to the first partition");
5188    }
5189
5190    /// The producer forwards the record's typed key/value AND the serialized bytes to
5191    /// the partitioner unchanged. A single send of key "mykey"/value "myval" with no
5192    /// explicit partition must record `key=Some("mykey")`, `key_bytes=b"mykey"`,
5193    /// `value=Some("myval")`, `value_bytes=b"myval"` (the `StringSerializer` output).
5194    #[tokio::test]
5195    async fn test_partitioner_receives_key_value_and_serialized_bytes() {
5196        let calls = Arc::new(Mutex::new(Vec::new()));
5197        let partitioner =
5198            RecordingPartitioner { inner: Box::new(RoundRobinPartitioner::new()), calls: Arc::clone(&calls) };
5199        let producer = create_producer_with_partitioner(
5200            ProducerConfig::defaults(),
5201            create_metadata_with_topic("topic", 3),
5202            create_accumulator(),
5203            Box::new(partitioner),
5204        );
5205
5206        let record =
5207            ProducerRecord::with_key("topic".to_string(), Some("mykey".to_string()), Some("myval".to_string()));
5208        producer.send(record).await.expect("send succeeds");
5209
5210        let recorded = calls.lock().unwrap();
5211        assert_eq!(1, recorded.len());
5212        let call = &recorded[0];
5213        assert_eq!("topic", call.topic);
5214        assert_eq!(Some("mykey".to_string()), call.key);
5215        assert_eq!(Some(b"mykey".to_vec()), call.key_bytes);
5216        assert_eq!(Some("myval".to_string()), call.value);
5217        assert_eq!(Some(b"myval".to_vec()), call.value_bytes);
5218    }
5219
5220    /// An explicit record partition short-circuits `compute_partition` before the
5221    /// partitioner is consulted (it returns the given `Some(p)` immediately). A send
5222    /// with an explicit partition therefore records ZERO partitioner calls and still
5223    /// succeeds.
5224    #[tokio::test]
5225    async fn test_explicit_partition_bypasses_partitioner() {
5226        let calls = Arc::new(Mutex::new(Vec::new()));
5227        let partitioner =
5228            RecordingPartitioner { inner: Box::new(RoundRobinPartitioner::new()), calls: Arc::clone(&calls) };
5229        let producer = create_producer_with_partitioner(
5230            ProducerConfig::defaults(),
5231            create_metadata_with_topic("topic", 3),
5232            create_accumulator(),
5233            Box::new(partitioner),
5234        );
5235
5236        let record = ProducerRecord::with_partition_key(
5237            "topic".to_string(),
5238            Some(2),
5239            Some("k".to_string()),
5240            Some("v".to_string()),
5241        )
5242        .unwrap();
5243        producer.send(record).await.expect("send with explicit partition succeeds");
5244
5245        assert!(
5246            calls.lock().unwrap().is_empty(),
5247            "explicit partition must bypass the partitioner"
5248        );
5249    }
5250
5251    /// `ProducerConfig::set_partitioner` replaces a built-in `partitioner.type`
5252    /// from the properties, as a later value for the same key would in Java.
5253    /// Verified via `configure`: only `PartitionerForClientId` records the client.id
5254    /// (`RoundRobinPartitioner`'s `configure` is the default no-op), so a single
5255    /// recorded id proves the user-written partitioner — not the RoundRobin named in
5256    /// the properties — was built and configured.
5257    #[tokio::test]
5258    async fn test_set_partitioner_replaces_partitioner_type_property() {
5259        let client_ids = Arc::new(Mutex::new(Vec::new()));
5260        let props = guard_props(&[("partitioner.type", "RoundRobinPartitioner")]);
5261        let config = ProducerConfig::new(&props)
5262            .expect("valid config")
5263            .set_partitioner::<String, String>(Box::new(PartitionerForClientId {
5264                client_ids: Arc::clone(&client_ids),
5265            }));
5266        let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
5267            .expect("set_partitioner overrides partitioner.type");
5268
5269        let recorded = client_ids.lock().unwrap();
5270        assert_eq!(1, recorded.len(), "PartitionerForClientId was configured, not RoundRobin");
5271        assert_eq!(recorded[0].as_str(), producer.client_id());
5272        assert!(
5273            !producer.accumulator.enable_adaptive_partitioning_for_test(),
5274            "a custom partitioner disables adaptive partitioning"
5275        );
5276    }
5277
5278    /// A partitioner set with `ProducerConfig::set_partitioner` for other
5279    /// record types than the producer's fails construction, as Java's
5280    /// `getConfiguredInstance` throws `KafkaException("<class> is not an instance of
5281    /// <type>")` for a class that does not implement the requested interface. The
5282    /// failure is relabelled like every other construction failure.
5283    #[tokio::test]
5284    async fn test_set_partitioner_with_other_record_types_fails() {
5285        let props = guard_props(&[]);
5286        let config = ProducerConfig::new(&props)
5287            .expect("valid config")
5288            .set_partitioner::<Vec<u8>, Vec<u8>>(Box::new(RoundRobinPartitioner::new()));
5289        let err = match KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer) {
5290            Ok(_) => panic!("a partitioner for other record types must be rejected"),
5291            Err(err) => err,
5292        };
5293        assert!(err.to_string().contains("Failed to construct kafka producer"), "{err}");
5294        let source = std::error::Error::source(&err)
5295            .expect("the cause is kept as the source")
5296            .to_string();
5297        assert!(
5298            source.contains(&format!(
5299                "{} is not an instance of {}",
5300                std::any::type_name::<dyn Partitioner<Vec<u8>, Vec<u8>>>(),
5301                std::any::type_name::<dyn Partitioner<String, String>>()
5302            )),
5303            "{source}"
5304        );
5305    }
5306
5307    /// Tests that a record can be successfully sent and appended to the accumulator.
5308    ///
5309    /// Translated from `KafkaProducerTest.testFlushCompleteSendOfInflightBatches`
5310    /// (the send part).
5311    #[tokio::test]
5312    async fn test_send_appends_to_accumulator() {
5313        let metadata = create_metadata_with_topic(TOPIC, 1);
5314        let accumulator = create_accumulator();
5315        let producer = create_producer(metadata, Arc::clone(&accumulator));
5316
5317        let record = ProducerRecord::with_key(TOPIC.to_string(), Some("key".to_string()), Some("value".to_string()));
5318        let result = producer.send(record).await;
5319        assert!(result.is_ok(), "Send should succeed");
5320
5321        // Verify that the accumulator has undrained batches
5322        assert!(accumulator.has_undrained());
5323    }
5324
5325    /// Tests that flush with no incomplete batches returns immediately.
5326    ///
5327    /// When there are no pending records, flush should complete instantly
5328    /// since there is nothing to wait for.
5329    #[tokio::test]
5330    async fn test_flush_with_no_pending_records() {
5331        let metadata = create_metadata_with_topic(TOPIC, 1);
5332        let accumulator = create_accumulator();
5333        let producer = create_producer(metadata, accumulator);
5334
5335        // Flush with nothing pending should succeed immediately
5336        let result = producer.flush().await;
5337        assert!(result.is_ok());
5338    }
5339
5340    /// Translated from `KafkaProducerTest.testDeliveryTimeoutAndLingerMsConfig`.
5341    ///
5342    /// `delivery.timeout.ms` must be >= `linger.ms + request.timeout.ms`. When the
5343    /// user set it explicitly, Java throws `ConfigException` (`KafkaProducer`)
5344    /// — and the Java test asserts `KafkaException.class`, so the error must answer
5345    /// `true` to `is_kafka_error()`.
5346    #[test]
5347    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testDeliveryTimeoutAndLingerMsConfig")]
5348    fn test_delivery_timeout_and_linger_ms_config() {
5349        let mut props = HashMap::new();
5350        props.insert("client.id".to_string(), "testDeliveryTimeoutAndLingerMsConfig".to_string());
5351        props.insert("bootstrap.servers".to_string(), "localhost:9999".to_string());
5352        props.insert("delivery.timeout.ms".to_string(), "1000".to_string());
5353        props.insert("linger.ms".to_string(), "1000".to_string());
5354        props.insert("request.timeout.ms".to_string(), "1".to_string());
5355        let config = ProducerConfig::new(&props).expect("these properties parse");
5356        let log_context = LogContext::new("[test] ".to_string());
5357
5358        let err = KafkaProducer::<String, String>::configure_delivery_timeout(&config, &log_context)
5359            .expect_err("Should reject an explicit delivery_timeout_ms < linger_ms + request_timeout_ms");
5360        assert_eq!(
5361            err.message(),
5362            "delivery.timeout.ms should be equal to or larger than linger.ms + request.timeout.ms"
5363        );
5364        assert!(matches!(err, Error::Config(_)), "Expected a Config error, got: {err:?}");
5365        // `ConfigException extends KafkaException`, which is what the Java test asserts.
5366        assert!(err.is_kafka_error(), "Java's ConfigException is a KafkaException");
5367        assert!(!err.is_api_error(), "Java's ConfigException is not an ApiException");
5368
5369        // Second half of the Java test: linger.ms = 999 makes the sum exactly 1000,
5370        // so construction succeeds.
5371        props.insert("linger.ms".to_string(), "999".to_string());
5372        let config = ProducerConfig::new(&props).expect("these properties parse");
5373        assert_eq!(
5374            KafkaProducer::<String, String>::configure_delivery_timeout(&config, &log_context).unwrap(),
5375            1000
5376        );
5377    }
5378
5379    /// Java's *other* arm: when `delivery.timeout.ms` was NOT supplied by the user,
5380    /// an inconsistency is clamped up to `linger.ms + request.timeout.ms` and warned
5381    /// about rather than rejected (`KafkaProducer`).
5382    ///
5383    /// `request.timeout.ms = 180000` with everything else defaulted
5384    /// (`delivery.timeout.ms = 120000`, `linger.ms = 5`) is a legal Kafka
5385    /// configuration: Java silently raises the delivery timeout to 180005.
5386    #[test]
5387    fn test_delivery_timeout_default_is_clamped_not_rejected() {
5388        let mut props = HashMap::new();
5389        props.insert("bootstrap.servers".to_string(), "localhost:9092".to_string());
5390        props.insert("request.timeout.ms".to_string(), "180000".to_string());
5391        let config = ProducerConfig::new(&props).expect("these properties parse");
5392        assert!(
5393            !config.user_configured(ProducerConfig::DELIVERY_TIMEOUT_MS_CONFIG),
5394            "the test relies on delivery.timeout.ms coming from the default"
5395        );
5396        assert_eq!(config.delivery_timeout_ms, 120_000);
5397        assert_eq!(config.linger_ms, 5);
5398
5399        let log_context = LogContext::new("[test] ".to_string());
5400        let delivery_timeout_ms = KafkaProducer::<String, String>::configure_delivery_timeout(&config, &log_context)
5401            .expect("an inconsistency the user did not ask for is clamped, not rejected");
5402        assert_eq!(delivery_timeout_ms, 180_005, "clamped to linger.ms + request.timeout.ms");
5403    }
5404
5405    /// Tests that configure_delivery_timeout accepts valid configurations.
5406    #[test]
5407    fn test_delivery_timeout_valid_config() {
5408        let log_context = LogContext::new("[test] ".to_string());
5409
5410        // Default values: delivery=120000, linger=5, request=30000
5411        let config = ProducerConfig::defaults();
5412        let result = KafkaProducer::<String, String>::configure_delivery_timeout(&config, &log_context);
5413        assert!(result.is_ok());
5414        assert_eq!(result.unwrap(), 120_000);
5415
5416        // Exactly equal: delivery = linger + request
5417        let config = ProducerConfig {
5418            delivery_timeout_ms: 30_010,
5419            linger_ms: 10,
5420            request_timeout_ms: 30_000,
5421            ..ProducerConfig::defaults()
5422        };
5423        let result = KafkaProducer::<String, String>::configure_delivery_timeout(&config, &log_context);
5424        assert!(result.is_ok());
5425        assert_eq!(result.unwrap(), 30_010);
5426    }
5427
5428    /// `linger.ms` values above `i32::MAX` are clamped to `i32::MAX`, as Java's
5429    /// `KafkaProducer.lingerMs` does, instead of being truncated to their low 32
5430    /// bits. The clamped value also bounds `linger.ms + request.timeout.ms` in
5431    /// `configure_delivery_timeout`.
5432    #[test]
5433    fn test_linger_ms_beyond_i32_max_is_clamped() {
5434        let log_context = LogContext::new("[test] ".to_string());
5435        let max = i32::MAX.to_string();
5436        let i64_max = i64::MAX.to_string();
5437        for (linger_ms, expected) in [
5438            ("0", 0),
5439            ("5", 5),
5440            (max.as_str(), i32::MAX),
5441            ("2147483648", i32::MAX),
5442            ("3000000000", i32::MAX),
5443            ("4294967296", i32::MAX),
5444            (i64_max.as_str(), i32::MAX),
5445        ] {
5446            let config = ProducerConfig::new(&guard_props(&[("linger.ms", linger_ms)])).expect("valid config");
5447            assert_eq!(
5448                KafkaProducer::<String, String>::linger_ms(&config),
5449                expected,
5450                "linger.ms={linger_ms}"
5451            );
5452            if expected == i32::MAX {
5453                assert_eq!(
5454                    KafkaProducer::<String, String>::configure_delivery_timeout(&config, &log_context).unwrap(),
5455                    i32::MAX,
5456                    "the default delivery.timeout.ms is raised to the clamped sum (linger.ms={linger_ms})"
5457                );
5458            }
5459        }
5460    }
5461
5462    /// The producer passes the clamped `linger.ms` to its accumulator, so a value
5463    /// above `i32::MAX` keeps batches lingering instead of making them ready at once.
5464    /// `#[tokio::test]`: `new` spawns the Sender task, which fails to reach
5465    /// localhost:9999 harmlessly and is dropped with the test.
5466    #[tokio::test]
5467    async fn test_linger_ms_beyond_i32_max_reaches_accumulator_clamped() {
5468        for linger_ms in ["3000000000", "4294967296"] {
5469            let config = ProducerConfig::new(&guard_props(&[("linger.ms", linger_ms)])).expect("valid config");
5470            let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
5471                .expect("a large linger.ms is a valid configuration");
5472            assert_eq!(producer.accumulator.linger_ms_for_test(), i32::MAX, "linger.ms={linger_ms}");
5473        }
5474    }
5475
5476    /// Tests that `negativePartitionShouldThrow` from Java is already handled
5477    /// by `ProducerRecord` validation. Negative partition is rejected at record
5478    /// construction time.
5479    ///
5480    /// Translated from `KafkaProducerTest.negativePartitionShouldThrow`.
5481    #[test]
5482    fn test_negative_partition_should_error() {
5483        let result: Result<ProducerRecord<String, String>, _> = ProducerRecord::with_partition_key(
5484            TOPIC.to_string(),
5485            Some(-1),
5486            Some("key".to_string()),
5487            Some("value".to_string()),
5488        );
5489        assert!(result.is_err(), "Negative partition should be rejected");
5490    }
5491
5492    /// Tests the wait_on_metadata method with an already-known topic.
5493    #[tokio::test]
5494    async fn test_wait_on_metadata_returns_immediately_for_known_topic() {
5495        let metadata = create_metadata_with_topic(TOPIC, 3);
5496        let accumulator = create_accumulator();
5497        let producer = create_producer(metadata, accumulator);
5498
5499        let now_ms = producer.now_ms();
5500        let result = producer.wait_on_metadata(TOPIC, None, now_ms, 1000).await;
5501        assert!(result.is_ok());
5502        let cwt = result.unwrap();
5503        assert_eq!(0, cwt.waited_on_metadata_ms);
5504        assert_eq!(3, cwt.cluster.partitions_for_topic(TOPIC).len());
5505    }
5506
5507    /// Tests that wait_on_metadata returns immediately when partition is within known range.
5508    #[tokio::test]
5509    async fn test_wait_on_metadata_returns_for_valid_partition() {
5510        let metadata = create_metadata_with_topic(TOPIC, 3);
5511        let accumulator = create_accumulator();
5512        let producer = create_producer(metadata, accumulator);
5513
5514        let now_ms = producer.now_ms();
5515        let result = producer.wait_on_metadata(TOPIC, Some(2), now_ms, 1000).await;
5516        assert!(result.is_ok());
5517    }
5518
5519    /// Tests that wait_on_metadata times out for a topic that does not exist.
5520    ///
5521    /// Translated from `KafkaProducerTest.testTopicNotExistingInMetadata`.
5522    #[tokio::test]
5523    async fn test_wait_on_metadata_times_out_for_unknown_topic() {
5524        let metadata = create_metadata_with_topic(TOPIC, 1);
5525        let accumulator = create_accumulator();
5526        let producer = create_producer(metadata, accumulator);
5527
5528        let now_ms = producer.now_ms();
5529        // Try to get metadata for a topic that doesn't exist with a very short timeout
5530        let result = producer.wait_on_metadata("nonexistent-topic", None, now_ms, 100).await;
5531        assert!(result.is_err());
5532        match result.unwrap_err() {
5533            Error::Timeout(msg) => {
5534                assert!(msg.message().contains("not present in metadata"), "Got: {}", msg);
5535            },
5536            other => panic!("Expected Timeout error, got: {:?}", other),
5537        }
5538    }
5539
5540    /// Tests the client_id accessor.
5541    ///
5542    /// Translated from `KafkaProducerTest.getClientId` (via visible for testing).
5543    #[test]
5544    fn test_get_client_id() {
5545        let config = ProducerConfig { client_id: "my-producer".to_string(), ..ProducerConfig::defaults() };
5546        let metadata = create_metadata_with_topic(TOPIC, 1);
5547        let accumulator = create_accumulator();
5548
5549        let producer = create_producer_with_config(config, metadata, accumulator);
5550
5551        assert_eq!("my-producer", producer.client_id());
5552    }
5553
5554    /// Tests that multiple sends to the same topic produce deterministic partition
5555    /// assignment when keys are provided.
5556    #[tokio::test]
5557    async fn test_send_multiple_records_with_same_key() {
5558        let metadata = create_metadata_with_topic(TOPIC, 3);
5559        let accumulator = create_accumulator();
5560        let producer = create_producer(metadata, Arc::clone(&accumulator));
5561
5562        let record1 =
5563            ProducerRecord::with_key(TOPIC.to_string(), Some("same-key".to_string()), Some("value1".to_string()));
5564        let record2 =
5565            ProducerRecord::with_key(TOPIC.to_string(), Some("same-key".to_string()), Some("value2".to_string()));
5566
5567        let future1 = producer.send(record1).await.unwrap();
5568        let future2 = producer.send(record2).await.unwrap();
5569
5570        // Both should succeed — each send returns a distinct future
5571        assert!(!future1.is_done());
5572        assert!(!future2.is_done());
5573    }
5574
5575    /// Tests that sending with a callback invokes the callback on completion.
5576    ///
5577    /// Translated from `KafkaProducerTest.testCallbackAndInterceptorHandleError`
5578    /// (the callback invocation part).
5579    #[tokio::test]
5580    async fn test_send_callback() {
5581        use std::sync::atomic::AtomicBool;
5582
5583        let metadata = create_metadata_with_topic(TOPIC, 1);
5584        let accumulator = create_accumulator();
5585        let producer = create_producer(metadata, Arc::clone(&accumulator));
5586
5587        let callback_called = Arc::new(AtomicBool::new(false));
5588        let callback_called_clone = Arc::clone(&callback_called);
5589        let callback: Box<dyn Callback> =
5590            Box::new(move |_metadata: Option<&RecordMetadata>, _error: Option<&Error>| {
5591                callback_called_clone.store(true, Ordering::SeqCst);
5592            });
5593
5594        let record = ProducerRecord::with_key(TOPIC.to_string(), Some("key".to_string()), Some("value".to_string()));
5595        let result = producer.send_with_callback(record, Some(callback)).await;
5596        assert!(result.is_ok(), "Send with callback should succeed");
5597    }
5598
5599    /// Tests that sending a record with null (None) key and value works.
5600    ///
5601    /// Translated from `KafkaProducerTest.testNullTopicName` (partial — tests
5602    /// that the producer handles empty/null values).
5603    #[tokio::test]
5604    async fn test_send_with_none_key_and_value() {
5605        let metadata = create_metadata_with_topic(TOPIC, 1);
5606        let accumulator = create_accumulator();
5607        let producer = create_producer(metadata, Arc::clone(&accumulator));
5608
5609        let record: ProducerRecord<String, String> = ProducerRecord::new(TOPIC.to_string(), None);
5610        let result = producer.send(record).await;
5611        assert!(result.is_ok(), "Send with None value should succeed");
5612    }
5613
5614    /// Translated from `KafkaProducerTest.testCallbackAndInterceptorHandleError`.
5615    ///
5616    /// Tests that when sending to a topic that will cause an ApiException
5617    /// (RecordTooLargeException), the callback is invoked with the error and
5618    /// a non-null RecordMetadata with appropriate defaults.
5619    #[tokio::test]
5620    async fn test_callback_invoked_on_api_error() {
5621        let config = ProducerConfig { max_request_size: 10, ..ProducerConfig::defaults() };
5622        let metadata = create_metadata_with_topic(TOPIC, 1);
5623        let accumulator = create_accumulator();
5624        let producer = create_producer_with_config(config, metadata, accumulator);
5625
5626        let callback_invoked = Arc::new(AtomicBool::new(false));
5627        let got_error = Arc::new(std::sync::Mutex::new(false));
5628        let got_metadata = Arc::new(std::sync::Mutex::new(false));
5629        let metadata_topic = Arc::new(std::sync::Mutex::new(String::new()));
5630        let metadata_offset = Arc::new(std::sync::Mutex::new(0i64));
5631
5632        let inv = Arc::clone(&callback_invoked);
5633        let err = Arc::clone(&got_error);
5634        let meta = Arc::clone(&got_metadata);
5635        let mt = Arc::clone(&metadata_topic);
5636        let mo = Arc::clone(&metadata_offset);
5637
5638        let callback: Box<dyn Callback> =
5639            Box::new(move |record_metadata: Option<&RecordMetadata>, error: Option<&Error>| {
5640                inv.store(true, Ordering::SeqCst);
5641                *err.lock().unwrap() = error.is_some();
5642                if let Some(rm) = record_metadata {
5643                    *meta.lock().unwrap() = true;
5644                    *mt.lock().unwrap() = rm.topic().to_string();
5645                    *mo.lock().unwrap() = rm.offset();
5646                }
5647            });
5648
5649        // Send a record that exceeds max_request_size
5650        let large_value = "a".repeat(100);
5651        let record = ProducerRecord::new(TOPIC.to_string(), Some(large_value));
5652        let result = producer.send_with_callback(record, Some(callback)).await;
5653
5654        // send() returns Ok with a failed future (Java's FutureFailure pattern)
5655        assert!(result.is_ok(), "send() should return Ok with a failed future");
5656        let future = result.unwrap();
5657        assert!(future.is_done(), "Failed future should be immediately done");
5658
5659        // Verify callback was invoked with the error
5660        assert!(callback_invoked.load(Ordering::SeqCst), "Callback should have been invoked");
5661        assert!(*got_error.lock().unwrap(), "Callback should receive error");
5662        assert!(*got_metadata.lock().unwrap(), "Callback should receive non-null metadata");
5663        assert_eq!(
5664            *metadata_topic.lock().unwrap(),
5665            TOPIC,
5666            "Callback metadata should have the topic"
5667        );
5668        assert_eq!(
5669            *metadata_offset.lock().unwrap(),
5670            RecordMetadata::INVALID_OFFSET,
5671            "Callback metadata should have INVALID_OFFSET"
5672        );
5673
5674        // Future.get() should return the error
5675        let err = future.get().await.unwrap_err();
5676        assert!(matches!(err, Error::RecordTooLarge(_)));
5677    }
5678
5679    /// Mirrors the `KafkaProducerTest.testCallbackAndInterceptorHandleError`
5680    /// contract for the failure mode that happens *inside*
5681    /// `RecordAccumulator.append` rather than before it: buffer exhaustion /
5682    /// `max.block.ms` expiry.
5683    ///
5684    /// Java's `BufferPool.allocate` throws `BufferExhaustedException` (an
5685    /// `ApiException`) out of `append`, and `doSend`'s
5686    /// `catch (ApiException e)` arm still fires the user callback with the
5687    /// placeholder metadata because it holds its own `callback` reference
5688    /// (`KafkaProducer`). Rust *moves* the callback into
5689    /// `append`, so the callback obligation (CLAUDE.md §7, §11.5) is only met
5690    /// because `AppendFailure` hands it back — this test is the regression guard
5691    /// for that hand-back.
5692    #[tokio::test]
5693    async fn test_callback_invoked_on_buffer_exhaustion() {
5694        const BATCH_SIZE: i32 = 16384;
5695
5696        // `max.block.ms = 0` so the second allocation fails immediately
5697        // instead of waiting for memory that will never be freed.
5698        let config = ProducerConfig { max_block_ms: 0, ..ProducerConfig::defaults() };
5699        // Two partitions so the second record needs a *new* batch (and hence a
5700        // new buffer) instead of appending to the first one.
5701        let metadata = create_metadata_with_topic(TOPIC, 2);
5702        // A pool that fits exactly one batch.
5703        let accumulator = Arc::new(RecordAccumulator::new_for_test(
5704            BATCH_SIZE,
5705            Compression::none().build(),
5706            5,
5707            100,
5708            1000,
5709            120_000,
5710            PartitionerConfig::new(true, 0, false, "").unwrap(),
5711            default_time(),
5712            Arc::new(BufferPool::new_for_test(BATCH_SIZE as i64, BATCH_SIZE as usize)),
5713            None,
5714        ));
5715        let producer = create_producer_with_config(config, metadata, Arc::clone(&accumulator));
5716
5717        // First record drains the whole pool.
5718        let first: ProducerRecord<String, String> =
5719            ProducerRecord::with_partition_key(TOPIC.to_string(), Some(0), None, Some("value".to_string())).unwrap();
5720        producer.send(first).await.expect("the first send should succeed");
5721
5722        // Second record targets a different partition, so it must allocate.
5723        let invoked = Arc::new(std::sync::atomic::AtomicI32::new(0));
5724        let saw_error = Arc::new(std::sync::Mutex::new(None::<String>));
5725        let saw_metadata = Arc::new(std::sync::Mutex::new(None::<(String, i32, i64)>));
5726
5727        let inv = Arc::clone(&invoked);
5728        let err_slot = Arc::clone(&saw_error);
5729        let meta_slot = Arc::clone(&saw_metadata);
5730        let callback: Box<dyn Callback> =
5731            Box::new(move |record_metadata: Option<&RecordMetadata>, exception: Option<&Error>| {
5732                inv.fetch_add(1, Ordering::SeqCst);
5733                *err_slot.lock().unwrap() = exception.map(|e| e.to_string());
5734                *meta_slot.lock().unwrap() =
5735                    record_metadata.map(|rm| (rm.topic().to_string(), rm.partition(), rm.offset()));
5736            });
5737
5738        let second: ProducerRecord<String, String> =
5739            ProducerRecord::with_partition_key(TOPIC.to_string(), Some(1), None, Some("value".to_string())).unwrap();
5740        let result = producer.send_with_callback(second, Some(callback)).await;
5741
5742        // Java returns a `FutureFailure`, not a thrown exception, for an
5743        // `ApiException`.
5744        let future = result.expect("an ApiException is reported through the future, not the Result");
5745        assert!(future.is_done(), "the failed future should be immediately done");
5746        let future_error = future.get().await.unwrap_err();
5747        assert!(
5748            matches!(future_error, Error::ProducerBufferExhausted(_)),
5749            "the future should report buffer exhaustion, got {:?}",
5750            future_error
5751        );
5752
5753        // The callback fired exactly once, with BOTH the placeholder metadata
5754        // and the error, exactly as Java's `catch (ApiException e)` arm does.
5755        assert_eq!(1, invoked.load(Ordering::SeqCst), "the callback must fire exactly once");
5756        let message = saw_error.lock().unwrap().clone().expect("the callback must receive the error");
5757        assert!(
5758            message.contains("Failed to allocate"),
5759            "the callback should receive the buffer-exhaustion error, got {}",
5760            message
5761        );
5762        let (topic, partition, offset) = saw_metadata
5763            .lock()
5764            .unwrap()
5765            .clone()
5766            .expect("the callback must receive non-null metadata");
5767        assert_eq!(TOPIC, topic);
5768        assert_eq!(1, partition);
5769        assert_eq!(RecordMetadata::INVALID_OFFSET, offset);
5770    }
5771
5772    /// When the producer chooses the partition, a buffer-exhaustion failure reports the
5773    /// partition `append` chose, not `UNKNOWN_PARTITION`.
5774    ///
5775    /// Java's `append` passes the chosen partition to `AppendCallbacks.setPartition`
5776    /// before it waits for memory, and `doSend` builds the callback's metadata from
5777    /// `appendCallbacks.topicPartition()`. Not a translation of a Java test:
5778    /// `test_callback_invoked_on_buffer_exhaustion` sends to an explicit partition, so it
5779    /// cannot tell the two apart.
5780    #[tokio::test]
5781    async fn test_buffer_exhaustion_reports_the_partition_chosen_by_the_partitioner() {
5782        const BATCH_SIZE: i32 = 16384;
5783
5784        // `max.block.ms = 0` so the second allocation fails immediately.
5785        let config = ProducerConfig { max_block_ms: 0, ..ProducerConfig::defaults() };
5786        // One partition, so the built-in partitioner can only choose partition 0.
5787        let metadata = create_metadata_with_topic(TOPIC, 1);
5788        // A pool that fits exactly one batch.
5789        let accumulator = Arc::new(RecordAccumulator::new_for_test(
5790            BATCH_SIZE,
5791            Compression::none().build(),
5792            5,
5793            100,
5794            1000,
5795            120_000,
5796            PartitionerConfig::new(true, 0, false, "").unwrap(),
5797            default_time(),
5798            Arc::new(BufferPool::new_for_test(BATCH_SIZE as i64, BATCH_SIZE as usize)),
5799            None,
5800        ));
5801        let producer = create_producer_with_config(config, metadata, Arc::clone(&accumulator));
5802
5803        // The first record takes the pool's only batch and leaves no room in it for
5804        // the second, so the second needs a new batch and must allocate.
5805        let first: ProducerRecord<String, String> =
5806            ProducerRecord::with_partition_key(TOPIC.to_string(), None, None, Some("a".repeat(15_000))).unwrap();
5807        producer.send(first).await.expect("the first send should succeed");
5808
5809        let saw_partition = Arc::new(std::sync::Mutex::new(None::<i32>));
5810        let partition_slot = Arc::clone(&saw_partition);
5811        let callback: Box<dyn Callback> =
5812            Box::new(move |record_metadata: Option<&RecordMetadata>, _exception: Option<&Error>| {
5813                *partition_slot.lock().unwrap() = record_metadata.map(|rm| rm.partition());
5814            });
5815        let second: ProducerRecord<String, String> =
5816            ProducerRecord::with_partition_key(TOPIC.to_string(), None, None, Some("b".repeat(2_000))).unwrap();
5817        let future = producer
5818            .send_with_callback(second, Some(callback))
5819            .await
5820            .expect("an ApiException is reported through the future, not the Result");
5821        assert!(
5822            matches!(future.get().await.unwrap_err(), Error::ProducerBufferExhausted(_)),
5823            "the second send must fail with buffer exhaustion"
5824        );
5825
5826        assert_eq!(
5827            Some(0),
5828            *saw_partition.lock().unwrap(),
5829            "the callback must report the partition the partitioner chose"
5830        );
5831    }
5832
5833    /// The mirror image of `test_callback_invoked_on_buffer_exhaustion`: the
5834    /// other failure that happens *inside* `RecordAccumulator.append` must
5835    /// **not** fire the callback.
5836    ///
5837    /// "Producer closed while send in progress" is a **bare** `KafkaException`
5838    /// in Java (`RecordAccumulator`), not an `ApiException`, so
5839    /// `doSend` skips `catch (ApiException e)` (the arm that invokes the
5840    /// callback) and lands in `catch (KafkaException e)`, which records the
5841    /// error, notifies the interceptors and **rethrows**
5842    /// (`KafkaProducer`). Firing the callback here would break
5843    /// the exactly-once obligation as surely as dropping one where Java fires.
5844    ///
5845    /// The accumulator is closed while the producer itself is not, which is
5846    /// exactly the race the Java message names: `ensure_not_closed()` and
5847    /// `wait_on_metadata` both pass and the rejection comes from `append`.
5848    #[tokio::test]
5849    async fn test_callback_not_invoked_when_the_accumulator_is_closed() {
5850        let metadata = create_metadata_with_topic(TOPIC, 1);
5851        let accumulator = create_accumulator();
5852        let producer = create_producer(metadata, Arc::clone(&accumulator));
5853        accumulator.close();
5854
5855        let invoked = Arc::new(std::sync::atomic::AtomicI32::new(0));
5856        let inv = Arc::clone(&invoked);
5857        let callback: Box<dyn Callback> = Box::new(move |_: Option<&RecordMetadata>, _: Option<&Error>| {
5858            inv.fetch_add(1, Ordering::SeqCst);
5859        });
5860
5861        let record: ProducerRecord<String, String> =
5862            ProducerRecord::with_partition_key(TOPIC.to_string(), Some(0), None, Some("value".to_string())).unwrap();
5863        let error = match producer.send_with_callback(record, Some(callback)).await {
5864            Ok(_) => panic!("a bare KafkaException must propagate as Err, not as a failed future"),
5865            Err(e) => e,
5866        };
5867
5868        assert!(
5869            matches!(error, Error::KafkaError(_)),
5870            "the closed-mid-send failure must be a bare KafkaException, not an ApiException — \
5871             `is_api_error()` is what routes it away from the callback, got {:?}",
5872            error
5873        );
5874        assert_eq!("Producer closed while send in progress", error.message());
5875        assert!(!error.is_api_error(), "a bare KafkaException is not an ApiException");
5876        assert!(error.is_kafka_error(), "it is still a KafkaException");
5877        assert_eq!(
5878            0,
5879            invoked.load(Ordering::SeqCst),
5880            "Java's `catch (KafkaException e)` arm rethrows without invoking the callback"
5881        );
5882    }
5883
5884    /// Translated from `KafkaProducerTest.testHeadersSuccess`.
5885    ///
5886    /// Tests that headers added to a ProducerRecord before send() are passed
5887    /// through serialization correctly and stored in the accumulator.
5888    #[tokio::test]
5889    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testHeadersSuccess")]
5890    async fn test_headers_success() {
5891        use crate::common::header::Headers;
5892        use crate::common::header::RecordHeader;
5893
5894        let metadata = create_metadata_with_topic(TOPIC, 1);
5895        let accumulator = create_accumulator();
5896        let producer = create_producer(metadata, Arc::clone(&accumulator));
5897
5898        let mut record: ProducerRecord<String, String> =
5899            ProducerRecord::with_key(TOPIC.to_string(), Some("key".to_string()), Some("value".to_string()));
5900
5901        // Add a header pre-send
5902        record
5903            .headers_mut()
5904            .add_with_header(RecordHeader::new("test".to_string(), Some(b"header-value".to_vec())))
5905            .unwrap();
5906
5907        let result = producer.send(record).await;
5908        assert!(result.is_ok(), "Send with headers should succeed");
5909        assert!(accumulator.has_undrained(), "Accumulator should have batches");
5910        assert_eq!(
5911            accumulator.record_headers_for_test(&TopicPartition::new(TOPIC.to_string(), 0)),
5912            vec![vec![("test".to_string(), Some(b"header-value".to_vec()))]]
5913        );
5914    }
5915
5916    /// A `String` serializer that adds a header named `name`, with the data as
5917    /// its value, and records the keys of the headers it was handed and
5918    /// whether they were writable.
5919    struct HeaderAddingSerializer {
5920        name: &'static str,
5921        seen: Arc<Mutex<Vec<SeenHeaders>>>,
5922    }
5923
5924    /// The header keys a serializer was handed, and whether they were writable.
5925    type SeenHeaders = (Vec<String>, bool);
5926
5927    impl Serializer<String> for HeaderAddingSerializer {
5928        async fn serialize<'a>(
5929            &'a self,
5930            _topic: &'a str,
5931            data: Option<&'a String>,
5932        ) -> Result<Option<Cow<'a, [u8]>>, Error> {
5933            Ok(data.map(|data| Cow::Borrowed(data.as_bytes())))
5934        }
5935
5936        async fn serialize_with_headers<'a, 'h>(
5937            &'a self,
5938            topic: &'a str,
5939            headers: &'h mut RecordHeaders,
5940            data: Option<&'a String>,
5941        ) -> Result<Option<Cow<'a, [u8]>>, Error>
5942        where
5943            'a: 'h,
5944        {
5945            use crate::common::header::Header;
5946            let keys = headers.iter().map(|header| header.key().to_string()).collect();
5947            self.seen.lock().unwrap().push((keys, !headers.is_read_only()));
5948            headers
5949                .add_with_key_value(self.name, data.map(|data| data.as_bytes()))
5950                .map_err(Error::LocalIllegalState)?;
5951            Serializer::serialize(self, topic, data).await
5952        }
5953    }
5954
5955    /// A serializer may add headers (Java's `Serializer.serialize(String,
5956    /// Headers, T)` javadoc): `doSend` hands the record's headers to the key
5957    /// serializer, then the same headers to the value serializer
5958    /// (`KafkaProducer#doSend`), and sends them as they are after
5959    /// both (`KafkaProducer`). The value serializer sees the key serializer's
5960    /// header, and both added headers reach the batch, after the record's own.
5961    #[tokio::test]
5962    async fn test_serializer_added_headers_are_sent() {
5963        use crate::common::header::RecordHeader;
5964
5965        let seen = Arc::new(Mutex::new(Vec::new()));
5966        let accumulator = create_accumulator();
5967        let producer: KafkaProducer<String, String> = KafkaProducer::with_options(
5968            KafkaProducerOptionsBuilder::new()
5969                .set_config(&ProducerConfig::defaults())
5970                .set_key_serializer(Box::new(HeaderAddingSerializer { name: "key-header", seen: Arc::clone(&seen) }))
5971                .set_value_serializer(Box::new(HeaderAddingSerializer {
5972                    name: "value-header",
5973                    seen: Arc::clone(&seen),
5974                }))
5975                .set_metadata(create_metadata_with_topic(TOPIC, 1))
5976                .set_accumulator(Arc::clone(&accumulator))
5977                .set_time(default_time())
5978                .set_pending_requests(Arc::new(Mutex::new(PendingRequests::new())))
5979                .build()
5980                .expect("KafkaProducerOptionsBuilder::build: every mandatory parameter is set above"),
5981        );
5982
5983        let mut record: ProducerRecord<String, String> =
5984            ProducerRecord::with_key(TOPIC.to_string(), Some("key".to_string()), Some("value".to_string()));
5985        record
5986            .headers_mut()
5987            .add_with_header(RecordHeader::new("test".to_string(), Some(b"header-value".to_vec())))
5988            .unwrap();
5989
5990        producer.send(record).await.expect("the send should be accepted");
5991        assert_eq!(
5992            *seen.lock().unwrap(),
5993            [
5994                (vec!["test".to_string()], true),
5995                (vec!["test".to_string(), "key-header".to_string()], true),
5996            ],
5997            "the key serializer runs first; both get the record's writable headers"
5998        );
5999        assert_eq!(
6000            accumulator.record_headers_for_test(&TopicPartition::new(TOPIC.to_string(), 0)),
6001            vec![vec![
6002                ("test".to_string(), Some(b"header-value".to_vec())),
6003                ("key-header".to_string(), Some(b"key".to_vec())),
6004                ("value-header".to_string(), Some(b"value".to_vec())),
6005            ]]
6006        );
6007    }
6008
6009    /// `KafkaProducer.setReadOnly`, which
6010    /// `doSend` runs after both serializers (`KafkaProducer#doSend`): a header added after it
6011    /// fails with `RecordHeaders`' read-only message, as Java's
6012    /// `testHeadersSuccess` asserts after `send` (`KafkaProducerTest`),
6013    /// and the headers already there are kept.
6014    #[test]
6015    fn test_headers_are_read_only_after_serialization() {
6016        use crate::common::header::Header;
6017
6018        let mut headers = RecordHeaders::new();
6019        headers.add_with_key_value("test", Some(b"header2")).unwrap();
6020        KafkaProducer::<String, String>::set_read_only(&mut headers);
6021
6022        assert!(headers.is_read_only());
6023        let error = headers.add_with_key_value("test", Some(b"test")).unwrap_err();
6024        assert_eq!("RecordHeaders has been closed.", Error::LocalIllegalState(error).message());
6025        assert_eq!(
6026            Some(&b"header2"[..]),
6027            headers.last_header("test").and_then(|header| header.value())
6028        );
6029    }
6030
6031    /// The borrowed byte-slice `send`, the path of the C FFI's `send_batch` and
6032    /// single-record sends (so of the C and Python `KafkaProducer`), appends the
6033    /// record's headers as Java's `doSend` does (`KafkaProducer#doSend`,
6034    /// `KafkaProducer#doSend`), in order, with an empty and a null value kept apart. It
6035    /// used to append none.
6036    #[tokio::test]
6037    async fn test_borrowed_send_appends_the_record_headers() {
6038        use crate::common::header::Headers;
6039        use crate::common::header::RecordHeaders;
6040        use crate::common::serialization::ByteArraySerializer;
6041
6042        let metadata = create_metadata_with_topic(TOPIC, 1);
6043        let accumulator = create_accumulator();
6044        let producer: KafkaProducer<Vec<u8>, Vec<u8>> = KafkaProducer::with_options(
6045            KafkaProducerOptionsBuilder::new()
6046                .set_config(&ProducerConfig::defaults())
6047                .set_key_serializer(Box::new(ByteArraySerializer))
6048                .set_value_serializer(Box::new(ByteArraySerializer))
6049                .set_metadata(metadata)
6050                .set_accumulator(Arc::clone(&accumulator))
6051                .set_time(default_time())
6052                .set_pending_requests(Arc::new(Mutex::new(PendingRequests::new())))
6053                .build()
6054                .expect("KafkaProducerOptionsBuilder::build: every mandatory parameter is set above"),
6055        );
6056
6057        let mut headers = RecordHeaders::new();
6058        headers.add_with_key_value("trace", Some(b"abc")).unwrap();
6059        headers.add_with_key_value("empty", Some(b"")).unwrap();
6060        headers.add_with_key_value("null", None).unwrap();
6061        let record = ProducerRecord::with_partition_key_headers(
6062            TOPIC.to_string(),
6063            Some(0),
6064            Some(b"key".as_slice()),
6065            Some(b"value".as_slice()),
6066            headers,
6067        )
6068        .unwrap();
6069
6070        producer
6071            .send(record, None)
6072            .await
6073            .expect("the borrowed send should append the record");
6074        assert_eq!(
6075            accumulator.record_headers_for_test(&TopicPartition::new(TOPIC.to_string(), 0)),
6076            vec![vec![
6077                ("trace".to_string(), Some(b"abc".to_vec())),
6078                ("empty".to_string(), Some(Vec::new())),
6079                ("null".to_string(), None),
6080            ]]
6081        );
6082    }
6083
6084    /// Translated from `KafkaProducerTest.testFlushCompleteSendOfInflightBatches`.
6085    ///
6086    /// Tests that flush waits for all in-flight batches to complete. We send
6087    /// records, then simulate the sender completing them by aborting incomplete
6088    /// batches (which calls `done()` on all `ProduceRequestResult`s). Flush
6089    /// should then return immediately because all results are satisfied.
6090    #[tokio::test(flavor = "multi_thread")]
6091    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testFlushCompleteSendOfInflightBatches")]
6092    async fn test_flush_complete_send_of_inflight_batches() {
6093        let metadata = create_metadata_with_topic(TOPIC, 1);
6094        let accumulator = create_accumulator();
6095        let producer = create_producer(metadata, Arc::clone(&accumulator));
6096
6097        // Send multiple records
6098        let mut futures = Vec::new();
6099        for i in 0..5 {
6100            let record = ProducerRecord::new(TOPIC.to_string(), Some(format!("value{}", i)));
6101            let future = producer.send(record).await.unwrap();
6102            futures.push(future);
6103        }
6104
6105        // None should be done yet (no sender to complete them)
6106        for f in &futures {
6107            assert!(!f.is_done(), "Future should not be done before sender completes it");
6108        }
6109
6110        // Simulate the sender completing all batches by aborting them.
6111        // abort() calls complete_future_and_fire_callbacks which calls
6112        // produce_future.set() and produce_future.done(), unblocking flush.
6113        accumulator.abort_incomplete_batches().await;
6114
6115        // Now all futures should be done (abort marks ProduceRequestResults as done)
6116        for f in &futures {
6117            assert!(f.is_done(), "Future should be done after batch abort");
6118        }
6119
6120        // Flush should return immediately since all results are satisfied
6121        let result = producer.flush().await;
6122        assert!(result.is_ok(), "Flush should succeed after batches are completed");
6123    }
6124
6125    /// Translated from `KafkaProducerTest.testCloseWhenWaitingForMetadataUpdate`.
6126    ///
6127    /// Tests that closing the producer unblocks a send() that is waiting for
6128    /// metadata by verifying that after close(), the producer rejects new sends
6129    /// with IllegalState.
6130    #[tokio::test]
6131    async fn test_close_unblocks_pending_operations() {
6132        let metadata = create_metadata_with_topic(TOPIC, 1);
6133        let accumulator = create_accumulator();
6134        let producer = create_producer(metadata, accumulator);
6135
6136        // Close the producer
6137        producer.close().await.unwrap();
6138
6139        // Subsequent send should fail with IllegalState
6140        let record = ProducerRecord::new(TOPIC.to_string(), Some("value".to_string()));
6141        let result = producer.send(record).await;
6142        assert!(result.is_err());
6143        assert!(
6144            matches!(result.unwrap_err(), Error::LocalIllegalState(_)),
6145            "Expected IllegalState error after close"
6146        );
6147    }
6148
6149    /// Tests that initiate_close calls accumulator.close(), preventing new appends.
6150    ///
6151    /// Verifies Issue 2 fix: initiate_close must close the accumulator before
6152    /// setting running=false.
6153    #[tokio::test]
6154    async fn test_initiate_close_closes_accumulator() {
6155        let metadata = create_metadata_with_topic(TOPIC, 1);
6156        let accumulator = create_accumulator();
6157        let producer = create_producer(metadata, Arc::clone(&accumulator));
6158
6159        // Before close, we can send
6160        let record = ProducerRecord::new(TOPIC.to_string(), Some("value".to_string()));
6161        assert!(producer.send(record).await.is_ok());
6162
6163        // Initiate close
6164        producer.initiate_close();
6165
6166        // After initiate_close, the accumulator should be closed.
6167        // Attempting to send should fail because the accumulator rejects new appends.
6168        let record2 = ProducerRecord::new(TOPIC.to_string(), Some("value2".to_string()));
6169        // The error will come from ensure_not_closed since running=false
6170        let result = producer.send(record2).await;
6171        assert!(result.is_err(), "Send should fail after initiate_close");
6172    }
6173
6174    /// Tests that the negative timeout check is gone since Rust Duration is unsigned.
6175    ///
6176    /// Documents Issue 7 resolution: Rust's Duration cannot be negative,
6177    /// so the negative timeout check from Java is correctly omitted.
6178    #[test]
6179    fn test_duration_cannot_be_negative() {
6180        // This test documents that Rust's std::time::Duration cannot represent
6181        // negative values, so the Java test `closeWithNegativeTimestampShouldThrow`
6182        // is not applicable. Duration::ZERO is the smallest possible value.
6183        let zero = Duration::ZERO;
6184        assert_eq!(0, zero.as_millis());
6185        // There is no way to construct a negative Duration in Rust.
6186        // Duration::from_millis(u64) always produces a non-negative value.
6187    }
6188
6189    /// Tests that close with timeout 0 force-closes (no graceful drain).
6190    ///
6191    /// Verifies the force-close path in close_with_timeout when timeout is zero.
6192    #[tokio::test]
6193    async fn test_close_timeout_zero_force_closes() {
6194        let metadata = create_metadata_with_topic(TOPIC, 1);
6195        let accumulator = create_accumulator();
6196        let producer = create_producer(metadata, Arc::clone(&accumulator));
6197
6198        // Send a record
6199        let record = ProducerRecord::new(TOPIC.to_string(), Some("value".to_string()));
6200        let _ = producer.send(record).await;
6201        assert!(accumulator.has_undrained());
6202
6203        // Close with timeout 0 should force-close
6204        let result = producer.close_with_timeout(Duration::ZERO).await;
6205        assert!(result.is_ok());
6206
6207        // After force-close, the producer should not accept new sends
6208        let record2 = ProducerRecord::new(TOPIC.to_string(), Some("value2".to_string()));
6209        assert!(producer.send(record2).await.is_err());
6210    }
6211
6212    /// Regression test with no Java counterpart: a timeout whose length in
6213    /// milliseconds exceeds `i64::MAX` must result in a graceful close, as `close()`
6214    /// does.
6215    ///
6216    /// Truncating these values with `as i64` yields -1 ms for `Duration::MAX` and
6217    /// `Duration::from_millis(u64::MAX)`, -1000 ms for `Duration::from_secs(u64::MAX)`,
6218    /// and 0 ms for `Duration::from_secs(1 << 61)` (125 × 2^64 ms). A negative value
6219    /// matches neither branch of `close_with_timeout` and leaves the producer running;
6220    /// 0 forces the close instead of waiting. Java cannot reach either state:
6221    /// `Duration.toMillis()` throws on overflow and `close` rejects a negative timeout
6222    /// (`KafkaProducer#close(Duration)`).
6223    #[tokio::test]
6224    async fn test_close_with_timeout_beyond_i64_millis_closes_gracefully() {
6225        for timeout in [
6226            Duration::MAX,
6227            Duration::from_millis(u64::MAX),
6228            Duration::from_secs(u64::MAX),
6229            Duration::from_secs(1 << 61),
6230        ] {
6231            let metadata = create_metadata_with_topic(TOPIC, 1);
6232            let producer = create_producer(metadata, create_accumulator());
6233
6234            producer.close_with_timeout(timeout).await.expect("close");
6235
6236            assert!(
6237                !producer.running.load(Ordering::Acquire),
6238                "{:?}: close must stop the producer",
6239                timeout
6240            );
6241            assert!(
6242                !producer.force_close.load(Ordering::Acquire),
6243                "{:?}: close must be graceful, not forced",
6244                timeout
6245            );
6246            let record = ProducerRecord::new(TOPIC.to_string(), Some("value".to_string()));
6247            let error = producer.send(record).await.expect_err("a closed producer rejects sends");
6248            assert_eq!(
6249                error.message(),
6250                "Cannot perform operation after producer has been closed",
6251                "{:?}",
6252                timeout
6253            );
6254        }
6255    }
6256
6257    /// Tests that the callback is invoked with error on invalid topic.
6258    ///
6259    /// Translated from `KafkaProducerTest.testCallbackAndInterceptorHandleError`
6260    /// (the invalid topic variant).
6261    #[tokio::test]
6262    async fn test_callback_invoked_on_invalid_topic() {
6263        use crate::MetadataResponseData;
6264        use crate::common::protocol::ApiKeys;
6265        use crate::common::protocol::Errors;
6266        use crate::common::requests::MetadataResponse;
6267        use crate::metadata_response_data::{MetadataResponseBroker, MetadataResponseTopic};
6268
6269        let metadata = Arc::new(ProducerMetadata::new(
6270            100,
6271            1000,
6272            300_000,
6273            300_000,
6274            ClusterResourceListeners::new(),
6275        ));
6276
6277        // Create metadata with an invalid topic
6278        let invalid_topic = "topic with spaces";
6279        let mut data = MetadataResponseData::new();
6280        data.set_controller_id(0);
6281
6282        let mut broker = MetadataResponseBroker::new();
6283        broker.set_node_id(0);
6284        broker.set_host("localhost".to_string());
6285        broker.set_port(9092);
6286        data.set_brokers(vec![broker]);
6287
6288        let mut topic_resp = MetadataResponseTopic::new();
6289        topic_resp.set_name(Some(invalid_topic.to_string()));
6290        topic_resp.set_error_code(Errors::InvalidTopicError.code());
6291        topic_resp.set_is_internal(false);
6292        data.set_topics(vec![topic_resp]);
6293
6294        let response = MetadataResponse::with_version(data, ApiKeys::METADATA.latest_version());
6295        metadata.add(invalid_topic, 0);
6296        metadata.update_with_current_request_version(&response, false, 0);
6297
6298        let accumulator = create_accumulator();
6299        let producer = create_producer(metadata, accumulator);
6300
6301        let callback_invoked = Arc::new(AtomicBool::new(false));
6302        let got_error = Arc::new(AtomicBool::new(false));
6303        let inv = Arc::clone(&callback_invoked);
6304        let err = Arc::clone(&got_error);
6305
6306        let callback: Box<dyn Callback> = Box::new(move |_metadata: Option<&RecordMetadata>, error: Option<&Error>| {
6307            inv.store(true, Ordering::SeqCst);
6308            err.store(error.is_some(), Ordering::SeqCst);
6309        });
6310
6311        let record = ProducerRecord::new(invalid_topic.to_string(), Some("value".to_string()));
6312        let result = producer.send_with_callback(record, Some(callback)).await;
6313
6314        // Should return a failed future, not propagate the error
6315        assert!(result.is_ok(), "send() should return Ok with a failed future for InvalidTopic");
6316        let future = result.unwrap();
6317        assert!(future.is_done());
6318
6319        // Verify callback was invoked with the error
6320        assert!(callback_invoked.load(Ordering::SeqCst), "Callback should have been invoked");
6321        assert!(got_error.load(Ordering::SeqCst), "Callback should receive error");
6322
6323        // Verify the future contains the error
6324        let err = future.get().await.unwrap_err();
6325        assert!(
6326            matches!(err, Error::InvalidTopic(_)),
6327            "Expected InvalidTopic error, got: {:?}",
6328            err
6329        );
6330        // Java's `new InvalidTopicException(topic)` sets the message to the topic
6331        // name and leaves `invalidTopics()` empty (the `(String message)` ctor).
6332        assert_eq!(err.message(), "topic with spaces");
6333        match &err {
6334            Error::InvalidTopic(e) => assert!(e.invalid_topics().is_empty()),
6335            _ => unreachable!(),
6336        }
6337    }
6338
6339    /// Tests that multiple calls to close are safe even with timeout.
6340    ///
6341    /// Translated from `KafkaProducerTest.closeShouldBeIdempotent`.
6342    #[tokio::test]
6343    async fn test_close_with_timeout_idempotent() {
6344        let metadata = create_metadata_with_topic(TOPIC, 1);
6345        let accumulator = create_accumulator();
6346        let producer = create_producer(metadata, accumulator);
6347
6348        producer.close_with_timeout(Duration::from_secs(1)).await.unwrap();
6349        producer.close_with_timeout(Duration::from_secs(1)).await.unwrap();
6350        producer.close_with_timeout(Duration::ZERO).await.unwrap();
6351    }
6352
6353    /// Tests that different keys produce different partition assignments.
6354    ///
6355    /// Translated from partition-related KafkaProducerTest tests.
6356    #[test]
6357    fn test_different_keys_may_produce_different_partitions() {
6358        let metadata = create_metadata_with_topic(TOPIC, 100);
6359        let accumulator = create_accumulator();
6360        let producer = create_producer(metadata.clone(), accumulator);
6361
6362        let cluster = metadata.fetch();
6363        let mut partitions = std::collections::HashSet::new();
6364        for i in 0..20 {
6365            let key = format!("key-{}", i);
6366            let record = ProducerRecord::with_key(TOPIC.to_string(), Some(key.clone()), Some("v".to_string()));
6367            let p = producer.partition(&record, Some(key.as_bytes()), Some(b"v"), &cluster);
6368            partitions.insert(p);
6369        }
6370        // With 100 partitions and 20 different keys, we should get at least 2 different partitions
6371        assert!(
6372            partitions.len() >= 2,
6373            "Expected multiple partitions for different keys, got: {:?}",
6374            partitions
6375        );
6376    }
6377
6378    /// `definition-of-done.md` §10 / CLAUDE.md §13: enabling idempotence must not add
6379    /// a single per-record heap allocation to the send path.
6380    ///
6381    /// This is the audit clause aimed at the public `send` entry point.
6382    /// `RecordAccumulator::drain` has its own budget test
6383    /// (`test_drain_allocations_do_not_scale_with_the_record_count`), but that
6384    /// measures the wrong layer: Critic 44 issue 1 was a `String` + `Arc<str>`
6385    /// allocation per record in `KafkaProducer::do_send_bytes`, one call *above* it.
6386    ///
6387    /// Measured as a delta between a producer that holds a `TransactionManager` and
6388    /// one that does not, over a steady-state append (batch already created, topic
6389    /// info already interned). Pinning an absolute count would break on unrelated
6390    /// refactors; the delta is exactly the cost the transaction wiring adds, and it
6391    /// must be zero.
6392    #[tokio::test]
6393    async fn test_send_allocations_do_not_grow_when_idempotence_is_enabled() {
6394        async fn steady_state_send_allocations(with_transaction_manager: bool) -> usize {
6395            let metadata = create_metadata_with_topic(TOPIC, 1);
6396            let transaction_manager = if with_transaction_manager {
6397                let manager = TransactionManager::new(
6398                    LogContext::empty(),
6399                    None,
6400                    60_000,
6401                    100,
6402                    Arc::new(ApiVersions::new()),
6403                    metadata.metadata_arc(),
6404                    false,
6405                );
6406                Some(Arc::new(Mutex::new(manager)))
6407            } else {
6408                None
6409            };
6410            // A large batch so every send below appends to the same batch.
6411            let accumulator = Arc::new(RecordAccumulator::new_for_test(
6412                1024 * 1024,
6413                Compression::none().build(),
6414                5,
6415                100,
6416                1000,
6417                120_000,
6418                PartitionerConfig::new(true, 0, false, "").unwrap(),
6419                default_time(),
6420                Arc::new(BufferPool::new_for_test(32 * 1024 * 1024, 1024 * 1024)),
6421                transaction_manager.clone(),
6422            ));
6423            let config = ProducerConfig::defaults();
6424            let producer = KafkaProducer::<String, String>::with_options(
6425                KafkaProducerOptionsBuilder::new()
6426                    .set_config(&config)
6427                    .set_key_serializer(Box::new(StringSerializer))
6428                    .set_value_serializer(Box::new(StringSerializer))
6429                    .set_metadata(Arc::clone(&metadata))
6430                    .set_accumulator(accumulator)
6431                    .set_time(default_time())
6432                    .set_transaction_manager(transaction_manager)
6433                    .set_pending_requests(Arc::new(Mutex::new(PendingRequests::new())))
6434                    .build()
6435                    .expect("KafkaProducerOptionsBuilder::build: every mandatory parameter is set above"),
6436            );
6437            let cluster = metadata.fetch();
6438
6439            // Warm up: create the topic info, the deque and the batch.
6440            for _ in 0..4 {
6441                producer
6442                    .do_send_bytes(
6443                        TOPIC,
6444                        Some(0),
6445                        Some(0),
6446                        Some(b"k"),
6447                        Some(b"v"),
6448                        &[],
6449                        None,
6450                        0,
6451                        0,
6452                        &cluster,
6453                        SendWait::MaxBlock,
6454                    )
6455                    .await
6456                    .expect("append should succeed");
6457            }
6458
6459            {
6460                let _guard = crate::AllocTrackingGuard::new();
6461                crate::AllocTrackingGuard::reset();
6462                producer
6463                    .do_send_bytes(
6464                        TOPIC,
6465                        Some(0),
6466                        Some(0),
6467                        Some(b"k"),
6468                        Some(b"v"),
6469                        &[],
6470                        None,
6471                        0,
6472                        0,
6473                        &cluster,
6474                        SendWait::MaxBlock,
6475                    )
6476                    .await
6477                    .expect("append should succeed");
6478                let count = crate::AllocTrackingGuard::count();
6479                assert!(count > 0, "the tracker must actually be measuring");
6480                count
6481            }
6482        }
6483
6484        let without = steady_state_send_allocations(false).await;
6485        let with = steady_state_send_allocations(true).await;
6486        assert_eq!(
6487            without, with,
6488            "enabling idempotence must add no per-record allocation to the send path; \
6489             got {without} without a TransactionManager vs {with} with one"
6490        );
6491    }
6492
6493    /// `definition-of-done.md` §10 / CLAUDE.md §13: dispatching through a custom
6494    /// `Partitioner<K, V>` must not add a single per-record heap allocation to the
6495    /// send path, matching the idempotence audit above
6496    /// (`test_send_allocations_do_not_grow_when_idempotence_is_enabled`).
6497    ///
6498    /// `partition: None` is passed to `do_send_bytes` (rather than the pre-resolved
6499    /// `Some(0)` the idempotence test uses) so `compute_partition` actually runs and,
6500    /// when a partitioner is configured, calls into it — otherwise this would measure
6501    /// nothing, since a pre-resolved partition short-circuits `compute_partition`
6502    /// before the partitioner is ever consulted.
6503    ///
6504    /// `RoundRobinPartitioner` is the concrete partitioner under measurement: its
6505    /// steady-state path is a `DashMap::get` on an already-present topic entry plus an
6506    /// `AtomicI32` increment (CLAUDE.md §13), so the delta versus no partitioner
6507    /// (built-in key-hash partitioning) must be zero.
6508    #[tokio::test]
6509    async fn test_send_allocations_do_not_grow_with_a_custom_partitioner() {
6510        async fn steady_state_send_allocations(with_partitioner: bool) -> usize {
6511            let partitioner: Option<Box<dyn Partitioner<String, String>>> = if with_partitioner {
6512                Some(Box::new(RoundRobinPartitioner::new()))
6513            } else {
6514                None
6515            };
6516            let metadata = create_metadata_with_topic(TOPIC, 1);
6517            // A large batch so every send below appends to the same batch.
6518            let accumulator = Arc::new(RecordAccumulator::new_for_test(
6519                1024 * 1024,
6520                Compression::none().build(),
6521                5,
6522                100,
6523                1000,
6524                120_000,
6525                PartitionerConfig::new(true, 0, false, "").unwrap(),
6526                default_time(),
6527                Arc::new(BufferPool::new_for_test(32 * 1024 * 1024, 1024 * 1024)),
6528                None,
6529            ));
6530            let config = ProducerConfig::defaults();
6531            let producer = KafkaProducer::<String, String>::with_options(
6532                KafkaProducerOptionsBuilder::new()
6533                    .set_config(&config)
6534                    .set_key_serializer(Box::new(StringSerializer))
6535                    .set_value_serializer(Box::new(StringSerializer))
6536                    .set_metadata(Arc::clone(&metadata))
6537                    .set_accumulator(accumulator)
6538                    .set_time(default_time())
6539                    .set_pending_requests(Arc::new(Mutex::new(PendingRequests::new())))
6540                    .set_partitioner(partitioner)
6541                    .build()
6542                    .expect("KafkaProducerOptionsBuilder::build: every mandatory parameter is set above"),
6543            );
6544            let cluster = metadata.fetch();
6545
6546            // Warm up: create the topic info, the deque and the batch, and (for the
6547            // partitioner case) the RoundRobinPartitioner's per-topic counter entry.
6548            for _ in 0..4 {
6549                producer
6550                    .do_send_bytes(
6551                        TOPIC,
6552                        None,
6553                        Some(0),
6554                        Some(b"k"),
6555                        Some(b"v"),
6556                        &[],
6557                        None,
6558                        0,
6559                        0,
6560                        &cluster,
6561                        SendWait::MaxBlock,
6562                    )
6563                    .await
6564                    .expect("append should succeed");
6565            }
6566
6567            {
6568                let _guard = crate::AllocTrackingGuard::new();
6569                crate::AllocTrackingGuard::reset();
6570                producer
6571                    .do_send_bytes(
6572                        TOPIC,
6573                        None,
6574                        Some(0),
6575                        Some(b"k"),
6576                        Some(b"v"),
6577                        &[],
6578                        None,
6579                        0,
6580                        0,
6581                        &cluster,
6582                        SendWait::MaxBlock,
6583                    )
6584                    .await
6585                    .expect("append should succeed");
6586                let count = crate::AllocTrackingGuard::count();
6587                assert!(count > 0, "the tracker must actually be measuring");
6588                count
6589            }
6590        }
6591
6592        let without = steady_state_send_allocations(false).await;
6593        let with = steady_state_send_allocations(true).await;
6594        assert_eq!(
6595            without, with,
6596            "dispatching through a custom Partitioner must add no per-record allocation \
6597             to the send path; got {without} without a partitioner vs {with} with one"
6598        );
6599    }
6600
6601    /// CLAUDE.md §13 / §14: with a custom partitioner, a `Vec<u8>` record goes
6602    /// through the serializers (the partitioner needs the typed key and value)
6603    /// and still allocates nothing for them: `ByteArraySerializer` borrows the
6604    /// record's own buffers, and its future fits the `InlineFuture` buffer. So
6605    /// the whole `send` costs exactly what appending the already-serialized
6606    /// bytes costs.
6607    #[tokio::test]
6608    async fn test_vec_send_with_a_custom_partitioner_allocates_nothing_for_the_serializers() {
6609        use crate::common::serialization::ByteArraySerializer;
6610
6611        let metadata = create_metadata_with_topic(TOPIC, 1);
6612        let accumulator = Arc::new(RecordAccumulator::new_for_test(
6613            1024 * 1024,
6614            Compression::none().build(),
6615            5,
6616            100,
6617            1000,
6618            120_000,
6619            PartitionerConfig::new(true, 0, false, "").unwrap(),
6620            default_time(),
6621            Arc::new(BufferPool::new_for_test(32 * 1024 * 1024, 1024 * 1024)),
6622            None,
6623        ));
6624        let config = ProducerConfig::defaults();
6625        let partitioner: Option<Box<dyn Partitioner<Vec<u8>, Vec<u8>>>> = Some(Box::new(RoundRobinPartitioner::new()));
6626        let producer = KafkaProducer::<Vec<u8>, Vec<u8>>::with_options(
6627            KafkaProducerOptionsBuilder::new()
6628                .set_config(&config)
6629                .set_key_serializer(Box::new(ByteArraySerializer))
6630                .set_value_serializer(Box::new(ByteArraySerializer))
6631                .set_metadata(Arc::clone(&metadata))
6632                .set_accumulator(accumulator)
6633                .set_time(default_time())
6634                .set_pending_requests(Arc::new(Mutex::new(PendingRequests::new())))
6635                .set_partitioner(partitioner)
6636                .build()
6637                .expect("KafkaProducerOptionsBuilder::build: every mandatory parameter is set above"),
6638        );
6639        let cluster = metadata.fetch();
6640        let record = || ProducerRecord::with_key(TOPIC.to_string(), Some(b"k".to_vec()), Some(b"v".to_vec()));
6641
6642        let send_bytes = || {
6643            producer.do_send_bytes(
6644                TOPIC,
6645                None,
6646                None,
6647                Some(b"k"),
6648                Some(b"v"),
6649                &[],
6650                None,
6651                0,
6652                0,
6653                &cluster,
6654                SendWait::MaxBlock,
6655            )
6656        };
6657
6658        // Warm up both paths: the topic info, the deque, the batch and the
6659        // partitioner's per-topic counter. Ten records, so the two measured
6660        // below (the 11th and 12th) do not land on a doubling of the batch's
6661        // per-record vectors, which would charge an amortized growth to one
6662        // side only.
6663        for _ in 0..5 {
6664            producer
6665                .do_send(record(), None, SendWait::MaxBlock)
6666                .await
6667                .expect("append should succeed");
6668            send_bytes().await.expect("append should succeed");
6669        }
6670
6671        let through_serializers = {
6672            let record = record();
6673            let _guard = crate::AllocTrackingGuard::new();
6674            crate::AllocTrackingGuard::reset();
6675            producer
6676                .do_send(record, None, SendWait::MaxBlock)
6677                .await
6678                .expect("append should succeed");
6679            crate::AllocTrackingGuard::count()
6680        };
6681        let bytes_only = {
6682            let _guard = crate::AllocTrackingGuard::new();
6683            crate::AllocTrackingGuard::reset();
6684            send_bytes().await.expect("append should succeed");
6685            crate::AllocTrackingGuard::count()
6686        };
6687        // `do_send` waits on the metadata before serializing, as Java's
6688        // `doSend` does; that step's own cost is not the serializers'.
6689        let metadata_wait = {
6690            let _guard = crate::AllocTrackingGuard::new();
6691            crate::AllocTrackingGuard::reset();
6692            producer
6693                .wait_on_metadata(TOPIC, None, 0, 0)
6694                .await
6695                .expect("the metadata is cached");
6696            crate::AllocTrackingGuard::count()
6697        };
6698        assert!(bytes_only > 0, "the tracker must actually be measuring");
6699        assert_eq!(
6700            bytes_only + metadata_wait,
6701            through_serializers,
6702            "serializing a Vec<u8> record must add no allocation to the send path; got \
6703             {through_serializers} through the serializers vs {bytes_only} for the bytes alone \
6704             plus {metadata_wait} for the metadata wait"
6705        );
6706    }
6707
6708    // =====================================================================
6709    // `KafkaProducerTest` transactional harness (Milestone 11, Phase 6)
6710    //
6711    // Java's `kafkaProducer(configs, keySer, valSer, metadata, client, interceptors,
6712    // time)` helper (`KafkaProducerTest#kafkaProducer`) builds a real `KafkaProducer`
6713    // over a `MockClient` and lets the constructor start a real Sender **thread**;
6714    // the test thread then pokes the `MockClient` beside it, relying on Java's
6715    // `MockClient` being internally synchronized.
6716    //
6717    // Rust cannot copy that directly: `Sender` owns its `C: KafkaClient` by value and
6718    // `MockClient` is a plain struct, so a `tokio::spawn`ed Sender would take the mock
6719    // with it and the test could no longer prepare responses or inspect requests. So
6720    // the `Sender` stays test-owned — exactly as `SenderTest` keeps it — and the
6721    // application call runs *concurrently with* a driver loop on the same task (see
6722    // [`drive`]). Everything the two share is shared the way production shares it: one
6723    // `TransactionManager`, one `PendingRequests`, one `RecordAccumulator`, one
6724    // `ProducerMetadata`, one `running` / `force_close` flag pair.
6725    // =====================================================================
6726
6727    /// Java's `NODE`, used as the coordinator in every
6728    /// `FindCoordinator` response below.
6729    fn coordinator_node() -> Node {
6730        Node::new(0, "host1".to_string(), 1000)
6731    }
6732
6733    /// The topic id [`TOPIC`] is published with, so a produce response can name the
6734    /// same one the request carried.
6735    fn topic_id() -> crate::common::Uuid {
6736        crate::common::Uuid::from_string("MKXx1fIkQy2J9jXHhK8m1w").expect("valid UUID")
6737    }
6738
6739    /// Publishes node `"0"`'s API versions with `transaction.version` finalized at
6740    /// `level`, which is what `maybeUpdateTransactionV2Enabled`
6741    /// (`TransactionManager#maybeUpdateTransactionV2Enabled`) reads.
6742    fn seed_transaction_version(api_versions: &Arc<ApiVersions>, level: i16) {
6743        use crate::NodeApiVersions;
6744        use crate::api_versions_response_data::{FinalizedFeatureKey, SupportedFeatureKey};
6745
6746        const FEATURE: &str = "transaction.version";
6747
6748        let mut supported = SupportedFeatureKey::new();
6749        supported.set_name(FEATURE.to_string());
6750        supported.set_max_version(level);
6751        supported.set_min_version(0);
6752
6753        let mut finalized = FinalizedFeatureKey::new();
6754        finalized.set_name(FEATURE.to_string());
6755        finalized.set_max_version_level(level);
6756        finalized.set_min_version_level(level);
6757
6758        api_versions.update(
6759            "0",
6760            NodeApiVersions::with_node_finalized_features_finalized_features_epoch(&[], &[supported], &[finalized], 0),
6761        );
6762    }
6763
6764    /// Java's `new MockTime()` started at `initial_ms`, with a frozen monotonic clock.
6765    fn mock_time(initial_ms: i64) -> Arc<MockTime> {
6766        Arc::new(MockTime::with_auto_tick_ms_current_time_ms_current_high_res_time_ns(
6767            0, initial_ms, 0,
6768        ))
6769    }
6770
6771    /// `RequestTestUtils.metadataUpdateWithIds(1, Map.of(topic, n), Map.of(topic, id))`,
6772    /// or `metadataUpdateWith(1, Map.of(topic, n))` (no ids) when `with_topic_ids` is
6773    /// `false`.
6774    fn topic_metadata_update(num_partitions: i32, with_topic_ids: bool) -> crate::common::requests::MetadataResponse {
6775        let topic_ids = if with_topic_ids {
6776            HashMap::from([(TOPIC.to_string(), topic_id())])
6777        } else {
6778            HashMap::new()
6779        };
6780        crate::common::requests::RequestTestUtils::metadata_update_with_ids(
6781            "kafka-cluster",
6782            1,
6783            &HashMap::new(),
6784            &HashMap::from([(TOPIC.to_string(), num_partitions)]),
6785            &|_| None,
6786            &topic_ids,
6787        )
6788    }
6789
6790    /// A `KafkaProducer` and the `Sender` that serves it, sharing every piece of
6791    /// state production shares.
6792    struct TxnProducerContext {
6793        producer: KafkaProducer<String, String>,
6794        sender: Sender<MockClient>,
6795        accumulator: Arc<RecordAccumulator>,
6796        metadata: Arc<ProducerMetadata>,
6797        transaction_manager: Arc<Mutex<TransactionManager>>,
6798        time: Arc<MockTime>,
6799    }
6800
6801    impl TxnProducerContext {
6802        /// Builds the context from `bootstrap.servers` plus `extra` configuration,
6803        /// mirroring the `configs` map each Java test assembles.
6804        ///
6805        /// `num_partitions` seeds the metadata for [`TOPIC`], standing in for Java's
6806        /// `RequestTestUtils.metadataUpdateWith(1, singletonMap("topic", 1))`.
6807        fn new(extra: &[(&str, &str)], num_partitions: i32) -> Self {
6808            Self::with_options(extra, num_partitions, false)
6809        }
6810
6811        /// As [`Self::new`], but with `transaction.version` finalized at level 2 so the
6812        /// manager enables KIP-890 Transaction V2 in `initTransactions`.
6813        ///
6814        /// Java seeds the same thing through `client.setNodeApiVersions(..)` plus
6815        /// `apiVersions.update(NODE.idString(), nodeApiVersions)`; only the second half
6816        /// is load-bearing, because `TransactionManager` reads `apiVersions` and never
6817        /// the client.
6818        fn transactional_v2(extra: &[(&str, &str)], num_partitions: i32) -> Self {
6819            Self::with_options(extra, num_partitions, true)
6820        }
6821
6822        fn with_options(extra: &[(&str, &str)], num_partitions: i32, transaction_v2: bool) -> Self {
6823            Self::with_metadata_seed(extra, num_partitions, transaction_v2, true, true)
6824        }
6825
6826        /// As [`Self::with_options`], choosing how the metadata starts out: whether
6827        /// [`TOPIC`] is already in the producer's tracked topic set (`metadata.add`),
6828        /// and whether the seeded snapshot carries its topic id. The KIP-1319 tests
6829        /// use an untracked topic, as Java's do, so `sendOffsetsToTransaction` has to
6830        /// refresh the metadata before it delegates.
6831        fn with_metadata_seed(
6832            extra: &[(&str, &str)],
6833            num_partitions: i32,
6834            transaction_v2: bool,
6835            track_topic: bool,
6836            with_topic_ids: bool,
6837        ) -> Self {
6838            let mut props = HashMap::from([("bootstrap.servers".to_string(), "localhost:9000".to_string())]);
6839            for (key, value) in extra {
6840                props.insert((*key).to_string(), (*value).to_string());
6841            }
6842            let config = ProducerConfig::new(&props).expect("valid config");
6843            let log_context = LogContext::new(format!("[Producer clientId={}] ", config.client_id));
6844            let time = mock_time(1_000);
6845            let api_versions = Arc::new(ApiVersions::new());
6846            if transaction_v2 {
6847                seed_transaction_version(&api_versions, 2);
6848            }
6849
6850            // The metadata comes first: the transaction manager reads its topic ids
6851            // (KIP-1319), as Java's constructor now builds it ahead of the manager.
6852            let metadata = Arc::new(ProducerMetadata::with_log_context(
6853                config.retry_backoff_ms,
6854                config.retry_backoff_max_ms,
6855                config.metadata_max_age_ms,
6856                config.metadata_max_idle_ms,
6857                ClusterResourceListeners::new(),
6858                log_context.clone(),
6859            ));
6860            let transaction_manager = KafkaProducer::<String, String>::configure_transaction_state(
6861                &config,
6862                &api_versions,
6863                &metadata.metadata_arc(),
6864                &log_context,
6865            )
6866            .expect("these tests always enable idempotence");
6867            let pending_requests = Arc::new(Mutex::new(PendingRequests::new()));
6868
6869            if track_topic {
6870                metadata.add(TOPIC, time.milliseconds());
6871            }
6872            // `metadata_update_with_ids` rather than `metadata_update_with`: the produce
6873            // path stamps the topic id from metadata onto the request, so a response has
6874            // to carry the same one to be matched back to its batch.
6875            let update = topic_metadata_update(num_partitions, with_topic_ids);
6876            metadata.update_with_current_request_version(&update, false, time.milliseconds());
6877
6878            let batch_size = config.batch_size.max(1);
6879            let metrics = Arc::new(Metrics::new());
6880            let buffer_pool = Arc::new(BufferPool::new(
6881                config.buffer_memory,
6882                batch_size as usize,
6883                Arc::clone(&metrics),
6884                Arc::clone(&time) as Arc<dyn Time>,
6885                KafkaProducer::<String, String>::PRODUCER_METRIC_GROUP_NAME,
6886            ));
6887            let accumulator = Arc::new(RecordAccumulator::with_log_context(
6888                batch_size,
6889                Compression::of(config.compression_type).build(),
6890                KafkaProducer::<String, String>::linger_ms(&config),
6891                config.retry_backoff_ms,
6892                config.retry_backoff_max_ms,
6893                config.delivery_timeout_ms,
6894                PartitionerConfig::new(
6895                    config.partitioner_adaptive_partitioning_enable,
6896                    config.partitioner_availability_timeout_ms,
6897                    config.partitioner_rack_aware,
6898                    &config.client_rack,
6899                )
6900                .unwrap(),
6901                Arc::clone(&metrics),
6902                KafkaProducer::<String, String>::PRODUCER_METRIC_GROUP_NAME,
6903                Arc::clone(&time) as Arc<dyn Time>,
6904                buffer_pool,
6905                Some(Arc::clone(&transaction_manager)),
6906                log_context.clone(),
6907            ));
6908
6909            let client = MockClient::with_static_nodes(vec![coordinator_node()], Arc::clone(&time) as Arc<dyn Time>);
6910            let wakeup = client.wakeup_notify();
6911            let running = Arc::new(AtomicBool::new(true));
6912            let force_close = Arc::new(AtomicBool::new(false));
6913
6914            let sender = Sender::new(
6915                client,
6916                Arc::clone(&metadata),
6917                Arc::clone(&accumulator),
6918                config.max_in_flight_requests_per_connection == 1,
6919                config.max_request_size,
6920                config.acks,
6921                config.retries,
6922                config.request_timeout_ms,
6923                config.retry_backoff_ms,
6924                SenderMetricsRegistry::new(Arc::clone(&metrics)),
6925                Arc::clone(&running),
6926                Arc::clone(&force_close),
6927                Arc::clone(&time) as Arc<dyn Time>,
6928                Some(Arc::clone(&transaction_manager)),
6929                Arc::clone(&pending_requests),
6930                log_context.clone(),
6931            );
6932
6933            let producer = KafkaProducer::with_options(
6934                KafkaProducerOptionsBuilder::new()
6935                    .set_config(&config)
6936                    .set_key_serializer(Box::new(StringSerializer))
6937                    .set_value_serializer(Box::new(StringSerializer))
6938                    .set_metadata(Arc::clone(&metadata))
6939                    .set_accumulator(Arc::clone(&accumulator))
6940                    .set_running(running)
6941                    .set_force_close(force_close)
6942                    .set_wakeup(wakeup)
6943                    .set_time(Arc::clone(&time) as Arc<dyn Time>)
6944                    .set_transaction_manager(Some(Arc::clone(&transaction_manager)))
6945                    .set_pending_requests(pending_requests)
6946                    .build()
6947                    .expect("KafkaProducerOptionsBuilder::build: every mandatory parameter is set above"),
6948            );
6949
6950            Self { producer, sender, accumulator, metadata, transaction_manager, time }
6951        }
6952
6953        /// The default transactional setup: `transactional.id=some.id`, one topic
6954        /// partition.
6955        fn transactional() -> Self {
6956            Self::new(&[("transactional.id", TRANSACTIONAL_ID)], 1)
6957        }
6958
6959        /// Drives `Sender::run_once` `iterations` times with no application call in
6960        /// flight, for the assertions Java makes with a bare `sender.runOnce()`.
6961        async fn run_sender(&mut self, iterations: usize) {
6962            for _ in 0..iterations {
6963                run_once(&mut self.sender).await;
6964            }
6965        }
6966
6967        /// Queues the `FindCoordinator` + `InitProducerId` pair every
6968        /// `initTransactions` needs, in the order the `Sender` sends them.
6969        fn prepare_init_transactions(&mut self, error: Errors, producer_id: i64, epoch: i16) {
6970            let node = coordinator_node();
6971            self.sender
6972                .client_mut()
6973                .prepare_response(find_coordinator_response(Errors::None, TRANSACTIONAL_ID, &node));
6974            self.sender
6975                .client_mut()
6976                .prepare_response(init_producer_id_response(error, producer_id, epoch));
6977        }
6978
6979        /// The producer id and epoch the manager currently holds.
6980        fn producer_id_and_epoch(&self) -> (i64, i16) {
6981            let manager = self.transaction_manager.lock().unwrap();
6982            let id_and_epoch = manager.producer_id_and_epoch();
6983            (id_and_epoch.producer_id, id_and_epoch.epoch)
6984        }
6985    }
6986
6987    /// `Sender.runOnce()` with Java's catch-and-log
6988    /// (`Sender`) rather than a propagating `?`.
6989    async fn run_once(sender: &mut Sender<MockClient>) {
6990        if let Err(error) = sender.run_once().await {
6991            eprintln!("run_once: {}", error);
6992        }
6993    }
6994
6995    /// Runs `op` — a call on the application task — while driving the `Sender` the way
6996    /// Java's spawned I/O thread would.
6997    ///
6998    /// `tokio::join!` rather than `tokio::select!`: `select!` drops the losing future
6999    /// and its side effects (CLAUDE.md §11.6.1), which here would abandon a half-sent
7000    /// transactional request. `join!` polls both to completion and never drops either.
7001    ///
7002    /// The loop stops as soon as `op` resolves. It `yield_now()`s rather than sleeps:
7003    /// `MockClient::poll` returns immediately, so the yield is what lets the runtime
7004    /// run `op` and fire its `max.block.ms` timer, and it is also what keeps the
7005    /// injected `MockTime` advancing (each `run_once` reads the clock several times, so
7006    /// with `set_auto_tick_ms` the loop is the only thing that moves time — a Java
7007    /// `MockTime(1)` plus a hot Sender thread behaves the same way).
7008    ///
7009    /// A free function rather than a method on [`TxnProducerContext`] so callers can
7010    /// pass `&mut ctx.sender` and a future borrowing `&ctx.producer` at the same time.
7011    async fn drive<T>(sender: &mut Sender<MockClient>, op: impl std::future::Future<Output = T>) -> T {
7012        // A wall-clock deadline, NOT an iteration count. `drive`'s ops wait on
7013        // `await_result_timeout` (a real `tokio::time::timeout` on `max.block.ms`), so a
7014        // test that legitimately waits a timeout out busy-spins this loop —
7015        // `run_once` + `yield_now`, neither of which sleeps — for the whole real-time
7016        // window. That is an unbounded, machine-speed-dependent number of iterations,
7017        // especially when MockTime is frozen (no `set_auto_tick_ms`, as in
7018        // `test_init_transactions_response_after_timeout`). An iteration cap could not be
7019        // sized to let those tests pass *and* fail a genuine hang fast; a wall-clock
7020        // budget can. It comfortably exceeds the longest `max.block.ms` any drive-based
7021        // test configures while still surfacing a stuck regression, mirroring the intent
7022        // of `producer_test_utils::MAX_TRIES` (fail fast, do not hang) with the primitive
7023        // that fits `drive`'s real-time waits — an iteration count fits `run_until` there
7024        // only because it drives the Sender alone, with no concurrent real-time timeout.
7025        const DRIVE_BUDGET: std::time::Duration = std::time::Duration::from_secs(60);
7026        let done = AtomicBool::new(false);
7027        let op = async {
7028            let out = op.await;
7029            done.store(true, Ordering::SeqCst);
7030            out
7031        };
7032        let start = std::time::Instant::now();
7033        let driver = async {
7034            while !done.load(Ordering::SeqCst) {
7035                assert!(
7036                    start.elapsed() < DRIVE_BUDGET,
7037                    "drive: op did not complete within {DRIVE_BUDGET:?} — a stuck regression, not a hang"
7038                );
7039                run_once(sender).await;
7040                tokio::task::yield_now().await;
7041            }
7042        };
7043        let (out, ()) = tokio::join!(op, driver);
7044        out
7045    }
7046
7047    /// `producer.initTransactions()` driven to completion — the first line of most
7048    /// Java transactional tests.
7049    async fn init_transactions(ctx: &mut TxnProducerContext) {
7050        ctx.prepare_init_transactions(Errors::None, PRODUCER_ID, EPOCH);
7051        drive(&mut ctx.sender, ctx.producer.init_transactions())
7052            .await
7053            .expect("initTransactions succeeds");
7054    }
7055
7056    /// Java's `initProducerIdResponse(long producerId, short epoch, Errors error)`
7057    /// (`KafkaProducerTest#initProducerIdResponse`).
7058    fn init_producer_id_response(error: Errors, producer_id: i64, epoch: i16) -> ConcreteResponse {
7059        use crate::InitProducerIdResponseData;
7060        use crate::common::requests::InitProducerIdResponse;
7061
7062        let mut data = InitProducerIdResponseData::new();
7063        data.set_error_code(error.code())
7064            .set_producer_id(producer_id)
7065            .set_producer_epoch(epoch)
7066            .set_throttle_time_ms(0);
7067        ConcreteResponse::InitProducerId(InitProducerIdResponse::new(data))
7068    }
7069
7070    /// `FindCoordinatorResponse.prepareResponse(error, key, node)`.
7071    fn find_coordinator_response(error: Errors, key: &str, node: &Node) -> ConcreteResponse {
7072        use crate::common::requests::FindCoordinatorResponse;
7073
7074        ConcreteResponse::FindCoordinator(FindCoordinatorResponse::prepare_response(error, key, node))
7075    }
7076
7077    /// Java's `endTxnResponse(Errors error)` (`KafkaProducerTest#endTxnResponse`).
7078    fn end_txn_response(error: Errors) -> ConcreteResponse {
7079        use crate::EndTxnResponseData;
7080        use crate::common::requests::EndTxnResponse;
7081
7082        let mut data = EndTxnResponseData::new();
7083        data.set_error_code(error.code()).set_throttle_time_ms(0);
7084        ConcreteResponse::EndTxn(EndTxnResponse::new(data))
7085    }
7086
7087    // -- Transactional `KafkaProducerTest` methods --------------------------
7088
7089    /// Translated from `KafkaProducerTest.testInitTransactionTimeout`.
7090    ///
7091    /// The `FindCoordinator` is answered but the `InitProducerId` is not, so
7092    /// `initTransactions` expires at `max.block.ms`. A retry then succeeds — which is
7093    /// only possible because a timed-out `TransactionalRequestResult` is **not**
7094    /// acked (Java's `await` sets `isAcked` only after the latch opens,
7095    /// `TransactionalRequestResult#await`), so
7096    /// `handleCachedTransactionRequestResult` hands the same pending result back.
7097    #[tokio::test]
7098    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testInitTransactionTimeout")]
7099    async fn test_init_transaction_timeout() {
7100        let mut ctx = TxnProducerContext::new(&[("transactional.id", "bad-transaction"), ("max.block.ms", "500")], 1);
7101        // Coarser tick reaches the simulated 500ms deadline in fewer real
7102        // `drive()` iterations, reducing flakiness under CPU contention.
7103        ctx.time.set_auto_tick_ms(20);
7104        let node = coordinator_node();
7105        ctx.sender
7106            .client_mut()
7107            .prepare_response(find_coordinator_response(Errors::None, "bad-transaction", &node));
7108
7109        let error = drive(&mut ctx.sender, ctx.producer.init_transactions())
7110            .await
7111            .expect_err("no InitProducerId response is prepared");
7112        // AK 4.3.1: `assertFutureThrowsWithMessageContaining(TimeoutException, future,
7113        // INIT_TXN_TIMEOUT_MSG)`.
7114        assert!(matches!(error, Error::Timeout(_)), "expected a TimeoutException, got {error}");
7115        assert!(
7116            error.message().contains(KafkaProducer::<String, String>::INIT_TXN_TIMEOUT_MSG),
7117            "expected the InitTransactions timeout reason in the message, got {error}"
7118        );
7119
7120        // Retry initialization should work.
7121        ctx.sender
7122            .client_mut()
7123            .prepare_response(find_coordinator_response(Errors::None, "bad-transaction", &node));
7124        ctx.sender
7125            .client_mut()
7126            .prepare_response(init_producer_id_response(Errors::None, PRODUCER_ID, EPOCH));
7127        drive(&mut ctx.sender, ctx.producer.init_transactions())
7128            .await
7129            .expect("the retry succeeds");
7130        assert_eq!(ctx.producer_id_and_epoch(), (PRODUCER_ID, EPOCH));
7131    }
7132
7133    /// Translated from `KafkaProducerTest.testInitTransactionsResponseAfterTimeout`.
7134    ///
7135    /// Java submits `initTransactions` to an executor, waits for the `InitProducerId`
7136    /// to be in flight, advances the clock past `max.block.ms`, asserts the future
7137    /// threw, *then* answers the request and calls `initTransactions` again. The
7138    /// second call must return normally rather than raise — the response completed the
7139    /// cached result.
7140    ///
7141    /// The executor is not needed here: [`drive`] already runs the application call
7142    /// and the Sender concurrently, and its return is the future Java asserts on.
7143    #[tokio::test]
7144    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testInitTransactionsResponseAfterTimeout")]
7145    async fn test_init_transactions_response_after_timeout() {
7146        let mut ctx = TxnProducerContext::new(&[("transactional.id", "bad-transaction"), ("max.block.ms", "500")], 1);
7147        let node = coordinator_node();
7148        ctx.sender
7149            .client_mut()
7150            .prepare_response(find_coordinator_response(Errors::None, "bad-transaction", &node));
7151
7152        let error = drive(&mut ctx.sender, ctx.producer.init_transactions())
7153            .await
7154            .expect_err("the InitProducerId is unanswered");
7155        // AK 4.3.1: `assertFutureThrowsWithMessageContaining(TimeoutException, future,
7156        // INIT_TXN_TIMEOUT_MSG)` — the timeout message carries the init-txn reason.
7157        assert!(matches!(error, Error::Timeout(_)), "expected a TimeoutException, got {error}");
7158        assert!(
7159            error.message().contains(KafkaProducer::<String, String>::INIT_TXN_TIMEOUT_MSG),
7160            "expected the InitTransactions timeout reason in the message, got {error}"
7161        );
7162        assert!(
7163            ctx.sender.client().in_flight_request_count() > 0,
7164            "the InitProducerId must still be in flight, which is what the late response answers"
7165        );
7166
7167        // Java's `client.respond(..)`: answer the request that is already in flight.
7168        ctx.sender
7169            .client_mut()
7170            .respond(init_producer_id_response(Errors::None, PRODUCER_ID, EPOCH));
7171        drive(&mut ctx.sender, ctx.producer.init_transactions())
7172            .await
7173            .expect("the late response completed the cached result");
7174        assert_eq!(ctx.producer_id_and_epoch(), (PRODUCER_ID, EPOCH));
7175    }
7176
7177    /// Translated from `KafkaProducerTest.testInitTransactionWhileThrottled`.
7178    ///
7179    /// The coordinator is throttled for 5 s while `max.block.ms` is 10 s, so
7180    /// `awaitNodeReady` has to wait the node out before the `InitProducerId` goes.
7181    #[tokio::test]
7182    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testInitTransactionWhileThrottled")]
7183    async fn test_init_transaction_while_throttled() {
7184        let mut ctx = TxnProducerContext::new(&[("transactional.id", TRANSACTIONAL_ID), ("max.block.ms", "10000")], 1);
7185        // Java's `new MockTime(1)`: without a ticking clock `awaitNodeReady`'s
7186        // `now - startTime < timeout` never advances, because this test drives the
7187        // Sender itself and nothing else moves the clock.
7188        ctx.time.set_auto_tick_ms(1);
7189        let node = coordinator_node();
7190        ctx.sender.client_mut().throttle(&node, 5000);
7191        ctx.prepare_init_transactions(Errors::None, PRODUCER_ID, EPOCH);
7192
7193        drive(&mut ctx.sender, ctx.producer.init_transactions())
7194            .await
7195            .expect("initTransactions rides out the throttle");
7196        assert_eq!(ctx.producer_id_and_epoch(), (PRODUCER_ID, EPOCH));
7197    }
7198
7199    /// Translated from `KafkaProducerTest.testClusterAuthorizationFailure`.
7200    ///
7201    /// `CLUSTER_AUTHORIZATION_FAILED` on the `InitProducerId` is an authorization
7202    /// error, so the Sender's `shouldHandleAuthorizationError`
7203    /// (`Sender#shouldHandleAuthorizationError`) fails the pending requests, aborts the batches and
7204    /// transitions back to `UNINITIALIZED` — which is what makes the retry Java
7205    /// performs possible at all (PLAN §9.16 pass-1 finding 1).
7206    #[tokio::test]
7207    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testClusterAuthorizationFailure")]
7208    async fn test_cluster_authorization_failure() {
7209        let mut ctx = TxnProducerContext::new(
7210            &[
7211                ("transactional.id", "some-txn"),
7212                ("enable.idempotence", "true"),
7213                ("max.block.ms", "500"),
7214            ],
7215            1,
7216        );
7217        ctx.time.set_auto_tick_ms(1);
7218        let node = coordinator_node();
7219        ctx.sender
7220            .client_mut()
7221            .prepare_response(find_coordinator_response(Errors::None, "some-txn", &node));
7222        ctx.sender.client_mut().prepare_response(init_producer_id_response(
7223            Errors::ClusterAuthorizationFailed,
7224            PRODUCER_ID,
7225            EPOCH,
7226        ));
7227
7228        let error = drive(&mut ctx.sender, ctx.producer.init_transactions())
7229            .await
7230            .expect_err("the cluster authorization failure surfaces");
7231        assert_eq!(error.error(), Errors::ClusterAuthorizationFailed);
7232        assert!(
7233            ctx.transaction_manager.lock().unwrap().has_abortable_error(),
7234            "CLUSTER_AUTHORIZATION_FAILED is an abortable error on InitProducerId \
7235             (`TransactionManager.InitProducerIdHandler#handleResponse`)"
7236        );
7237
7238        // Java retries with
7239        // `TestUtils.retryOnExceptionWithTimeout(1000, 100, producer::initTransactions)`
7240        // because its Sender runs on an independent thread and the recovery has not
7241        // necessarily happened yet: the manager is in ABORTABLE_ERROR the instant the
7242        // result fails, and only the Sender's *next* `runOnce` takes
7243        // `shouldHandleAuthorizationError`'s path back to UNINITIALIZED
7244        // (`Sender#shouldHandleAuthorizationError`). Until it does, an attempt is rejected with "we are
7245        // in an error state".
7246        //
7247        // `drive` stops the moment the application call resolves, so that iteration is
7248        // asked for explicitly here — which makes the recovery deterministic rather
7249        // than raced, and lets the state be asserted directly instead of retried
7250        // around.
7251        ctx.run_sender(1).await;
7252        assert!(
7253            !ctx.transaction_manager.lock().unwrap().has_error(),
7254            "one runOnce must clear the abortable error via transitionToUninitialized"
7255        );
7256        assert!(
7257            !ctx.transaction_manager.lock().unwrap().has_producer_id(),
7258            "UNINITIALIZED means the producer id is gone too"
7259        );
7260
7261        // Only an `InitProducerId` is prepared, as in Java: `transitionToUninitialized`
7262        // does not forget the coordinator, so no second `FindCoordinator` is sent.
7263        ctx.sender
7264            .client_mut()
7265            .prepare_response(init_producer_id_response(Errors::None, PRODUCER_ID, EPOCH));
7266        drive(&mut ctx.sender, ctx.producer.init_transactions())
7267            .await
7268            .expect("the retry succeeds once the manager is back in UNINITIALIZED");
7269        assert_eq!(ctx.producer_id_and_epoch(), (PRODUCER_ID, EPOCH));
7270        ctx.producer.close().await.expect("close");
7271    }
7272
7273    /// Translated from `KafkaProducerTest.testAbortTransaction`.
7274    ///
7275    /// The whole `FindCoordinator` -> `InitProducerId` -> `EndTxn(ABORT)` sequence,
7276    /// with no records in the transaction.
7277    #[tokio::test]
7278    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testAbortTransaction")]
7279    async fn test_abort_transaction() {
7280        let mut ctx = TxnProducerContext::transactional();
7281        init_transactions(&mut ctx).await;
7282        ctx.producer.begin_transaction().expect("beginTransaction");
7283
7284        ctx.sender.client_mut().prepare_response(end_txn_response(Errors::None));
7285        drive(&mut ctx.sender, ctx.producer.abort_transaction())
7286            .await
7287            .expect("abortTransaction");
7288    }
7289
7290    /// Translated from
7291    /// `KafkaProducerTest.testOnlyCanExecuteCloseAfterInitTransactionsTimeout`.
7292    ///
7293    /// Nothing answers the `FindCoordinator`, so `initTransactions` expires at
7294    /// `max.block.ms=5`. After that failure every other transactional operation must
7295    /// be rejected, and only `close` is allowed.
7296    #[tokio::test]
7297    #[doc(
7298        alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testOnlyCanExecuteCloseAfterInitTransactionsTimeout"
7299    )]
7300    async fn test_only_can_execute_close_after_init_transactions_timeout() {
7301        let mut ctx = TxnProducerContext::new(&[("transactional.id", "bad-transaction"), ("max.block.ms", "5")], 1);
7302
7303        let error = drive(&mut ctx.sender, ctx.producer.init_transactions())
7304            .await
7305            .expect_err("nothing answers the FindCoordinator");
7306        // AK 4.3.1: `assertTrue(timeoutEx.getMessage().contains(INIT_TXN_TIMEOUT_MSG))`.
7307        assert!(
7308            error.message().contains(KafkaProducer::<String, String>::INIT_TXN_TIMEOUT_MSG),
7309            "expected the InitTransactions timeout reason in the message, got {error}"
7310        );
7311        assert_eq!(
7312            error.message(),
7313            format!(
7314                "Timeout expired after 5ms while awaiting InitProducerId. {}",
7315                KafkaProducer::<String, String>::INIT_TXN_TIMEOUT_MSG
7316            )
7317        );
7318
7319        // Other transactional operations are not allowed once the caller has taken the
7320        // error from a failed initTransactions: the manager is still INITIALIZING with
7321        // an unacked pending transition.
7322        let begin_error = ctx.producer.begin_transaction().expect_err("beginTransaction is rejected");
7323        assert_eq!(
7324            begin_error.message(),
7325            "Cannot attempt operation `beginTransaction` because the previous call to \
7326             `initTransactions` timed out and must be retried"
7327        );
7328
7329        ctx.producer
7330            .close_with_timeout(Duration::from_millis(0))
7331            .await
7332            .expect("close is the one allowed operation");
7333    }
7334
7335    /// Translated from `KafkaProducerTest.testPartitionAddedToTransaction`.
7336    ///
7337    /// Pins the producer-level transactional wiring: `doSend` must call
7338    /// `transactionManager.maybeAddPartition(tp)` after the append succeeds, with the
7339    /// partition the accumulator actually chose (`KafkaProducer#doSend`). The
7340    /// send's future must also still be pending — the record is buffered, not sent.
7341    ///
7342    /// # Deviation: a real manager instead of Mockito's `verify`
7343    ///
7344    /// Java builds the producer through `KafkaProducerTestContext`, which injects
7345    /// `mock(TransactionManager.class)` (`KafkaProducerTest.KafkaProducerTestContext#transactionManager`, passed at `KafkaProducerTest`), and asserts with
7346    /// `verify(ctx.transactionManager).maybeAddPartition(topicPartition)`.
7347    /// `TransactionManager` is a concrete struct here, so there is nothing to stub; the
7348    /// substitute is a **real** transactional manager driven into `IN_TRANSACTION`, and
7349    /// the assertion is `is_partition_pending_add` (`TransactionManager`) — the
7350    /// state `maybeAddPartition` exists to produce.
7351    ///
7352    /// That is stronger than `verify` on one axis: Mockito confirms the call was made,
7353    /// while this confirms it was made *and* had its effect. It is **not** stronger for
7354    /// carrying the right partition — `verify(..).maybeAddPartition(topicPartition)`
7355    /// checks the argument too, so that clause is parity, not superiority.
7356    ///
7357    /// The effect clause is a real gain, and holds because the state read is isolated.
7358    /// `is_partition_pending_add` reads the union
7359    /// `new_partitions_in_transaction ∪ pending_partitions_in_transaction`
7360    /// (`TransactionManager`), and in this crate:
7361    /// `new_partitions_in_transaction.insert` has exactly one call site, inside
7362    /// `maybe_add_partition`; and `pending_partitions_in_transaction` is only ever filled
7363    /// by `add_partitions_to_transaction_handler`'s
7364    /// `.extend(new_partitions_in_transaction.iter())`, so it is downstream of that same
7365    /// insert. The union is therefore non-empty only if `maybe_add_partition` ran, and
7366    /// the test asserts the negative pre-condition first.
7367    ///
7368    /// It is also why this test could be written at all rather than deferred like
7369    /// `SenderTest`'s one mock-injected entry, whose stub makes a real method *throw* and
7370    /// so has no real-state equivalent.
7371    ///
7372    /// # The one Java assertion not carried across
7373    ///
7374    /// Java opens with `assertEquals(future, producer.send(record))`, comparing
7375    /// the returned future against a **pre-built** `FutureRecordMetadata` that
7376    /// `expectAppend` installed by stubbing `ctx.accumulator.append(..)` and
7377    /// `ctx.partitioner.partition(..)` to return it. That assertion is about the
7378    /// *accumulator* mock, not the manager mock the deviation above discusses, and it is
7379    /// unrepresentable here for the same reason: with a real `RecordAccumulator` the
7380    /// future is created inside `append`, so there is no pre-known value to compare
7381    /// identity against. What survives of its intent — that `send` hands back the
7382    /// accumulator's own pending future rather than a completed or failed one — is
7383    /// asserted directly by `!future.is_done()` below.
7384    ///
7385    /// # Why this method was missing until Critic 46 issue 4
7386    ///
7387    /// It reaches the transactional path only through the injected mock, so its body
7388    /// names no public transactional method and no transactional config key — and the
7389    /// accounting block's marker set contained neither `TransactionManager` nor
7390    /// `maybeAddPartition`. Both are markers now; see the accounting block.
7391    #[tokio::test]
7392    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testPartitionAddedToTransaction")]
7393    async fn test_partition_added_to_transaction() {
7394        let mut ctx = TxnProducerContext::transactional();
7395        init_transactions(&mut ctx).await;
7396        ctx.producer.begin_transaction().expect("beginTransaction");
7397
7398        let partition = TopicPartition::new(TOPIC.to_string(), 0);
7399        assert!(
7400            !ctx.transaction_manager.lock().unwrap().is_partition_pending_add(&partition),
7401            "nothing may be pending before the send"
7402        );
7403
7404        let record = ProducerRecord::with_options(
7405            ProducerRecordOptionsBuilder::new()
7406                .set_topic(TOPIC.to_string())
7407                .set_value(Some("value".to_string()))
7408                .set_timestamp(Some(ctx.time.milliseconds()))
7409                .set_key(Some("key".to_string()))
7410                .build()
7411                .unwrap(),
7412        )
7413        .expect("a valid record");
7414        let future = ctx.producer.send(record).await.expect("send");
7415
7416        assert!(
7417            !future.is_done(),
7418            "the record is buffered in the accumulator, not sent — nothing has driven the Sender"
7419        );
7420        assert!(
7421            ctx.transaction_manager.lock().unwrap().is_partition_pending_add(&partition),
7422            "doSend must call maybeAddPartition with the partition the accumulator chose"
7423        );
7424    }
7425
7426    // =====================================================================
7427    // `maybeAddPartition`'s failure arm in `doSend`
7428    //
7429    // `TransactionManagerTest` covers what `maybeAddPartition` raises in each state
7430    // (`testFailIfNotReadyForSend*`, translated in
7431    // `transaction_manager.rs`). What follows covers the other half — how `doSend`
7432    // *surfaces* each of those, which only `KafkaProducer` decides — and it has no
7433    // Java counterpart: Java's own `KafkaProducerTest` reaches this arm once, through
7434    // a Mockito stub, in `testPartitionAddedToTransaction` above. These rows therefore
7435    // do NOT enter the Phase-6 accounting block's denominator.
7436    //
7437    // They exist because the arm shipped with a self-deadlock: written inline as
7438    // `if let Err(e) = tm.lock().unwrap().maybe_add_partition(..)`, the scrutinee's
7439    // `MutexGuard` outlives the whole success arm (edition 2024 only shortens it
7440    // across `else`), and the body re-locked the same non-reentrant
7441    // `std::sync::Mutex` through `handle_api_error` →
7442    // `maybe_transition_to_error_state`. Java's monitor is reentrant, so no Java test
7443    // could have caught it.
7444    // =====================================================================
7445
7446    /// How long [`bounded`] waits before calling a send wedged.
7447    ///
7448    /// Generous against a loaded CI box; the operations under test do no I/O and
7449    /// finish in microseconds.
7450    const DEADLOCK_BOUND: Duration = Duration::from_secs(10);
7451
7452    /// Runs `body` on its own thread and returns its value, failing the test if it
7453    /// does not finish within [`DEADLOCK_BOUND`].
7454    ///
7455    /// A `tokio::time::timeout` around the send would NOT bound these tests. The
7456    /// regression they guard blocks the thread inside `std::sync::Mutex::lock`, so the
7457    /// future never yields, the runtime never advances its timers, and the timeout can
7458    /// never fire — the symptom is a hung `cargo test`, not a failing one. Only a
7459    /// second thread can observe a wedged first one, so `body` gets a thread and its
7460    /// own current-thread runtime while the test thread waits on a channel.
7461    ///
7462    /// On a timeout the worker stays blocked forever, holding the producer it built.
7463    /// That is deliberate and harmless: it shares nothing with any other test, and
7464    /// libtest ends the process with `exit(2)` rather than joining stray threads.
7465    fn bounded<T: Send + 'static>(what: &str, body: impl FnOnce() -> T + Send + 'static) -> T {
7466        let (tx, rx) = std::sync::mpsc::channel();
7467        std::thread::spawn(move || {
7468            // `send` fails only if the receiver timed out and gave up; nothing to do.
7469            let _ = tx.send(body());
7470        });
7471        rx.recv_timeout(DEADLOCK_BOUND).unwrap_or_else(|_| {
7472            panic!(
7473                "{what} did not return within {DEADLOCK_BOUND:?} — the send path is wedged. \
7474                 doSend's maybeAddPartition arm must release the TransactionManager guard \
7475                 before handling the error, because handle_api_error re-locks it."
7476            )
7477        })
7478    }
7479
7480    /// Runs `body` on a fresh current-thread runtime inside [`bounded`].
7481    fn bounded_block_on<F>(what: &str, body: impl FnOnce() -> F + Send + 'static) -> F::Output
7482    where
7483        F: std::future::Future,
7484        F::Output: Send + 'static,
7485    {
7486        bounded(what, move || {
7487            tokio::runtime::Builder::new_current_thread()
7488                .enable_all()
7489                .build()
7490                .expect("a current-thread runtime")
7491                .block_on(body())
7492        })
7493    }
7494
7495    /// The record every test below sends; its contents are irrelevant because no send
7496    /// is expected to reach a batch's wire form.
7497    fn misuse_record() -> ProducerRecord<String, String> {
7498        ProducerRecord::with_key(TOPIC.to_string(), Some("key".to_string()), Some("value".to_string()))
7499    }
7500
7501    /// `send()` on a transactional producer that never called `initTransactions`.
7502    ///
7503    /// The state `TransactionManagerTest.testFailIfNotReadyForSendNoProducerId`
7504    /// asserts on, surfaced through `doSend`. `maybeAddPartition` raises
7505    /// `IllegalStateException` (`TransactionManager`), which is not a
7506    /// `KafkaException` at all, so it misses `catch (ApiException e)` *and*
7507    /// `catch (KafkaException e)`, reaches `catch (Exception e)`
7508    /// (`KafkaProducer`) and is rethrown out of `send()`.
7509    #[test]
7510    fn test_send_before_init_transactions_returns_illegal_state() {
7511        let error = bounded_block_on("send before initTransactions", || async {
7512            let ctx = TxnProducerContext::transactional();
7513            ctx.producer
7514                .send(misuse_record())
7515                .await
7516                .expect_err("an uninitialized transactional producer cannot send")
7517        });
7518
7519        assert!(
7520            matches!(error, Error::LocalIllegalState(_)),
7521            "an illegal-state error is not an API error, so it must be returned by send() \
7522             rather than reported through the future; got {error:?}"
7523        );
7524        assert_eq!(
7525            error.to_string(),
7526            format!(
7527                "LocalIllegalStateError: Cannot add partition {TOPIC}-0 to transaction before completing a call to initTransactions"
7528            )
7529        );
7530    }
7531
7532    /// `send()` on an initialized transactional producer with no open transaction.
7533    ///
7534    /// The state `TransactionManagerTest.testFailIfNotReadyForSendNoOngoingTransaction`
7535    /// asserts on, surfaced through `doSend`. Same
7536    /// `IllegalStateException` treatment as above, from
7537    /// `TransactionManager`, whose message carries the state and Java's
7538    /// double space before it.
7539    #[test]
7540    fn test_send_outside_transaction_returns_illegal_state() {
7541        let error = bounded_block_on("send outside a transaction", || async {
7542            let mut ctx = TxnProducerContext::transactional();
7543            init_transactions(&mut ctx).await;
7544            ctx.producer
7545                .send(misuse_record())
7546                .await
7547                .expect_err("a send needs an open transaction")
7548        });
7549
7550        assert!(
7551            matches!(error, Error::LocalIllegalState(_)),
7552            "expected the IllegalState to be returned by send(), got {error:?}"
7553        );
7554        assert_eq!(
7555            error.to_string(),
7556            format!("LocalIllegalStateError: Cannot add partition {TOPIC}-0 to transaction while in state  READY")
7557        );
7558    }
7559
7560    /// `send()` after an abortable error, the state
7561    /// `TransactionManagerTest.testFailIfNotReadyForSendAfterAbortableError`
7562    /// asserts on.
7563    ///
7564    /// `maybeFailWithError` raises a **bare** `KafkaException`
7565    /// (`TransactionManager#maybeFailWithError`). `ApiException extends KafkaException`, not the
7566    /// other way round, so `catch (ApiException e)` does not match and
7567    /// `catch (KafkaException e)` (`KafkaProducer`) rethrows it — a
7568    /// block that, unlike the `ApiException` one, never calls
7569    /// `maybeTransitionToErrorState`. Hence the second assertion: routing this error to
7570    /// the `ApiException` arm would overwrite `lastError` with the "we are in an error
7571    /// state" wrapper and lose the cause the application needs.
7572    #[test]
7573    fn test_send_after_abortable_error_returns_error_without_overwriting_last_error() {
7574        let (error, last_error) = bounded_block_on("send after an abortable error", || async {
7575            let mut ctx = TxnProducerContext::transactional();
7576            init_transactions(&mut ctx).await;
7577            ctx.producer.begin_transaction().expect("beginTransaction");
7578            ctx.transaction_manager
7579                .lock()
7580                .unwrap()
7581                .transition_to_abortable_error(Error::with_message(Errors::InvalidTxnState, "cause"), Caller::App)
7582                .expect("IN_TRANSACTION -> ABORTABLE_ERROR is valid");
7583
7584            let error = ctx
7585                .producer
7586                .send(misuse_record())
7587                .await
7588                .expect_err("an abortable error blocks further sends");
7589            let last_error = ctx.transaction_manager.lock().unwrap().last_error().cloned();
7590            (error, last_error)
7591        });
7592
7593        assert_eq!(
7594            error.error(),
7595            Errors::UnknownServerError,
7596            "this crate spells Java's bare Kafka error as UnknownServerError; got {error:?}"
7597        );
7598        assert_eq!(
7599            error.message(),
7600            "Cannot execute transactional method because we are in an error state"
7601        );
7602        let last_error = last_error.expect("the manager keeps the abortable cause");
7603        assert_eq!(
7604            last_error.error(),
7605            Errors::InvalidTxnState,
7606            "the rethrowing catch block must not run maybeTransitionToErrorState, which \
7607             would replace the cause with the wrapper; got {last_error:?}"
7608        );
7609    }
7610
7611    /// `send()` after a fatal error, the state
7612    /// `TransactionManagerTest.testFailIfNotReadyForSendAfterFatalError`
7613    /// asserts on. Same bare-`KafkaException` treatment as the abortable case.
7614    #[test]
7615    fn test_send_after_fatal_error_returns_error() {
7616        // The closure also reports the manager's state: Java records fatality there
7617        // (`hasFatalError()` == `currentState == FATAL_ERROR`), not on the error.
7618        let (error, still_fatal) = bounded_block_on("send after a fatal error", || async {
7619            let mut ctx = TxnProducerContext::transactional();
7620            init_transactions(&mut ctx).await;
7621            ctx.transaction_manager
7622                .lock()
7623                .unwrap()
7624                .transition_to_fatal_error(
7625                    Error::with_message(Errors::ClusterAuthorizationFailed, "cause"),
7626                    Caller::App,
7627                )
7628                .expect("FATAL_ERROR is always reachable");
7629            let error = ctx
7630                .producer
7631                .send(misuse_record())
7632                .await
7633                .expect_err("a fatal error blocks further sends");
7634            let still_fatal = ctx.transaction_manager.lock().unwrap().has_fatal_error();
7635            (error, still_fatal)
7636        });
7637
7638        assert_eq!(error.error(), Errors::UnknownServerError, "got {error:?}");
7639        assert_eq!(
7640            error.message(),
7641            "Cannot execute transactional method because we are in an error state"
7642        );
7643        // Java records fatality in the state machine, not on the exception
7644        // (`TransactionManager.hasFatalError()` == `currentState == FATAL_ERROR`;
7645        // `lastError` is a plain RuntimeException). The application distinguishes a
7646        // dead producer by the exception *type* `maybe_fail_with_error` surfaces —
7647        // asserted above — so the regression guard is the state, checked here.
7648        assert!(still_fatal, "the producer must remain in the fatal state: {error:?}");
7649    }
7650
7651    /// `send()` on a purely **idempotent** producer whose manager is in a fatal state,
7652    /// the state
7653    /// `TransactionManagerTest.testFailIfNotReadyForSendIdempotentProducerFatalError`
7654    /// asserts on.
7655    ///
7656    /// `maybeFailWithError` runs before `maybeAddPartition`'s `isTransactional()` test
7657    /// (`TransactionManager#maybeAddPartition` vs `TransactionManager#maybeAddPartition`), so a producer with no
7658    /// `transactional.id` reaches the same arm.
7659    #[test]
7660    fn test_idempotent_send_after_fatal_error_returns_error() {
7661        let error = bounded_block_on("idempotent send after a fatal error", || async {
7662            let ctx = TxnProducerContext::new(&[], 1);
7663            ctx.transaction_manager
7664                .lock()
7665                .unwrap()
7666                .transition_to_fatal_error(Error::with_message(Errors::UnsupportedVersion, "cause"), Caller::App)
7667                .expect("FATAL_ERROR is always reachable");
7668            ctx.producer
7669                .send(misuse_record())
7670                .await
7671                .expect_err("a fatal error blocks further sends")
7672        });
7673
7674        assert_eq!(error.error(), Errors::UnknownServerError, "got {error:?}");
7675    }
7676
7677    /// The other side of the split: an `ApiException` out of `maybeAddPartition` still
7678    /// takes `catch (ApiException e)` and is reported through the future.
7679    ///
7680    /// This is the arm that actually re-locks the manager —
7681    /// `handle_api_error` → `maybe_transition_to_error_state` — so it is the direct
7682    /// regression test for the deadlock. `maybeFailWithError` re-raises a fenced
7683    /// producer as `ProducerFencedException` (`TransactionManager#maybeFailWithError`),
7684    /// which IS an `ApiException`.
7685    ///
7686    /// The manager is seeded **abortable**, not fatal, so that
7687    /// `maybeTransitionToErrorState` has somewhere to move it: `ProducerFenced` is in
7688    /// that method's fatal set (Java `TransactionManager#maybeTransitionToErrorState`), so the
7689    /// `ApiException` block drives `ABORTABLE_ERROR -> FATAL_ERROR`. Seeding
7690    /// `FATAL_ERROR` up front — as this test first did — makes the state assertion
7691    /// tautological: it would hold even if the arm were changed to `return Err(error)`
7692    /// and never call `maybe_transition_to_error_state` at all. Both `has_error()`
7693    /// arms reach the same `maybeFailWithError` branch, which keys on
7694    /// `last_error`'s code, so the error the send observes is unchanged.
7695    #[test]
7696    fn test_send_after_producer_fenced_fails_the_future() {
7697        let (send_result, fatal_before, fatal_after) = bounded_block_on("send after being fenced", || async {
7698            let mut ctx = TxnProducerContext::transactional();
7699            init_transactions(&mut ctx).await;
7700            ctx.producer.begin_transaction().expect("beginTransaction");
7701            ctx.transaction_manager
7702                .lock()
7703                .unwrap()
7704                .transition_to_abortable_error(Error::with_message(Errors::ProducerFenced, "fenced"), Caller::App)
7705                .expect("IN_TRANSACTION -> ABORTABLE_ERROR is valid");
7706            let fatal_before = ctx.transaction_manager.lock().unwrap().has_fatal_error();
7707
7708            let send_result = match ctx.producer.send(misuse_record()).await {
7709                Ok(future) => Ok(future.get().await.expect_err("the fenced send cannot be acked")),
7710                Err(error) => Err(error),
7711            };
7712            let fatal_after = ctx.transaction_manager.lock().unwrap().has_fatal_error();
7713            (send_result, fatal_before, fatal_after)
7714        });
7715
7716        let error = send_result.expect("an API error is reported through the future, not the call");
7717        assert_eq!(
7718            error.error(),
7719            Errors::ProducerFenced,
7720            "a producer-fenced error is an API error, so doSend returns a failed future; got {error:?}"
7721        );
7722        // `maybeFailWithError` re-raises rather than re-throwing `lastError`, so the
7723        // message is the fresh one `TransactionManager#maybeFailWithError` builds — not the "fenced" text the
7724        // test seeded.
7725        assert_eq!(
7726            error.message(),
7727            format!(
7728                "Producer with transactionalId '{TRANSACTIONAL_ID}' and \
7729                 (producerId={PRODUCER_ID}, epoch={EPOCH}) has been fenced by another producer \
7730                 with the same transactionalId"
7731            )
7732        );
7733        assert!(
7734            !fatal_before,
7735            "the manager starts abortable, so the transition below is observable"
7736        );
7737        assert!(
7738            fatal_after,
7739            "the API-error block runs maybeTransitionToErrorState, which moves a fenced \
7740             producer from ABORTABLE_ERROR to FATAL_ERROR; the rethrow path would not"
7741        );
7742    }
7743
7744    /// The `maybeAddPartition` `ApiException` arm of `do_send_bytes` passes `None`
7745    /// (not the record's callback) to `handle_api_error`, so the callback is
7746    /// **not** fired synchronously there.
7747    ///
7748    /// The callback was moved into `accumulator.append(..)` and registered with the
7749    /// record's future, which fires it exactly once when the batch is later
7750    /// completed/aborted. Passing it here too would double-fire (CLAUDE.md §11.5).
7751    /// (Java's `doSend` catch additionally fires the raw `callback` at
7752    /// `KafkaProducer#doSend`, but keeps `callback` as a reference separate from
7753    /// the `appendCallbacks` it registered, so Java can invoke it twice on this path;
7754    /// Rust's single-owner model fires it exactly once.)
7755    #[test]
7756    fn test_send_api_exception_after_append_fires_callback_exactly_once() {
7757        let (count_after_send, count_final) = bounded_block_on("fenced send with callback", || async {
7758            let mut ctx = TxnProducerContext::transactional();
7759            init_transactions(&mut ctx).await;
7760            ctx.producer.begin_transaction().expect("beginTransaction");
7761            // Seed abortable (not fatal) so `maybeAddPartition`'s `maybe_fail_with_error`
7762            // re-raises `ProducerFenced` — an `ApiException` — after the append succeeds.
7763            ctx.transaction_manager
7764                .lock()
7765                .unwrap()
7766                .transition_to_abortable_error(Error::with_message(Errors::ProducerFenced, "fenced"), Caller::App)
7767                .expect("IN_TRANSACTION -> ABORTABLE_ERROR is valid");
7768
7769            let count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
7770            let cb_count = Arc::clone(&count);
7771            let callback: Box<dyn Callback> =
7772                Box::new(move |_metadata: Option<&RecordMetadata>, _error: Option<&Error>| {
7773                    cb_count.fetch_add(1, Ordering::SeqCst);
7774                });
7775
7776            // The append succeeds; `maybeAddPartition` then raises `ProducerFenced`,
7777            // taking the `handle_api_error(.., None)` arm.
7778            let future = ctx
7779                .producer
7780                .send_with_callback(misuse_record(), Some(callback))
7781                .await
7782                .expect("an ApiException is reported through the future, not the call");
7783            let error = future.get().await.expect_err("the fenced send cannot be acked");
7784            assert_eq!(error.error(), Errors::ProducerFenced, "got {error:?}");
7785
7786            // `handle_api_error` was passed `None`, so it did NOT fire the callback.
7787            let count_after_send = count.load(Ordering::SeqCst);
7788
7789            // The record still sits in the accumulator carrying the callback; the
7790            // accumulator fires it exactly once when the batch is aborted (as the
7791            // Sender's abort/close path would).
7792            ctx.accumulator.abort_incomplete_batches().await;
7793            (count_after_send, count.load(Ordering::SeqCst))
7794        });
7795
7796        assert_eq!(
7797            count_after_send, 0,
7798            "the maybeAddPartition ApiException path must NOT fire the callback (it is deferred to \
7799             the accumulator); firing it here would double-fire"
7800        );
7801        assert_eq!(
7802            count_final, 1,
7803            "the callback fires exactly once, from the accumulator when the batch is aborted"
7804        );
7805    }
7806
7807    /// Translated from
7808    /// `KafkaProducerTest.testCommitTransactionWithRecordTooLargeException`.
7809    ///
7810    /// A record larger than `max.request.size` fails its future with
7811    /// `RecordTooLargeException`, and because `doSend`'s `catch (ApiException e)` runs
7812    /// `maybeTransitionToErrorState` (`KafkaProducer#doSend`) the transaction
7813    /// is now abortable — so the following `commitTransaction` must fail rather than
7814    /// commit a partial transaction.
7815    #[tokio::test]
7816    #[doc(
7817        alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testCommitTransactionWithRecordTooLargeException"
7818    )]
7819    async fn test_commit_transaction_with_record_too_large_error() {
7820        let mut ctx =
7821            TxnProducerContext::new(&[("transactional.id", TRANSACTIONAL_ID), ("max.request.size", "1000")], 1);
7822        ctx.time.set_auto_tick_ms(1);
7823        init_transactions(&mut ctx).await;
7824        ctx.producer.begin_transaction().expect("beginTransaction");
7825
7826        let large_string = "*".repeat(1000);
7827        let record = ProducerRecord::with_key(TOPIC.to_string(), Some("large string".to_string()), Some(large_string));
7828        let future = ctx
7829            .producer
7830            .send(record)
7831            .await
7832            .expect("an API error is reported through the future, not the call");
7833        let send_error = future.get().await.expect_err("the record is too large");
7834        assert!(
7835            matches!(send_error, Error::RecordTooLarge(_)),
7836            "expected RecordTooLarge, got {:?}",
7837            send_error
7838        );
7839
7840        // Java asserts a bare `KafkaException`, which is what `maybeFailWithError`
7841        // (`TransactionManager#maybeFailWithError`) raises for a non-`IllegalStateException`
7842        // `lastError` — the original cause is wrapped, not re-raised.
7843        let commit_error = drive(&mut ctx.sender, ctx.producer.commit_transaction())
7844            .await
7845            .expect_err("the transaction is abortable after a failed send");
7846        assert_eq!(
7847            commit_error.message(),
7848            "Cannot execute transactional method because we are in an error state"
7849        );
7850    }
7851
7852    /// Translated from
7853    /// `KafkaProducerTest.testCommitTransactionWithMetadataTimeoutForMissingTopic`.
7854    ///
7855    /// # Deviation: how the metadata wait is made to expire
7856    ///
7857    /// Java stubs `metadata.fetch()` with Mockito to return an *empty* cluster and, on
7858    /// the sixth invocation, to jump `MockTime` forward by 70 s past a
7859    /// `max.block.ms` of 60 s. `ProducerMetadata` is a concrete type here, so there is
7860    /// no stub to install; the equivalent is a metadata instance that genuinely has no
7861    /// topic plus a `max.block.ms` short enough to expire in test time. What the test
7862    /// observes is unchanged: the send's future fails with a timeout, and the
7863    /// subsequent `commitTransaction` fails because the failed send made the
7864    /// transaction abortable.
7865    #[tokio::test]
7866    #[doc(
7867        alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testCommitTransactionWithMetadataTimeoutForMissingTopic"
7868    )]
7869    async fn test_commit_transaction_with_metadata_timeout_for_missing_topic() {
7870        let mut ctx = TxnProducerContext::new(
7871            &[("transactional.id", TRANSACTIONAL_ID), ("max.block.ms", "200")],
7872            // No partitions: `metadataUpdateWith(1, emptyMap())`, Java's `emptyCluster`.
7873            0,
7874        );
7875        init_transactions(&mut ctx).await;
7876        ctx.producer.begin_transaction().expect("beginTransaction");
7877
7878        let record = ProducerRecord::new(TOPIC.to_string(), Some("value".to_string()));
7879        let future = ctx
7880            .producer
7881            .send(record)
7882            .await
7883            .expect("a timeout is an API error and is reported through the future");
7884        let send_error = future.get().await.expect_err("the topic never appears in metadata");
7885        assert!(
7886            matches!(send_error, Error::Timeout(_)),
7887            "expected Timeout, got {:?}",
7888            send_error
7889        );
7890
7891        // Java asserts a bare `KafkaException` — see
7892        // `test_commit_transaction_with_record_too_large_error`.
7893        let commit_error = drive(&mut ctx.sender, ctx.producer.commit_transaction())
7894            .await
7895            .expect_err("the transaction is abortable after a failed send");
7896        assert_eq!(
7897            commit_error.message(),
7898            "Cannot execute transactional method because we are in an error state"
7899        );
7900    }
7901
7902    /// Translated from
7903    /// `KafkaProducerTest.testCommitTransactionWithMetadataTimeoutForPartitionOutOfRange`.
7904    ///
7905    /// As the previous test, but the metadata does contain the topic — with one
7906    /// partition — and the record names partition 2, so `waitOnMetadata` waits for a
7907    /// partition that never arrives. Same deviation on how the wait is made to expire.
7908    #[tokio::test]
7909    #[doc(
7910        alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testCommitTransactionWithMetadataTimeoutForPartitionOutOfRange"
7911    )]
7912    async fn test_commit_transaction_with_metadata_timeout_for_partition_out_of_range() {
7913        let mut ctx = TxnProducerContext::new(&[("transactional.id", TRANSACTIONAL_ID), ("max.block.ms", "200")], 1);
7914        init_transactions(&mut ctx).await;
7915        ctx.producer.begin_transaction().expect("beginTransaction");
7916
7917        let record = ProducerRecord::with_partition_key(TOPIC.to_string(), Some(2), None, Some("value".to_string()))
7918            .expect("a valid partition");
7919        let future = ctx
7920            .producer
7921            .send(record)
7922            .await
7923            .expect("a timeout is an API error and is reported through the future");
7924        let send_error = future.get().await.expect_err("partition 2 never appears in metadata");
7925        assert!(
7926            matches!(send_error, Error::Timeout(_)),
7927            "expected Timeout, got {:?}",
7928            send_error
7929        );
7930
7931        // Java asserts a bare `KafkaException` — see
7932        // `test_commit_transaction_with_record_too_large_error`.
7933        let commit_error = drive(&mut ctx.sender, ctx.producer.commit_transaction())
7934            .await
7935            .expect_err("the transaction is abortable after a failed send");
7936        assert_eq!(
7937            commit_error.message(),
7938            "Cannot execute transactional method because we are in an error state"
7939        );
7940    }
7941
7942    /// Translated from
7943    /// `KafkaProducerTest.testCommitTransactionWithSendToInvalidTopic`.
7944    ///
7945    /// An invalid topic name fails the send's future with `InvalidTopicException` and
7946    /// leaves the transaction abortable, so the commit fails.
7947    ///
7948    /// Java arranges the invalid topic through `client.prepareMetadataUpdate(..)`, a
7949    /// `MockClient` facility this port does not have; the metadata is seeded with the
7950    /// `INVALID_TOPIC_EXCEPTION` topic directly instead, which is the state that
7951    /// update would have produced.
7952    #[tokio::test]
7953    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testCommitTransactionWithSendToInvalidTopic")]
7954    async fn test_commit_transaction_with_send_to_invalid_topic() {
7955        use crate::MetadataResponseData;
7956        use crate::common::protocol::ApiKeys;
7957        use crate::common::requests::MetadataResponse;
7958        use crate::metadata_response_data::{MetadataResponseBroker, MetadataResponseTopic};
7959
7960        const INVALID_TOPIC: &str = "topic abc"; // Invalid topic name due to space.
7961
7962        let mut ctx = TxnProducerContext::new(&[("transactional.id", TRANSACTIONAL_ID), ("max.block.ms", "15000")], 1);
7963        init_transactions(&mut ctx).await;
7964        ctx.producer.begin_transaction().expect("beginTransaction");
7965
7966        let mut data = MetadataResponseData::new();
7967        data.set_controller_id(0);
7968        data.set_cluster_id(Some("test-cluster".to_string()));
7969        let mut broker = MetadataResponseBroker::new();
7970        broker.set_node_id(0);
7971        broker.set_host("localhost".to_string());
7972        broker.set_port(9092);
7973        data.set_brokers(vec![broker]);
7974        let mut invalid = MetadataResponseTopic::new();
7975        invalid.set_name(Some(INVALID_TOPIC.to_string()));
7976        invalid.set_error_code(Errors::InvalidTopicError.code());
7977        data.set_topics(vec![invalid]);
7978        let response = MetadataResponse::with_version(data, ApiKeys::METADATA.latest_version());
7979        ctx.metadata.add(INVALID_TOPIC, ctx.time.milliseconds());
7980        ctx.metadata
7981            .update_with_current_request_version(&response, false, ctx.time.milliseconds());
7982
7983        let record = ProducerRecord::new(INVALID_TOPIC.to_string(), Some("HelloKafka".to_string()));
7984        let future = ctx
7985            .producer
7986            .send(record)
7987            .await
7988            .expect("an invalid-topic error is reported through the future");
7989        let send_error = future.get().await.expect_err("the topic name is invalid");
7990        assert!(
7991            matches!(send_error, Error::InvalidTopic(_)),
7992            "expected InvalidTopic, got {:?}",
7993            send_error
7994        );
7995
7996        // Java asserts a bare `KafkaException` — see
7997        // `test_commit_transaction_with_record_too_large_error`.
7998        let commit_error = drive(&mut ctx.sender, ctx.producer.commit_transaction())
7999            .await
8000            .expect_err("the transaction is abortable after a failed send");
8001        assert_eq!(
8002            commit_error.message(),
8003            "Cannot execute transactional method because we are in an error state"
8004        );
8005    }
8006
8007    /// Translated from `KafkaProducerTest.testSendTxnOffsetsWithGroupId`.
8008    ///
8009    /// The offsets map Java passes is **empty**, so `sendOffsetsToTransaction` takes
8010    /// `KafkaProducer#sendOffsetsToTransaction`'s early exit and sends nothing at all: the
8011    /// `AddOffsetsToTxn`, second `FindCoordinator` and `TxnOffsetCommit` responses the
8012    /// Java test queues are never consumed, and only the `EndTxn` is. That is
8013    /// preserved rather than "fixed" — `testSendTxnOffsetsWithGroupIdTransactionV2`
8014    /// below is the sibling that passes a real offset.
8015    ///
8016    /// What the test does cover is that an empty map is a no-op even while the
8017    /// coordinator is throttled, and that the commit that follows succeeds.
8018    #[tokio::test]
8019    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testSendTxnOffsetsWithGroupId")]
8020    async fn test_send_txn_offsets_with_group_id() {
8021        let mut ctx = TxnProducerContext::new(&[("transactional.id", TRANSACTIONAL_ID), ("max.block.ms", "10000")], 1);
8022        ctx.time.set_auto_tick_ms(1);
8023        let node = coordinator_node();
8024        ctx.sender.client_mut().throttle(&node, 5000);
8025        init_transactions(&mut ctx).await;
8026        ctx.producer.begin_transaction().expect("beginTransaction");
8027
8028        let group_metadata = ConsumerGroupMetadataImpl::new("group");
8029        let sent_before = ctx.sender.client().request_count();
8030        drive(
8031            &mut ctx.sender,
8032            ctx.producer.send_offsets_to_transaction(HashMap::new(), &group_metadata),
8033        )
8034        .await
8035        .expect("an empty offsets map is a no-op");
8036        assert_eq!(
8037            ctx.sender.client().request_count(),
8038            sent_before,
8039            "an empty offsets map returns from `KafkaProducer#sendOffsetsToTransaction` before touching \
8040             the transaction state"
8041        );
8042
8043        ctx.sender.client_mut().prepare_response(end_txn_response(Errors::None));
8044        drive(&mut ctx.sender, ctx.producer.commit_transaction())
8045            .await
8046            .expect("commitTransaction");
8047    }
8048
8049    /// Translated from `KafkaProducerTest.testSendTxnOffsetsWithGroupMetadata`.
8050    ///
8051    /// Java's offsets map here is empty too, so as in
8052    /// [`test_send_txn_offsets_with_group_id`] nothing is sent; the group metadata it
8053    /// builds carries a generation id and member id, which is what the request matcher
8054    /// would have checked had a request gone out. The check that *is* reachable is that
8055    /// a fully populated `ConsumerGroupMetadata` passes
8056    /// `throwIfInvalidGroupMetadata` — `generationId > 0` **with** a known member id.
8057    #[tokio::test]
8058    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testSendTxnOffsetsWithGroupMetadata")]
8059    async fn test_send_txn_offsets_with_group_metadata() {
8060        let mut ctx = TxnProducerContext::new(&[("transactional.id", TRANSACTIONAL_ID), ("max.block.ms", "10000")], 1);
8061        ctx.time.set_auto_tick_ms(1);
8062        let node = coordinator_node();
8063        ctx.sender.client_mut().throttle(&node, 5000);
8064        init_transactions(&mut ctx).await;
8065        ctx.producer.begin_transaction().expect("beginTransaction");
8066
8067        let group_metadata = ConsumerGroupMetadataImpl::with_generation_id_member_id_group_instance_id(
8068            "group",
8069            5,
8070            "member",
8071            Some("instance".to_string()),
8072        );
8073        drive(
8074            &mut ctx.sender,
8075            ctx.producer.send_offsets_to_transaction(HashMap::new(), &group_metadata),
8076        )
8077        .await
8078        .expect("a populated group metadata is valid and an empty offsets map is a no-op");
8079
8080        ctx.sender.client_mut().prepare_response(end_txn_response(Errors::None));
8081        drive(&mut ctx.sender, ctx.producer.commit_transaction())
8082            .await
8083            .expect("commitTransaction");
8084    }
8085
8086    /// Translated from
8087    /// `KafkaProducerTest.testInvalidGenerationIdAndMemberIdCombinedInSendOffsets`, which calls the `verifyInvalidGroupMetadata` helper
8088    /// with `new ConsumerGroupMetadata("group", 2, UNKNOWN_MEMBER_ID,
8089    /// Optional.empty())`.
8090    ///
8091    /// `KafkaProducerTest.testNullGroupMetadataInSendOffsets` is the
8092    /// other caller of that helper, passing `null`. It is **not translated**: the
8093    /// parameter is a `ConsumerGroupMetadata` value in Rust, so a null cannot be
8094    /// constructed and the arm it exercises
8095    /// (`KafkaProducer`) is enforced by the type system rather than by
8096    /// a runtime check. This is the same reasoning that dropped the arm from
8097    /// [`KafkaProducer::throw_if_invalid_group_metadata`].
8098    #[tokio::test]
8099    #[doc(
8100        alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testInvalidGenerationIdAndMemberIdCombinedInSendOffsets"
8101    )]
8102    async fn test_invalid_generation_id_and_member_id_combined_in_send_offsets() {
8103        let mut ctx = TxnProducerContext::new(&[("transactional.id", TRANSACTIONAL_ID), ("max.block.ms", "10000")], 1);
8104        ctx.time.set_auto_tick_ms(1);
8105        let node = coordinator_node();
8106        ctx.sender.client_mut().throttle(&node, 5000);
8107        init_transactions(&mut ctx).await;
8108        ctx.producer.begin_transaction().expect("beginTransaction");
8109
8110        let group_metadata = ConsumerGroupMetadataImpl::with_generation_id_member_id_group_instance_id(
8111            "group",
8112            2,
8113            crate::common::requests::TxnOffsetCommitRequest::UNKNOWN_MEMBER_ID,
8114            None,
8115        );
8116        let error = ctx
8117            .producer
8118            .send_offsets_to_transaction(HashMap::new(), &group_metadata)
8119            .await
8120            .expect_err("generationId > 0 with an unknown member id is rejected");
8121        assert_eq!(
8122            error.message(),
8123            format!(
8124                "Passed in group metadata {} has generationId > 0 but the member.id is unknown",
8125                group_metadata
8126            )
8127        );
8128    }
8129
8130    /// A `ProduceResponse` for one partition, mirroring
8131    /// `KafkaProducerTest.produceResponse(TopicIdPartition, long, Errors, int, int)`.
8132    fn produce_response(partition: i32, base_offset: i64, error: Errors, log_start_offset: i64) -> ConcreteResponse {
8133        use crate::ProduceResponseData;
8134        use crate::common::requests::ProduceResponse;
8135        use crate::produce_response_data::{PartitionProduceResponse, TopicProduceResponse};
8136
8137        let mut partition_response = PartitionProduceResponse::new();
8138        partition_response.set_index(partition);
8139        partition_response.set_base_offset(base_offset);
8140        partition_response.set_error_code(error.code());
8141        partition_response.set_log_start_offset(log_start_offset);
8142
8143        let mut topic_response = TopicProduceResponse::new();
8144        topic_response.set_topic_id(topic_id());
8145        topic_response.set_name(TOPIC.to_string());
8146        topic_response.set_partition_responses(vec![partition_response]);
8147
8148        let mut data = ProduceResponseData::new();
8149        data.set_responses(vec![topic_response]);
8150        ConcreteResponse::Produce(ProduceResponse::new(data))
8151    }
8152
8153    /// Java's `addOffsetsToTxnResponse(Errors error)`
8154    /// (`KafkaProducerTest#addOffsetsToTxnResponse`).
8155    fn add_offsets_to_txn_response(error: Errors) -> ConcreteResponse {
8156        use crate::AddOffsetsToTxnResponseData;
8157        use crate::common::requests::AddOffsetsToTxnResponse;
8158
8159        let mut data = AddOffsetsToTxnResponseData::new();
8160        data.set_error_code(error.code()).set_throttle_time_ms(10);
8161        ConcreteResponse::AddOffsetsToTxn(AddOffsetsToTxnResponse::new(data))
8162    }
8163
8164    /// Java's `txnOffsetsCommitResponse(Map<TopicPartition, Errors>)`
8165    /// (`KafkaProducerTest#txnOffsetsCommitResponse`).
8166    fn txn_offsets_commit_response(errors: &[(TopicPartition, Errors)]) -> ConcreteResponse {
8167        use crate::common::requests::TxnOffsetCommitResponse;
8168
8169        let error_map: HashMap<TopicPartition, Errors> = errors.iter().cloned().collect();
8170        ConcreteResponse::TxnOffsetCommit(TxnOffsetCommitResponse::with_request_throttle_ms_response_data(
8171            10, &error_map,
8172        ))
8173    }
8174
8175    /// Translated from `KafkaProducerTest.testTransactionV2Produce`.
8176    ///
8177    /// The full Transaction V2 round trip: `FindCoordinator`, `InitProducerId`, one
8178    /// produce, `EndTxn`. No `AddPartitionsToTxn` is prepared or expected — under
8179    /// KIP-890 the broker adds the partition implicitly, which is exactly what
8180    /// `maybeAddPartition`'s V2 arm (`TransactionManager#maybeAddPartition`) relies on.
8181    #[tokio::test]
8182    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testTransactionV2Produce")]
8183    async fn test_transaction_v2_produce() {
8184        let mut ctx = TxnProducerContext::transactional_v2(&[("transactional.id", "some-txn")], 1);
8185        ctx.time.set_auto_tick_ms(1);
8186        let node = coordinator_node();
8187        ctx.sender
8188            .client_mut()
8189            .prepare_response(find_coordinator_response(Errors::None, "some-txn", &node));
8190        ctx.sender
8191            .client_mut()
8192            .prepare_response(init_producer_id_response(Errors::None, PRODUCER_ID, EPOCH));
8193        drive(&mut ctx.sender, ctx.producer.init_transactions())
8194            .await
8195            .expect("initTransactions");
8196        assert!(
8197            ctx.transaction_manager.lock().unwrap().is_transaction_v2_enabled(),
8198            "initTransactions ends with maybeUpdateTransactionV2Enabled(true)"
8199        );
8200
8201        ctx.producer.begin_transaction().expect("beginTransaction");
8202        ctx.sender
8203            .client_mut()
8204            .prepare_response(produce_response(0, 1, Errors::None, 0));
8205        ctx.sender.client_mut().prepare_response(end_txn_response(Errors::None));
8206
8207        let record = ProducerRecord::with_partition_key(
8208            TOPIC.to_string(),
8209            Some(0),
8210            Some("key".to_string()),
8211            Some("value".to_string()),
8212        )
8213        .expect("a valid partition");
8214        let future = ctx.producer.send(record).await.expect("send");
8215        let metadata = drive(&mut ctx.sender, future.get()).await.expect("the produce succeeds");
8216        assert_eq!(metadata.offset(), 1);
8217
8218        drive(&mut ctx.sender, ctx.producer.commit_transaction())
8219            .await
8220            .expect("commitTransaction");
8221    }
8222
8223    /// Translated from
8224    /// `KafkaProducerTest.testTransactionV2ProduceWithConcurrentTransactionError`.
8225    ///
8226    /// As [`test_transaction_v2_produce`], but the first produce is answered with
8227    /// `CONCURRENT_TRANSACTIONS`. That is retriable, so the batch is re-enqueued and
8228    /// the second response completes it — which is what makes the commit that follows
8229    /// succeed rather than fail on an abortable error.
8230    #[tokio::test]
8231    #[doc(
8232        alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testTransactionV2ProduceWithConcurrentTransactionError"
8233    )]
8234    async fn test_transaction_v2_produce_with_concurrent_transaction_error() {
8235        let mut ctx = TxnProducerContext::transactional_v2(&[("transactional.id", "some-txn")], 1);
8236        ctx.time.set_auto_tick_ms(1);
8237        let node = coordinator_node();
8238        ctx.sender
8239            .client_mut()
8240            .prepare_response(find_coordinator_response(Errors::None, "some-txn", &node));
8241        ctx.sender
8242            .client_mut()
8243            .prepare_response(init_producer_id_response(Errors::None, PRODUCER_ID, EPOCH));
8244        drive(&mut ctx.sender, ctx.producer.init_transactions())
8245            .await
8246            .expect("initTransactions");
8247        ctx.producer.begin_transaction().expect("beginTransaction");
8248
8249        ctx.sender
8250            .client_mut()
8251            .prepare_response(produce_response(0, 1, Errors::ConcurrentTransactions, 0));
8252        ctx.sender
8253            .client_mut()
8254            .prepare_response(produce_response(0, 1, Errors::None, 0));
8255        ctx.sender.client_mut().prepare_response(end_txn_response(Errors::None));
8256
8257        let record = ProducerRecord::with_partition_key(
8258            TOPIC.to_string(),
8259            Some(0),
8260            Some("key".to_string()),
8261            Some("value".to_string()),
8262        )
8263        .expect("a valid partition");
8264        let future = ctx.producer.send(record).await.expect("send");
8265        let metadata = drive(&mut ctx.sender, future.get()).await.expect("the retry succeeds");
8266        assert_eq!(metadata.offset(), 1);
8267
8268        drive(&mut ctx.sender, ctx.producer.commit_transaction())
8269            .await
8270            .expect("commitTransaction");
8271    }
8272
8273    /// Translated from `KafkaProducerTest.testSendTxnOffsetsWithGroupIdTransactionV2`.
8274    ///
8275    /// With Transaction V2 the client skips `AddOffsetsToTxn` and sends the
8276    /// `TxnOffsetCommit` straight away (`TransactionManager`), after a
8277    /// `FindCoordinator` for the *group* coordinator. Java's prepared sequence says the
8278    /// same thing: `FindCoordinator`, `InitProducerId`, `FindCoordinator`,
8279    /// `TxnOffsetCommit`, `EndTxn` — five responses with no `AddOffsetsToTxn` between
8280    /// the second and third, unlike its V1 sibling.
8281    #[tokio::test]
8282    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testSendTxnOffsetsWithGroupIdTransactionV2")]
8283    async fn test_send_txn_offsets_with_group_id_transaction_v2() {
8284        let mut ctx = TxnProducerContext::transactional_v2(
8285            &[("transactional.id", TRANSACTIONAL_ID), ("max.block.ms", "10000")],
8286            1,
8287        );
8288        ctx.time.set_auto_tick_ms(1);
8289        let node = coordinator_node();
8290        ctx.sender.client_mut().throttle(&node, 5000);
8291        ctx.prepare_init_transactions(Errors::None, PRODUCER_ID, EPOCH);
8292        drive(&mut ctx.sender, ctx.producer.init_transactions())
8293            .await
8294            .expect("initTransactions");
8295        assert!(ctx.transaction_manager.lock().unwrap().is_transaction_v2_enabled());
8296        ctx.producer.begin_transaction().expect("beginTransaction");
8297
8298        const GROUP_ID: &str = "group";
8299        let partition = TopicPartition::new(TOPIC.to_string(), 0);
8300        ctx.sender
8301            .client_mut()
8302            .prepare_response(find_coordinator_response(Errors::None, GROUP_ID, &node));
8303        ctx.sender
8304            .client_mut()
8305            .prepare_response(txn_offsets_commit_response(&[(partition.clone(), Errors::None)]));
8306        ctx.sender.client_mut().prepare_response(end_txn_response(Errors::None));
8307
8308        let group_metadata = ConsumerGroupMetadataImpl::new(GROUP_ID);
8309        let offsets = HashMap::from([(partition, OffsetAndMetadata::new(5).expect("a non-negative offset"))]);
8310        drive(
8311            &mut ctx.sender,
8312            ctx.producer.send_offsets_to_transaction(offsets, &group_metadata),
8313        )
8314        .await
8315        .expect("sendOffsetsToTransaction");
8316
8317        drive(&mut ctx.sender, ctx.producer.commit_transaction())
8318            .await
8319            .expect("commitTransaction");
8320    }
8321
8322    /// What Java's `MockClient.poll` does for metadata (`MockClient#poll`):
8323    /// whenever an update is requested, apply `update` — the queued
8324    /// `prepareMetadataUpdate` response, or the current snapshot standing in for
8325    /// `updateWithCurrentMetadata`. This crate's `MockClient` has no metadata
8326    /// updater, so the tests that need one run this beside the operation.
8327    async fn with_metadata_responder<T>(
8328        metadata: &ProducerMetadata,
8329        time: &MockTime,
8330        update: &crate::common::requests::MetadataResponse,
8331        op: impl std::future::Future<Output = T>,
8332    ) -> T {
8333        let done = AtomicBool::new(false);
8334        let op = async {
8335            let out = op.await;
8336            done.store(true, Ordering::SeqCst);
8337            out
8338        };
8339        let responder = async {
8340            while !done.load(Ordering::SeqCst) {
8341                if metadata.update_requested() {
8342                    metadata.update_with_current_request_version(update, false, time.milliseconds());
8343                }
8344                tokio::task::yield_now().await;
8345            }
8346        };
8347        tokio::join!(op, responder).0
8348    }
8349
8350    /// The shared body of the two KIP-1319 `KafkaProducerTest` cases (6208dfc014,
8351    /// with 930ebc5608's frozen `MockTime`): a Transaction V2 producer whose
8352    /// metadata does not yet track [`TOPIC`] sends offsets for it. The refresh that
8353    /// `sendOffsetsToTransaction` triggers returns `refreshed`; the `TxnOffsetCommit`
8354    /// must then be at v6+ and carry the topic id.
8355    async fn send_offsets_to_transaction_negotiates_v6(initial_has_topic_id: bool) {
8356        let mut ctx = TxnProducerContext::with_metadata_seed(
8357            &[("transactional.id", "some.id"), ("max.block.ms", "10000")],
8358            1,
8359            true,
8360            false,
8361            initial_has_topic_id,
8362        );
8363        const GROUP_ID: &str = "group";
8364        let node = coordinator_node();
8365        let partition = TopicPartition::new(TOPIC.to_string(), 0);
8366
8367        ctx.sender
8368            .client_mut()
8369            .prepare_response(find_coordinator_response(Errors::None, "some.id", &node));
8370        ctx.sender
8371            .client_mut()
8372            .prepare_response(init_producer_id_response(Errors::None, 1, 5));
8373        ctx.sender
8374            .client_mut()
8375            .prepare_response(find_coordinator_response(Errors::None, "some.id", &node));
8376        ctx.sender.client_mut().prepare_response_matcher(
8377            Box::new(|request| {
8378                let crate::common::requests::AbstractRequest::TxnOffsetCommit(request) = request else {
8379                    panic!("expected a TxnOffsetCommit request, got {request}");
8380                };
8381                assert_eq!(request.data().group_id, GROUP_ID);
8382                assert!(
8383                    request.version() >= 6,
8384                    "Expected TxnOffsetCommit at v6+, got {}",
8385                    request.version()
8386                );
8387                assert_eq!(request.data().topics.len(), 1);
8388                assert_eq!(request.data().topics[0].topic_id, topic_id());
8389                true
8390            }),
8391            txn_offsets_commit_response(&[(partition.clone(), Errors::None)]),
8392        );
8393        ctx.sender.client_mut().prepare_response(end_txn_response(Errors::None));
8394
8395        drive(&mut ctx.sender, ctx.producer.init_transactions())
8396            .await
8397            .expect("initTransactions");
8398        ctx.producer.begin_transaction().expect("beginTransaction");
8399        assert!(!ctx.metadata.contains_topic(TOPIC), "the topic starts out untracked");
8400
8401        let refreshed = topic_metadata_update(1, true);
8402        let group_metadata = ConsumerGroupMetadataImpl::new(GROUP_ID);
8403        let offsets = HashMap::from([(partition, OffsetAndMetadata::new(5).expect("a non-negative offset"))]);
8404        let metadata = Arc::clone(&ctx.metadata);
8405        let time = Arc::clone(&ctx.time);
8406        drive(
8407            &mut ctx.sender,
8408            with_metadata_responder(
8409                &metadata,
8410                &time,
8411                &refreshed,
8412                ctx.producer.send_offsets_to_transaction(offsets, &group_metadata),
8413            ),
8414        )
8415        .await
8416        .expect("sendOffsetsToTransaction");
8417        assert!(ctx.metadata.contains_topic(TOPIC), "awaitTopicMetadata tracks the topic");
8418
8419        drive(&mut ctx.sender, ctx.producer.commit_transaction())
8420            .await
8421            .expect("commitTransaction");
8422    }
8423
8424    /// Translated from
8425    /// `KafkaProducerTest.testSendOffsetsToTransactionNegotiatesV6WhenMetadataKnowsTopicId`.
8426    #[tokio::test]
8427    #[doc(
8428        alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testSendOffsetsToTransactionNegotiatesV6WhenMetadataKnowsTopicId"
8429    )]
8430    async fn test_send_offsets_to_transaction_negotiates_v6_when_metadata_knows_topic_id() {
8431        send_offsets_to_transaction_negotiates_v6(true).await;
8432    }
8433
8434    /// Translated from
8435    /// `KafkaProducerTest.testSendOffsetsToTransactionTriggersMetadataRefreshThenNegotiatesV6`:
8436    /// the initial snapshot has the topic but no id; only the refresh
8437    /// `sendOffsetsToTransaction` triggers supplies it.
8438    #[tokio::test]
8439    #[doc(
8440        alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testSendOffsetsToTransactionTriggersMetadataRefreshThenNegotiatesV6"
8441    )]
8442    async fn test_send_offsets_to_transaction_triggers_metadata_refresh_then_negotiates_v6() {
8443        send_offsets_to_transaction_negotiates_v6(false).await;
8444    }
8445
8446    /// `awaitTopicMetadata` waits at most `max.block.ms` for the refresh it
8447    /// requests, and reports Java's `awaitUpdate` timeout when none arrives.
8448    ///
8449    /// `max.block.ms` also bounds the `initTransactions` that precedes the
8450    /// wait, and `drive` runs on real time: 500 ms, as the other tests that
8451    /// wait a timeout out use, leaves the mock round trips headroom when the
8452    /// whole suite runs under load, where 100 ms timed `initTransactions` out.
8453    #[tokio::test]
8454    async fn test_send_offsets_to_transaction_times_out_waiting_for_topic_metadata() {
8455        let mut ctx = TxnProducerContext::with_metadata_seed(
8456            &[("transactional.id", "some.id"), ("max.block.ms", "500")],
8457            1,
8458            true,
8459            false,
8460            false,
8461        );
8462        let node = coordinator_node();
8463        ctx.sender
8464            .client_mut()
8465            .prepare_response(find_coordinator_response(Errors::None, "some.id", &node));
8466        ctx.sender
8467            .client_mut()
8468            .prepare_response(init_producer_id_response(Errors::None, 1, 5));
8469        drive(&mut ctx.sender, ctx.producer.init_transactions())
8470            .await
8471            .expect("initTransactions");
8472        ctx.producer.begin_transaction().expect("beginTransaction");
8473
8474        let offsets = HashMap::from([(
8475            TopicPartition::new(TOPIC.to_string(), 0),
8476            OffsetAndMetadata::new(5).expect("a non-negative offset"),
8477        )]);
8478        let error = drive(
8479            &mut ctx.sender,
8480            ctx.producer
8481                .send_offsets_to_transaction(offsets, &ConsumerGroupMetadataImpl::new("group")),
8482        )
8483        .await
8484        .expect_err("no metadata update arrives");
8485        assert!(error.is_timeout_error(), "{error:?}");
8486        assert_eq!(error.message(), "Failed to update metadata after 500 ms.");
8487        assert!(
8488            !ctx.transaction_manager.lock().unwrap().has_pending_offset_commits(),
8489            "the manager is not reached when the metadata wait fails"
8490        );
8491    }
8492
8493    /// Reads a producer-level metric's current value. Translated from
8494    /// `KafkaProducerTest#getMetricValue`:
8495    /// `metrics.metric(metrics.metricName(name, "producer-metrics")).metricValue()`.
8496    fn get_metric_value(producer: &KafkaProducer<String, String>, name: &str) -> f64 {
8497        // `metric_value()` is a `Metric`-trait method (Java's `Metric.metricValue()`).
8498        use crate::common::Metric;
8499        let mn = producer.metrics.metric_name(name, "producer-metrics");
8500        producer
8501            .metrics
8502            .metric(&mn)
8503            .expect("metric present")
8504            .metric_value()
8505            .as_double()
8506            .expect("a double metric value")
8507    }
8508
8509    /// Translated from `KafkaProducerTest#getAndAssertDurationAtLeast`: reads the duration metric, asserts it is at least `floor`, and
8510    /// returns it.
8511    ///
8512    /// The producer times transaction control with its injected `time`, as Java
8513    /// does, so a test that installs `MockTime` with an auto-tick gets Java's
8514    /// durations: every `time.nanoseconds()` read advances the clock by one tick,
8515    /// and callers pass Java's `tick.toNanos()` floors unchanged.
8516    fn get_and_assert_duration_at_least(producer: &KafkaProducer<String, String>, name: &str, floor: f64) -> f64 {
8517        let value = get_metric_value(producer, name);
8518        assert!(value >= floor, "{name} duration {value} is below floor {floor}");
8519        value
8520    }
8521
8522    /// Translated from `KafkaProducerTest.assertDurationAtLeast`:
8523    /// asserts the duration is at least `floor`, discarding the value.
8524    fn assert_duration_at_least(producer: &KafkaProducer<String, String>, name: &str, floor: f64) {
8525        get_and_assert_duration_at_least(producer, name, floor);
8526    }
8527
8528    /// Translated from `KafkaProducerTest.testMeasureAbortTransactionDuration`.
8529    ///
8530    /// Runs two complete `beginTransaction` / `abortTransaction` cycles over one
8531    /// `initTransactions` — which proves `abortTransaction` leaves the manager in a
8532    /// state a *second* transaction can start from — and asserts, as Java does, that
8533    /// `txn-abort-time-ns-total` is positive after the first abort and strictly larger
8534    /// after the second. Java's `assertTrue(first > 0)` / `assertTrue(second > first)`
8535    /// carry over unchanged: the producer times with the injected `MockTime`, whose
8536    /// one-millisecond auto-tick (Java's `new MockTime(1)`) advances every read, so
8537    /// each abort records a strictly positive duration and the cumulative total grows.
8538    #[tokio::test]
8539    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testMeasureAbortTransactionDuration")]
8540    async fn test_measure_abort_transaction_duration() {
8541        let mut ctx = TxnProducerContext::transactional();
8542        ctx.time.set_auto_tick_ms(1);
8543        init_transactions(&mut ctx).await;
8544
8545        // Java captures `first` after abort #0 (`first > 0`) and asserts abort #1 is
8546        // `> first`. A zero-initialised `previous` folds both into one strict-growth
8547        // check: attempt 0 asserts `value > 0`, attempt 1 asserts `value > first`.
8548        let mut previous = 0.0_f64;
8549        for attempt in 0..2 {
8550            ctx.sender.client_mut().prepare_response(end_txn_response(Errors::None));
8551            ctx.producer
8552                .begin_transaction()
8553                .unwrap_or_else(|error| panic!("beginTransaction {}: {}", attempt, error));
8554            drive(&mut ctx.sender, ctx.producer.abort_transaction())
8555                .await
8556                .unwrap_or_else(|error| panic!("abortTransaction {}: {}", attempt, error));
8557
8558            let value = get_metric_value(&ctx.producer, "txn-abort-time-ns-total");
8559            assert!(
8560                value > previous,
8561                "txn-abort-time-ns-total must grow after abort {attempt}: {value} <= {previous}"
8562            );
8563            previous = value;
8564        }
8565    }
8566
8567    /// Translated from `KafkaProducerTest.testMeasureTransactionDurations`.
8568    ///
8569    /// Asserts every transaction-control latency sensor Java asserts —
8570    /// `txn-init-time-ns-total`, `txn-begin-time-ns-total`,
8571    /// `txn-send-offsets-time-ns-total`, `txn-commit-time-ns-total` — around a full V1
8572    /// offsets round trip run **twice** over one `initTransactions`: `AddOffsetsToTxn`,
8573    /// a `FindCoordinator` for the group, `TxnOffsetCommit`, `EndTxn` — and, on the
8574    /// second pass, no second `FindCoordinator`, because the group coordinator is
8575    /// already known. That asymmetry is Java's too (the second batch of prepared
8576    /// responses omits it).
8577    ///
8578    /// Round 1 captures each duration and asserts it is at least one tick; round 2
8579    /// asserts it grew by at least one more ([`get_and_assert_duration_at_least`] /
8580    /// [`assert_duration_at_least`]) — Java's `tick.toNanos()` floors, reachable
8581    /// because the producer times with the injected one-second-tick `MockTime`.
8582    ///
8583    /// Kafka 4.4 pre-tracks the topic (`metadata.add("topic", ..)`) so
8584    /// that `sendOffsetsToTransaction`'s KIP-1319 `awaitTopicMetadata` finds it
8585    /// already tracked and does not wait for a refresh, which under the one-second
8586    /// auto-tick would exhaust `max.block.ms`. [`TxnProducerContext::new`] tracks
8587    /// [`TOPIC`] the same way.
8588    #[tokio::test]
8589    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testMeasureTransactionDurations")]
8590    async fn test_measure_transaction_durations() {
8591        let mut ctx = TxnProducerContext::new(&[("transactional.id", TRANSACTIONAL_ID), ("max.block.ms", "10000")], 1);
8592        // Java's `new MockTime(Duration.ofSeconds(1).toMillis())` — a one-second tick
8593        // on every read of the clock the producer times transaction control with.
8594        const TICK_MS: i64 = 1000;
8595        // Java's `tick.toNanos()`.
8596        const TICK_NANOS: f64 = (TICK_MS * 1_000_000) as f64;
8597        ctx.time.set_auto_tick_ms(TICK_MS);
8598        init_transactions(&mut ctx).await;
8599
8600        // Java: assertDurationAtLeast(producer, "txn-init-time-ns-total", tick.toNanos()).
8601        assert_duration_at_least(&ctx.producer, "txn-init-time-ns-total", TICK_NANOS);
8602
8603        const GROUP_ID: &str = "group";
8604        let node = coordinator_node();
8605        let partition = TopicPartition::new(TOPIC.to_string(), 0);
8606
8607        // Round-1 captures; round 2's `first + tick` floors hold because these are
8608        // CumulativeSum sensors — the round-2 total is the round-1 total plus a
8609        // second delta of at least one tick.
8610        let mut begin_first = 0.0_f64;
8611        let mut send_off_first = 0.0_f64;
8612        let mut commit_first = 0.0_f64;
8613
8614        for (attempt, offset) in [(0usize, 5i64), (1, 10)] {
8615            ctx.sender
8616                .client_mut()
8617                .prepare_response(add_offsets_to_txn_response(Errors::None));
8618            if attempt == 0 {
8619                ctx.sender
8620                    .client_mut()
8621                    .prepare_response(find_coordinator_response(Errors::None, GROUP_ID, &node));
8622            }
8623            ctx.sender
8624                .client_mut()
8625                .prepare_response(txn_offsets_commit_response(&[(partition.clone(), Errors::None)]));
8626            ctx.sender.client_mut().prepare_response(end_txn_response(Errors::None));
8627
8628            ctx.producer
8629                .begin_transaction()
8630                .unwrap_or_else(|error| panic!("beginTransaction {}: {}", attempt, error));
8631            // Java: getAndAssertDurationAtLeast (round 1) / assertDurationAtLeast (round 2).
8632            if attempt == 0 {
8633                begin_first = get_and_assert_duration_at_least(&ctx.producer, "txn-begin-time-ns-total", TICK_NANOS);
8634            } else {
8635                assert_duration_at_least(&ctx.producer, "txn-begin-time-ns-total", begin_first + TICK_NANOS);
8636            }
8637
8638            let group_metadata = ConsumerGroupMetadataImpl::new(GROUP_ID);
8639            let offsets = HashMap::from([(
8640                partition.clone(),
8641                OffsetAndMetadata::new(offset).expect("a non-negative offset"),
8642            )]);
8643            drive(
8644                &mut ctx.sender,
8645                ctx.producer.send_offsets_to_transaction(offsets, &group_metadata),
8646            )
8647            .await
8648            .unwrap_or_else(|error| panic!("sendOffsetsToTransaction {}: {}", attempt, error));
8649            if attempt == 0 {
8650                send_off_first =
8651                    get_and_assert_duration_at_least(&ctx.producer, "txn-send-offsets-time-ns-total", TICK_NANOS);
8652            } else {
8653                assert_duration_at_least(&ctx.producer, "txn-send-offsets-time-ns-total", send_off_first + TICK_NANOS);
8654            }
8655
8656            drive(&mut ctx.sender, ctx.producer.commit_transaction())
8657                .await
8658                .unwrap_or_else(|error| panic!("commitTransaction {}: {}", attempt, error));
8659            if attempt == 0 {
8660                commit_first = get_and_assert_duration_at_least(&ctx.producer, "txn-commit-time-ns-total", TICK_NANOS);
8661            } else {
8662                assert_duration_at_least(&ctx.producer, "txn-commit-time-ns-total", commit_first + TICK_NANOS);
8663            }
8664        }
8665    }
8666
8667    /// A producer whose `Sender` is **spawned**, exactly as
8668    /// `KafkaProducer::with_client_options` does in production.
8669    ///
8670    /// Needed by the three `testCloseIsForcedOn*` methods, whose subject is the
8671    /// force-close path in `Sender::run`'s tail (`Sender#run`): it runs only
8672    /// once the run loop itself exits, which driving `run_once` cannot reach. The price
8673    /// is Java's — the `MockClient` moves into the task, so `prepare` queues every
8674    /// response up front and the test cannot inspect the mock afterwards.
8675    fn spawned_transactional_producer(
8676        extra: &[(&str, &str)],
8677        prepare: impl FnOnce(&mut MockClient),
8678    ) -> KafkaProducer<String, String> {
8679        spawned_transactional_producer_with_exit_hook(extra, prepare, None)
8680    }
8681
8682    /// As [`spawned_transactional_producer`], but runs `on_exit` on the Sender's own
8683    /// thread once `Sender::run` has returned.
8684    ///
8685    /// This is what makes "did `close` actually join the Sender?" observable — see
8686    /// [`test_close_joins_the_sender_after_forcing`].
8687    fn spawned_transactional_producer_with_exit_hook(
8688        extra: &[(&str, &str)],
8689        prepare: impl FnOnce(&mut MockClient),
8690        on_exit: Option<Arc<dyn Fn() + Send + Sync>>,
8691    ) -> KafkaProducer<String, String> {
8692        let mut props = HashMap::from([("bootstrap.servers".to_string(), "localhost:9000".to_string())]);
8693        for (key, value) in extra {
8694            props.insert((*key).to_string(), (*value).to_string());
8695        }
8696        let config = ProducerConfig::new(&props).expect("valid config");
8697        let log_context = LogContext::new(format!("[Producer clientId={}] ", config.client_id));
8698        let time = mock_time(1_000);
8699        let api_versions = Arc::new(ApiVersions::new());
8700
8701        let metadata = Arc::new(ProducerMetadata::with_log_context(
8702            config.retry_backoff_ms,
8703            config.retry_backoff_max_ms,
8704            config.metadata_max_age_ms,
8705            config.metadata_max_idle_ms,
8706            ClusterResourceListeners::new(),
8707            log_context.clone(),
8708        ));
8709        let transaction_manager = KafkaProducer::<String, String>::configure_transaction_state(
8710            &config,
8711            &api_versions,
8712            &metadata.metadata_arc(),
8713            &log_context,
8714        )
8715        .expect("these tests always enable idempotence");
8716        metadata.add(TOPIC, time.milliseconds());
8717        let update = crate::common::requests::RequestTestUtils::metadata_update_with(
8718            1,
8719            &HashMap::from([(TOPIC.to_string(), 1)]),
8720        );
8721        metadata.update_with_current_request_version(&update, false, time.milliseconds());
8722
8723        let batch_size = config.batch_size.max(1);
8724        let metrics = Arc::new(Metrics::new());
8725        let buffer_pool = Arc::new(BufferPool::new(
8726            config.buffer_memory,
8727            batch_size as usize,
8728            Arc::clone(&metrics),
8729            Arc::clone(&time) as Arc<dyn Time>,
8730            KafkaProducer::<String, String>::PRODUCER_METRIC_GROUP_NAME,
8731        ));
8732        let accumulator = Arc::new(RecordAccumulator::with_log_context(
8733            batch_size,
8734            Compression::of(config.compression_type).build(),
8735            KafkaProducer::<String, String>::linger_ms(&config),
8736            config.retry_backoff_ms,
8737            config.retry_backoff_max_ms,
8738            config.delivery_timeout_ms,
8739            PartitionerConfig::new(
8740                config.partitioner_adaptive_partitioning_enable,
8741                config.partitioner_availability_timeout_ms,
8742                config.partitioner_rack_aware,
8743                &config.client_rack,
8744            )
8745            .unwrap(),
8746            Arc::clone(&metrics),
8747            KafkaProducer::<String, String>::PRODUCER_METRIC_GROUP_NAME,
8748            Arc::clone(&time) as Arc<dyn Time>,
8749            buffer_pool,
8750            Some(Arc::clone(&transaction_manager)),
8751            log_context,
8752        ));
8753
8754        let mut client = MockClient::with_static_nodes(vec![coordinator_node()], Arc::clone(&time) as Arc<dyn Time>);
8755        prepare(&mut client);
8756        let wakeup = client.wakeup_notify();
8757        let running = Arc::new(AtomicBool::new(true));
8758        let force_close = Arc::new(AtomicBool::new(false));
8759        let pending_requests = Arc::new(Mutex::new(PendingRequests::new()));
8760
8761        let mut sender = Sender::new(
8762            client,
8763            Arc::clone(&metadata),
8764            Arc::clone(&accumulator),
8765            config.max_in_flight_requests_per_connection == 1,
8766            config.max_request_size,
8767            config.acks,
8768            config.retries,
8769            config.request_timeout_ms,
8770            config.retry_backoff_ms,
8771            SenderMetricsRegistry::new(Arc::clone(&metrics)),
8772            Arc::clone(&running),
8773            Arc::clone(&force_close),
8774            Arc::clone(&time) as Arc<dyn Time>,
8775            Some(Arc::clone(&transaction_manager)),
8776            Arc::clone(&pending_requests),
8777            LogContext::empty(),
8778        );
8779
8780        // # Why the Sender gets its own thread and runtime here
8781        //
8782        // `with_client_options` would `tokio::task::spawn` it onto the test's runtime, and that
8783        // deadlocks: `Sender::run` over a `MockClient` never awaits anything that is
8784        // pending — `MockClient::poll` returns immediately — so the task never yields.
8785        // In tokio only a worker parks on the time driver, and the sole awake worker is
8786        // then stuck inside that task, so **no timer in the whole runtime ever fires**:
8787        // the test's own `sleep` never returns and `close` is never called. (Diagnosed
8788        // from a thread sample: the second worker sitting in `park_condvar` while the
8789        // first spun in `run_once`.) A real `NetworkClient` cannot cause this, because
8790        // its `poll` awaits the selector.
8791        //
8792        // `spawn_blocking` plus a private current-thread runtime gives the Sender its
8793        // own OS thread and its own driver — which is also closer to Java, where the
8794        // Sender genuinely *is* a separate thread (`ioThread`). The returned handle is
8795        // still a `tokio::task::JoinHandle<()>`, so `close`'s join works unchanged.
8796        let sender_handle = tokio::task::spawn_blocking(move || {
8797            tokio::runtime::Builder::new_current_thread()
8798                .enable_all()
8799                .build()
8800                .expect("a current-thread runtime for the Sender")
8801                .block_on(sender.run());
8802            if let Some(on_exit) = on_exit {
8803                on_exit();
8804            }
8805        });
8806
8807        KafkaProducer::with_options(
8808            KafkaProducerOptionsBuilder::new()
8809                .set_config(&config)
8810                .set_key_serializer(Box::new(StringSerializer))
8811                .set_value_serializer(Box::new(StringSerializer))
8812                .set_metadata(metadata)
8813                .set_accumulator(accumulator)
8814                .set_running(running)
8815                .set_force_close(force_close)
8816                .set_wakeup(wakeup)
8817                .set_sender_handle(Some(sender_handle))
8818                .set_time(Arc::clone(&time) as Arc<dyn Time>)
8819                .set_transaction_manager(Some(transaction_manager))
8820                .set_pending_requests(pending_requests)
8821                .build()
8822                .expect("KafkaProducerOptionsBuilder::build: every mandatory parameter is set above"),
8823        )
8824    }
8825
8826    /// The body all three `testCloseIsForcedOn*` methods share: start
8827    /// `initTransactions` on another task, let its request go out, then `close` with a
8828    /// one-second timeout and assert `close` **returned** instead of blocking behind the
8829    /// pending request.
8830    ///
8831    /// Java writes it three times over, submitting to an `ExecutorService` and waiting
8832    /// on a `CountDownLatch`; a spawned task and its `JoinHandle` are the direct
8833    /// equivalent. `client.waitForRequests(1, 2000)` becomes a short sleep: the
8834    /// `MockClient` has moved into the Sender task, so its request count is no longer
8835    /// observable from here.
8836    ///
8837    /// # What is asserted, and why not more
8838    ///
8839    /// Java's last line is `assertionDoneLatch.await(5000, MILLISECONDS)` and it
8840    /// **discards the boolean result**, so the test does not in fact require
8841    /// `initTransactions` to have returned by then — and it cannot have, on either
8842    /// side: the request that is in flight when `close` runs was already dequeued from
8843    /// `pendingRequests` by `nextRequest`, so `TransactionManager.close`'s
8844    /// `pendingRequests.forEach(handler -> handler.fail(..))`
8845    /// (`TransactionManager#close`) has nothing to fail, and neither
8846    /// `NetworkClient.close` nor `MockClient.close` runs completion handlers. The
8847    /// caller is released by its own `max.block.ms` instead, 60 s later by default.
8848    ///
8849    /// So the assertion is the one the test names: `close` is *forced* — it returns
8850    /// within its own timeout rather than waiting on a request that will never be
8851    /// answered. The `initTransactions` task is then given a bounded window purely so
8852    /// a run that *does* complete has its error inspected, exactly as far as Java goes.
8853    ///
8854    /// `multi_thread` is required at the call sites. `Sender::run` over a `MockClient`
8855    /// never blocks — `MockClient::poll` returns immediately — so on the default
8856    /// current-thread runtime it would starve the task doing the closing.
8857    async fn assert_close_forces_pending_transactional_request(producer: KafkaProducer<String, String>) {
8858        let producer = Arc::new(producer);
8859        let init = {
8860            let producer = Arc::clone(&producer);
8861            tokio::task::spawn(async move { producer.init_transactions().await })
8862        };
8863
8864        tokio::time::sleep(Duration::from_millis(200)).await;
8865        let started = std::time::Instant::now();
8866        producer.close_with_timeout(Duration::from_millis(1000)).await.expect("close");
8867        let elapsed = started.elapsed();
8868        assert!(
8869            elapsed < Duration::from_secs(5),
8870            "close must be forced after its 1000 ms timeout, not blocked behind the \
8871             pending transactional request; it took {:?}",
8872            elapsed
8873        );
8874
8875        // Java's ignored `assertionDoneLatch.await(5000, ..)`. If the call did return,
8876        // it must have returned an error — never a successful initTransactions.
8877        if let Ok(joined) = tokio::time::timeout(Duration::from_millis(500), init).await {
8878            let result = joined.expect("the initTransactions task did not panic");
8879            assert!(result.is_err(), "initTransactions cannot succeed once the producer is closed");
8880        }
8881    }
8882
8883    // -- Rust-side regression tests, no Java counterpart --------------------
8884    //
8885    // `close`'s join is a Rust-only failure mode: Java's `ioThread.join()` cannot "lose"
8886    // its thread, whereas `tokio::time::timeout(t, handle)` consumes the `JoinHandle` and
8887    // drops it on expiry. Both tests below exist to make reverting
8888    // `await_sender_handle`'s `&mut` fail — which Critic 46 issue 6 showed the three
8889    // translated `testCloseIsForcedOn*` tests do **not**: with the bug restored,
8890    // `await_sender_handle_indefinitely` finds `None` and returns *sooner*, so every
8891    // assertion in those tests still passes.
8892
8893    /// The mechanism: an expired [`KafkaProducer::await_sender_handle`] must leave the
8894    /// handle in place for [`KafkaProducer::await_sender_handle_indefinitely`] to join.
8895    ///
8896    /// Java's `close` force-closes and *then* joins unconditionally
8897    /// (`KafkaProducer#close`), and CLAUDE.md §11.4 requires the Rust
8898    /// translation to await the handle rather than merely signal it. Passing the handle
8899    /// by value into `tokio::time::timeout` breaks that silently: `timeout` takes
8900    /// ownership and drops it on `Elapsed`.
8901    #[tokio::test]
8902    async fn test_await_sender_handle_keeps_the_handle_when_it_expires() {
8903        let ctx = TxnProducerContext::transactional();
8904
8905        // A task that outlives the wait below, so the wait is guaranteed to expire.
8906        let handle = tokio::task::spawn(async { tokio::time::sleep(Duration::from_secs(30)).await });
8907        *ctx.producer.sender_handle.lock().unwrap() = Some(handle);
8908
8909        let completed = ctx.producer.await_sender_handle(Duration::from_millis(50)).await;
8910        assert!(!completed, "the task outlives the wait, so it cannot have completed");
8911        assert!(
8912            ctx.producer.sender_handle.lock().unwrap().is_some(),
8913            "an expired wait must put the handle back — otherwise close's later \
8914             unconditional join has nothing to join and returns while the Sender runs"
8915        );
8916
8917        // And the retained handle is still the live one: aborting the task and waiting
8918        // again resolves against it, rather than short-circuiting on `None`.
8919        ctx.producer.sender_handle.lock().unwrap().as_ref().expect("retained").abort();
8920        assert!(
8921            ctx.producer.await_sender_handle(Duration::from_secs(5)).await,
8922            "the retained handle must be awaitable, not a husk"
8923        );
8924    }
8925
8926    /// The contract: `close_with_timeout` must not return before the Sender has finished.
8927    ///
8928    /// Same setup as the three `testCloseIsForcedOn*` translations — a transactional
8929    /// request left in flight so the graceful wait expires and `close` force-closes —
8930    /// but with the Sender's exit made *observable*: the harness runs an exit hook on
8931    /// the Sender's own thread once `Sender::run` returns, after a deliberate 300 ms of
8932    /// shutdown cost.
8933    ///
8934    /// The sleep is instrumentation, not padding. Real shutdown work takes time (the
8935    /// force-close path fails pending requests, aborts batches, closes the client), but
8936    /// with a `MockClient` it is instant, which would leave the assertion racing the task
8937    /// instead of observing the join. 300 ms against a 1000 ms graceful timeout is a
8938    /// wide, one-sided margin: with the join the flag is necessarily set; without it
8939    /// `close` returns ~300 ms early.
8940    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8941    async fn test_close_joins_the_sender_after_forcing() {
8942        let exited = Arc::new(AtomicBool::new(false));
8943        let producer = {
8944            let exited = Arc::clone(&exited);
8945            spawned_transactional_producer_with_exit_hook(
8946                &[("transactional.id", "this-is-a-transactional-id")],
8947                |_client| {},
8948                Some(Arc::new(move || {
8949                    std::thread::sleep(Duration::from_millis(300));
8950                    exited.store(true, Ordering::SeqCst);
8951                })),
8952            )
8953        };
8954
8955        let producer = Arc::new(producer);
8956        let init = {
8957            let producer = Arc::clone(&producer);
8958            tokio::task::spawn(async move { producer.init_transactions().await })
8959        };
8960        tokio::time::sleep(Duration::from_millis(200)).await;
8961
8962        producer.close_with_timeout(Duration::from_millis(1000)).await.expect("close");
8963        assert!(
8964            exited.load(Ordering::SeqCst),
8965            "close returned before the Sender task finished: the graceful wait expired, \
8966             so close force-closed and must then have joined the handle \
8967             (the `ioThread.join()` after the force close in `KafkaProducer#close(Duration, boolean)`, \
8968             CLAUDE.md §11.4)"
8969        );
8970
8971        init.abort();
8972    }
8973
8974    /// Regression test with no Java counterpart: `close_with_timeout(Duration::MAX)`
8975    /// must deliver the records buffered in the accumulator before returning, as Java's
8976    /// `close(Duration.ofMillis(Long.MAX_VALUE))` does (`KafkaProducer#close()`).
8977    ///
8978    /// A 30 s `linger.ms` with a frozen clock keeps the record buffered, so only a
8979    /// graceful close can release it: `initiate_close` closes the accumulator, which
8980    /// makes the batch ready, and the Sender's shutdown loop drains it before exiting
8981    /// (`Sender#run`).
8982    ///
8983    /// The Sender runs on a dedicated thread and runtime, for the reason given in
8984    /// [`spawned_transactional_producer_with_exit_hook`].
8985    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8986    async fn test_close_with_duration_max_delivers_buffered_records() {
8987        // Idempotent (the default) rather than transactional: the Sender requires only
8988        // a producer id before draining the batch.
8989        let mut ctx = TxnProducerContext::new(&[("linger.ms", "30000")], 1);
8990        ctx.sender
8991            .client_mut()
8992            .prepare_response(init_producer_id_response(Errors::None, PRODUCER_ID, EPOCH));
8993        ctx.sender
8994            .client_mut()
8995            .prepare_response(produce_response(0, 1, Errors::None, 0));
8996
8997        let record = ProducerRecord::with_partition_key(
8998            TOPIC.to_string(),
8999            Some(0),
9000            Some("key".to_string()),
9001            Some("value".to_string()),
9002        )
9003        .expect("a valid partition");
9004        let future = ctx.producer.send(record).await.expect("send");
9005        assert!(!future.is_done(), "linger.ms keeps the record buffered until close");
9006
9007        let TxnProducerContext { producer, mut sender, .. } = ctx;
9008        let sender_handle = tokio::task::spawn_blocking(move || {
9009            tokio::runtime::Builder::new_current_thread()
9010                .enable_all()
9011                .build()
9012                .expect("a current-thread runtime for the Sender")
9013                .block_on(sender.run());
9014        });
9015        *producer.sender_handle.lock().unwrap() = Some(sender_handle);
9016
9017        tokio::time::timeout(Duration::from_secs(10), producer.close_with_timeout(Duration::MAX))
9018            .await
9019            .expect("close returns once the buffered record is delivered")
9020            .expect("close");
9021
9022        assert!(future.is_done(), "close returned before the buffered record completed");
9023        let metadata = future.get().await.expect("the record is delivered, not aborted");
9024        assert_eq!(metadata.offset(), 1);
9025    }
9026
9027    /// Translated from `KafkaProducerTest.testTransactionalMethodThrowsWhenSenderClosed`.
9028    #[tokio::test]
9029    async fn test_transactional_method_throws_when_sender_closed() {
9030        let ctx = TxnProducerContext::new(&[("transactional.id", "this-is-a-transactional-id")], 1);
9031        ctx.producer.close().await.expect("close");
9032        let error = ctx.producer.init_transactions().await.expect_err("the producer is closed");
9033        assert_eq!(error.message(), "Cannot perform operation after producer has been closed");
9034    }
9035
9036    /// Translated from `KafkaProducerTest.testCloseIsForcedOnPendingFindCoordinator`.
9037    ///
9038    /// No response is prepared, so the `FindCoordinator` is the request left in flight
9039    /// when `close` runs.
9040    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9041    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testCloseIsForcedOnPendingFindCoordinator")]
9042    async fn test_close_is_forced_on_pending_find_coordinator() {
9043        let producer =
9044            spawned_transactional_producer(&[("transactional.id", "this-is-a-transactional-id")], |_client| {});
9045        assert_close_forces_pending_transactional_request(producer).await;
9046    }
9047
9048    /// Translated from `KafkaProducerTest.testCloseIsForcedOnPendingInitProducerId`.
9049    ///
9050    /// The `FindCoordinator` is answered, so the `InitProducerId` is the request left
9051    /// in flight.
9052    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9053    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testCloseIsForcedOnPendingInitProducerId")]
9054    async fn test_close_is_forced_on_pending_init_producer_id() {
9055        let producer =
9056            spawned_transactional_producer(&[("transactional.id", "this-is-a-transactional-id")], |client| {
9057                client.prepare_response(find_coordinator_response(
9058                    Errors::None,
9059                    "this-is-a-transactional-id",
9060                    &coordinator_node(),
9061                ));
9062            });
9063        assert_close_forces_pending_transactional_request(producer).await;
9064    }
9065
9066    /// Translated from `KafkaProducerTest.testCloseIsForcedOnPendingAddOffsetRequest`.
9067    ///
9068    /// In Apache Kafka 4.2 this method's body is **identical** to
9069    /// `testCloseIsForcedOnPendingInitProducerId`'s — it prepares one
9070    /// `FindCoordinator` and submits `initTransactions`, never reaching an
9071    /// `AddOffsetsToTxn` despite the name. Translated as written rather than
9072    /// "corrected": inventing the `sendOffsetsToTransaction` the name implies would be
9073    /// a different test from the one Java runs.
9074    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9075    #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testCloseIsForcedOnPendingAddOffsetRequest")]
9076    async fn test_close_is_forced_on_pending_add_offset_request() {
9077        let producer =
9078            spawned_transactional_producer(&[("transactional.id", "this-is-a-transactional-id")], |client| {
9079                client.prepare_response(find_coordinator_response(
9080                    Errors::None,
9081                    "this-is-a-transactional-id",
9082                    &coordinator_node(),
9083                ));
9084            });
9085        assert_close_forces_pending_transactional_request(producer).await;
9086    }
9087
9088    // =====================================================================
9089    // PHASE-6 TEST ACCOUNTING — the transactional `KafkaProducerTest` methods
9090    //
9091    // SCOPE CRITERION. A `KafkaProducerTest.java` method is in scope for Phase 6 iff its
9092    // body (or a helper it calls) mentions one of the five public transactional methods,
9093    // `TRANSACTIONAL_ID_CONFIG`, `TransactionManager`, `maybeAddPartition`, or the
9094    // `verifyInvalidGroupMetadata` helper. That is the marker set below, exactly.
9095    //
9096    // Two corrections to the criterion as first written (Critic 46 issue 4):
9097    //
9098    //   - It said "one of the **three** transactional config keys" while the marker set
9099    //     held one. The narrow *program* was right and the prose wrong: adding
9100    //     `ENABLE_IDEMPOTENCE_CONFIG` yields 36 rows and drags in 8 plainly
9101    //     non-transactional tests that merely set `enable.idempotence=false`
9102    //     (`testMetadataFetch`, `testMetadataExpiry`, `testMetadataTimeoutWith*`,
9103    //     `testMetadataWithPartitionOutOfRange`, `testTopicRefreshInMetadata`,
9104    //     `testFlushCompleteSendOfInflightBatches`, `shouldNotInvokeFlushInCallback`).
9105    //     "Three" described no realizable marker set, so the prose now matches the
9106    //     program.
9107    //   - `TransactionManager` / `maybeAddPartition` were **missing**, which made every
9108    //     mock-injected transactional test invisible. `testPartitionAddedToTransaction`
9109    //     reaches the transactional path only through
9110    //     `KafkaProducerTestContext`'s `mock(TransactionManager.class)`
9111    //     (`KafkaProducerTest.KafkaProducerTestContext`), so it named no marker at all and was absent from the denominator.
9112    //     It was also a real coverage gap, not only bookkeeping: `maybe_add_partition`'s
9113    //     production call site in this file had no test.
9114    //
9115    // The classifier lesson generalises, and is the same one the sibling `sender.rs`
9116    // block now records: a completeness check is only as strong as the assumption its
9117    // classifier makes, and when the Java side and the Rust side share that assumption
9118    // the resulting diff is **vacuous** for whatever they agree to ignore. Here the
9119    // shared assumption was "a transactional test names a transactional symbol", which a
9120    // Mockito-injected test does not.
9121    //
9122    // DERIVATION. The splitter counts braces rather than matching a declaration
9123    // regexp, because several of these bodies contain anonymous classes and lambdas
9124    // whose members would otherwise end a block early. Run from the repo root; awk
9125    // version 20200816, exit 0:
9126    //
9127    //   T=kafka/clients/src/test/java/org/apache/kafka/clients/producer/KafkaProducerTest.java
9128    //   M='initTransactions,beginTransaction,commitTransaction,abortTransaction,'
9129    //   M="$M"'sendOffsetsToTransaction,verifyInvalidGroupMetadata,TRANSACTIONAL_ID_CONFIG,'
9130    //   M="$M"'TransactionManager,maybeAddPartition'
9131    //   awk -v MARKERS="$M" '
9132    //     BEGIN { n = split(MARKERS, m, ","); depth = 0; inm = 0 }
9133    //     {
9134    //       line = $0
9135    //       if (depth == 1 && !inm && line ~ /^    [a-zA-Z@<].*\(.*\{[ \t]*$/ &&
9136    //           line !~ /^    (class|enum|interface|static \{)/) {
9137    //         match(line, /[a-zA-Z0-9_]+\(/)
9138    //         if (RSTART > 0) { name = substr(line, RSTART, RLENGTH-1); start = NR; inm = 1; delete hard }
9139    //       }
9140    //       if (inm) { for (i = 1; i <= n; i++) if (index(line, m[i]) > 0) hard[m[i]] = 1 }
9141    //       o = gsub(/\{/, "{", line); c = gsub(/\}/, "}", line); depth += o - c
9142    //       if (inm && depth <= 1) {
9143    //         hits = ""
9144    //         for (i = 1; i <= n; i++) if (m[i] in hard) hits = hits (hits == "" ? "" : "+") m[i]
9145    //         if (hits != "") printf "%d\t%s\n", start, name
9146    //         inm = 0
9147    //       }
9148    //     }' "$T"
9149    //
9150    // It prints **29** rows. The inline `for (i = 1; i <= n; i++)` walks `MARKERS` in
9151    // declaration order rather than `for (k in hard)`, so the output is reproducible on
9152    // any awk (the sibling blocks in `sender.rs` / `transaction_manager.rs` record why
9153    // that matters). An earlier revision credited an `emit` function for that property;
9154    // there is none in *this* program — the logic is inline, and `emit` belongs to the
9155    // sibling blocks (Critic 46 issue 4).
9156    //
9157    // ARITHMETIC. 29 printed rows = 1 helper (`verifyInvalidGroupMetadata`, 2000 — not a
9158    // test method) + **28** test methods. §Phase-6 says 27; 28 is the corrected
9159    // denominator, the extra being `testPartitionAddedToTransaction` (2423; Critic 46 issue 4).
9160    // Those 28 partition with no overlap into
9161    //
9162    //   23 translated here
9163    // +  1 not translated, justified (`testNullGroupMetadataInSendOffsets`)
9164    // +  4 translated in Phase 1, in `producer_config.rs`
9165    // = 28.
9166    //
9167    // Each of the three groups is listed in full below, so the sum can be checked
9168    // against the lists rather than taken on trust.
9169    //
9170    // TRANSLATED HERE (23 Rust tests):
9171    //   1290 testInitTransactionsResponseAfterTimeout
9172    //          -> test_init_transactions_response_after_timeout
9173    //   1329 testInitTransactionTimeout               -> test_init_transaction_timeout
9174    //   1364 testInitTransactionWhileThrottled        -> test_init_transaction_while_throttled
9175    //   1390 testClusterAuthorizationFailure          -> test_cluster_authorization_failure
9176    //   1419 testAbortTransaction                     -> test_abort_transaction
9177    //   1444 testTransactionV2ProduceWithConcurrentTransactionError
9178    //          -> test_transaction_v2_produce_with_concurrent_transaction_error
9179    //   1503 testMeasureAbortTransactionDuration      -> test_measure_abort_transaction_duration
9180    //   1533 testCommitTransactionWithRecordTooLargeException
9181    //          -> test_commit_transaction_with_record_too_large_error
9182    //   1563 testCommitTransactionWithMetadataTimeoutForMissingTopic
9183    //          -> test_commit_transaction_with_metadata_timeout_for_missing_topic
9184    //   1600 testCommitTransactionWithMetadataTimeoutForPartitionOutOfRange
9185    //          -> test_commit_transaction_with_metadata_timeout_for_partition_out_of_range
9186    //   1637 testCommitTransactionWithSendToInvalidTopic
9187    //          -> test_commit_transaction_with_send_to_invalid_topic
9188    //   1677 testSendTxnOffsetsWithGroupId            -> test_send_txn_offsets_with_group_id
9189    //   1715 testSendTxnOffsetsWithGroupIdTransactionV2
9190    //          -> test_send_txn_offsets_with_group_id_transaction_v2
9191    //   1772 testTransactionV2Produce                 -> test_transaction_v2_produce
9192    //   1842 testMeasureTransactionDurations          -> test_measure_transaction_durations
9193    //   1895 testSendTxnOffsetsWithGroupMetadata      -> test_send_txn_offsets_with_group_metadata
9194    //   1950 testInvalidGenerationIdAndMemberIdCombinedInSendOffsets
9195    //          -> test_invalid_generation_id_and_member_id_combined_in_send_offsets
9196    //   2054 testOnlyCanExecuteCloseAfterInitTransactionsTimeout
9197    //          -> test_only_can_execute_close_after_init_transactions_timeout
9198    //   2163 testTransactionalMethodThrowsWhenSenderClosed
9199    //          -> test_transactional_method_throws_when_sender_closed
9200    //   2182 testCloseIsForcedOnPendingFindCoordinator
9201    //          -> test_close_is_forced_on_pending_find_coordinator
9202    //   2210 testCloseIsForcedOnPendingInitProducerId
9203    //          -> test_close_is_forced_on_pending_init_producer_id
9204    //   2239 testCloseIsForcedOnPendingAddOffsetRequest
9205    //          -> test_close_is_forced_on_pending_add_offset_request
9206    //   2423 testPartitionAddedToTransaction         -> test_partition_added_to_transaction
9207    //          The mock-injected one. Translated with a real manager and
9208    //          `is_partition_pending_add` in place of Mockito's `verify`; the test's own
9209    //          rustdoc argues why that is stronger rather than weaker.
9210    //
9211    // NOT TRANSLATED, JUSTIFIED (1):
9212    //   1944 testNullGroupMetadataInSendOffsets — passes `null` for the
9213    //     `ConsumerGroupMetadata`. The Rust parameter is a value, so the argument
9214    //     cannot be constructed and the arm it exercises
9215    //     (`KafkaProducer`) is enforced by the type system instead of a
9216    //     runtime check. Recorded again on
9217    //     `test_invalid_generation_id_and_member_id_combined_in_send_offsets`, the
9218    //     other caller of the same Java helper, which *is* translated.
9219    //
9220    // TRANSLATED IN PHASE 1, in `producer_config.rs` (4): these reference
9221    // `TRANSACTIONAL_ID_CONFIG` only as an input to
9222    // `postProcessAndValidateIdempotenceConfigs`, so they belong to `ProducerConfig`,
9223    // not to this file. Verified present, not assumed:
9224    //
9225    //   $ grep -c 'fn test_overwrite_acks_and_retries_for_idempotent_producers\|fn test_acks_and_idempotence_for_idempotent_producers\|fn test_retries_and_idempotence_for_idempotent_producers\|fn test_inflight_requests_and_idempotence_for_idempotent_producers' src/producer/producer_config.rs
9226    //   4
9227    //
9228    //   222 testOverwriteAcksAndRetriesForIdempotentProducers
9229    //   238 testAcksAndIdempotenceForIdempotentProducers
9230    //   341 testRetriesAndIdempotenceForIdempotentProducers
9231    //   413 testInflightRequestsAndIdempotenceForIdempotentProducers
9232    //
9233    // TRANSLATED IN FULL, ASSERTIONS INCLUDED (2 methods):
9234    // `testMeasureAbortTransactionDuration` and `testMeasureTransactionDurations`.
9235    // `KafkaProducerMetrics` and `org.apache.kafka.common.metrics` are translated and
9236    // wired (`KafkaProducer::{init_transactions, begin_transaction,
9237    // send_offsets_to_transaction, commit_transaction, abort_transaction}` now call the
9238    // matching `record_*` sensor), so every `getMetricValue(producer,
9239    // "txn-*-time-ns-total")` assertion is translated via `get_metric_value` /
9240    // `get_and_assert_duration_at_least` / `assert_duration_at_least`, against Java's
9241    // own `tick.toNanos()` floors: the producer times with the injected `MockTime`.
9242    // =====================================================================
9243
9244    // -- `configureTransactionState` tests ----------------------------------
9245    //
9246    // These began life covering the temporary MILESTONE-11 GUARD in `new`.
9247    // Phase 4 turned the idempotence cases from rejections into constructions, and
9248    // Phase 6 removed the guard's last (transactional) arm — so what they now cover
9249    // is `configureTransactionState` across its three
9250    // outcomes: no manager, an idempotent manager, and a transactional manager.
9251
9252    fn guard_props(extra: &[(&str, &str)]) -> HashMap<String, String> {
9253        let mut props = HashMap::from([("bootstrap.servers".to_string(), "localhost:9999".to_string())]);
9254        for (key, value) in extra {
9255            props.insert((*key).to_string(), (*value).to_string());
9256        }
9257        props
9258    }
9259
9260    fn from_guard_props(props: &HashMap<String, String>) -> Result<(), Error> {
9261        let config = ProducerConfig::new(props)?;
9262        KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer).map(|_| ())
9263    }
9264
9265    /// KIP-1123 (KAFKA-19193): `partitioner.rack.aware` and `client.rack` reach the
9266    /// accumulator's `PartitionerConfig` (`KafkaProducer#KafkaProducer`), and a blank
9267    /// rack in rack-aware mode fails construction with `PartitionerConfig`'s
9268    /// `ConfigException`, wrapped like every other construction failure. Java checks
9269    /// it even with a custom partitioner, so the `RoundRobinPartitioner` case fails
9270    /// too.
9271    #[tokio::test]
9272    async fn test_partitioner_rack_awareness_wiring() {
9273        let props = guard_props(&[("partitioner.rack.aware", "true"), ("client.rack", "rack0")]);
9274        let config = ProducerConfig::new(&props).expect("valid config");
9275        let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
9276            .expect("a rack-aware producer with a rack constructs");
9277        assert_eq!((true, "rack0"), producer.accumulator.partitioner_rack_for_test());
9278
9279        let props = guard_props(&[("client.rack", "rack0")]);
9280        let config = ProducerConfig::new(&props).expect("valid config");
9281        let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
9282            .expect("default config constructs");
9283        assert_eq!((false, "rack0"), producer.accumulator.partitioner_rack_for_test());
9284
9285        for partitioner_type in [None, Some(ProducerConfig::ROUND_ROBIN_PARTITIONER)] {
9286            let mut extra = vec![("partitioner.rack.aware", "true")];
9287            if let Some(name) = partitioner_type {
9288                extra.push(("partitioner.type", name));
9289            }
9290            let config = ProducerConfig::new(&guard_props(&extra)).expect("valid config");
9291            let error = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
9292                .expect_err("rack-aware without client.rack must fail construction");
9293            assert_eq!(error.message(), "Failed to construct kafka producer", "{partitioner_type:?}");
9294            let cause: &Error = std::error::Error::source(&error)
9295                .and_then(|e| e.downcast_ref::<Error>())
9296                .expect("the cause is a crate Error");
9297            assert!(matches!(cause, Error::Config(_)), "got {cause:?}");
9298            assert_eq!(
9299                cause.message(),
9300                "client.rack must be provided if partitioner.rack.aware is enabled"
9301            );
9302        }
9303    }
9304
9305    /// KAFKA-2121 for the KIP-1123 check: a custom partitioner is configured before
9306    /// `PartitionerConfig` rejects a blank `client.rack`, so the failed construction
9307    /// must close it, as Java's constructor `catch` -> `close(0, true)` does
9308    /// (`KafkaProducer#KafkaProducer` -> `KafkaProducer#close`). Critic 96 F1.
9309    #[test]
9310    fn test_partitioner_closed_when_construction_fails_after_configure() {
9311        let _guard = MockPartitioner::lock_counters();
9312        MockPartitioner::reset_counters();
9313
9314        let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap();
9315        runtime.block_on(async {
9316            let partitioner: Box<dyn Partitioner<String, String>> = Box::new(MockPartitioner::new());
9317            let config = ProducerConfig::new(&guard_props(&[("partitioner.rack.aware", "true")]))
9318                .expect("valid config")
9319                .set_partitioner(partitioner);
9320            let init_count = AtomicUsize::new(0);
9321            let serializer_closes = Arc::new(AtomicUsize::new(0));
9322            let error = KafkaProducer::<String, String>::new(
9323                config,
9324                MockSerializer::new(&init_count, &serializer_closes),
9325                MockSerializer::new(&init_count, &serializer_closes),
9326            )
9327            .expect_err("rack-aware without client.rack must fail construction");
9328            assert_eq!(error.message(), "Failed to construct kafka producer");
9329
9330            assert_eq!(1, MockPartitioner::init_count().load(Ordering::SeqCst));
9331            assert_eq!(1, MockPartitioner::close_count().load(Ordering::SeqCst));
9332            // `close(Duration.ofMillis(0), true)` closes the serializers too
9333            // (`KafkaProducer#close`).
9334            assert_eq!(2, serializer_closes.load(Ordering::SeqCst));
9335        });
9336
9337        MockPartitioner::reset_counters();
9338    }
9339
9340    /// KIP-1332 (`KafkaProducer#KafkaProducer`): `buffer.memory.allocation.strategy=incremental`
9341    /// (any case) builds a `ChunkedRecordAccumulator` over a pool serving 16 KiB chunks when
9342    /// `batch.size` is at least one chunk, and falls back to the full strategy, with a pool of
9343    /// `batch.size` buffers, when it is not. The default is the full strategy.
9344    #[tokio::test]
9345    async fn test_buffer_memory_allocation_strategy_wiring() {
9346        use crate::producer::internals::buffer_pool::AllocationMode;
9347        type Case<'a> = (&'a [(&'a str, &'a str)], bool, usize);
9348        let cases: [Case; 5] = [
9349            (&[], false, 16384),
9350            (&[("buffer.memory.allocation.strategy", "incremental")], true, 16384),
9351            (
9352                &[
9353                    ("buffer.memory.allocation.strategy", "Incremental"),
9354                    ("batch.size", "1048576"),
9355                ],
9356                true,
9357                16384,
9358            ),
9359            // Below one chunk: the warning, then the full strategy.
9360            (
9361                &[
9362                    ("buffer.memory.allocation.strategy", "incremental"),
9363                    ("batch.size", "16383"),
9364                ],
9365                false,
9366                16383,
9367            ),
9368            (
9369                &[
9370                    ("buffer.memory.allocation.strategy", "incremental"),
9371                    ("batch.size", "0"),
9372                ],
9373                false,
9374                1,
9375            ),
9376        ];
9377        for (extra, incremental, poolable_size) in cases {
9378            let config = ProducerConfig::new(&guard_props(extra)).expect("valid config");
9379            let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
9380                .expect("the producer constructs");
9381            assert_eq!(incremental, producer.chunked_accumulator.is_some(), "{extra:?}");
9382            let pool = producer.accumulator.buffer_pool_for_test();
9383            let mode = if incremental {
9384                AllocationMode::Incremental
9385            } else {
9386                AllocationMode::Full
9387            };
9388            assert_eq!(mode, pool.allocation_mode(), "{extra:?}");
9389            assert_eq!(poolable_size, pool.poolable_size(), "{extra:?}");
9390            if let Some(chunked) = &producer.chunked_accumulator {
9391                // The chunked accumulator extends the very accumulator the Sender drains.
9392                assert!(Arc::ptr_eq(chunked.base(), &producer.accumulator), "{extra:?}");
9393            }
9394        }
9395    }
9396
9397    /// KIP-1332: the incremental strategy does not support compression yet, so the combination
9398    /// fails construction with Java's `ConfigException` (`KafkaProducer#KafkaProducer`), wrapped
9399    /// like every other construction failure. Below one chunk the strategy falls back to the full
9400    /// one first, so compression is allowed there, as in Java.
9401    #[cfg(feature = "gzip")]
9402    #[tokio::test]
9403    async fn test_incremental_allocation_strategy_rejects_compression() {
9404        let config = ProducerConfig::new(&guard_props(&[
9405            ("buffer.memory.allocation.strategy", "incremental"),
9406            ("compression.type", "gzip"),
9407        ]))
9408        .expect("valid config");
9409        let error = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
9410            .expect_err("incremental with compression must fail construction");
9411        assert_eq!(error.message(), "Failed to construct kafka producer");
9412        let cause: &Error = std::error::Error::source(&error)
9413            .and_then(|e| e.downcast_ref::<Error>())
9414            .expect("the cause is a crate Error");
9415        assert!(matches!(cause, Error::Config(_)), "got {cause:?}");
9416        assert_eq!(
9417            cause.message(),
9418            "The incremental buffer.memory.allocation.strategy does not support compression yet. \
9419             compression.type must be set to none."
9420        );
9421
9422        let config = ProducerConfig::new(&guard_props(&[
9423            ("buffer.memory.allocation.strategy", "incremental"),
9424            ("compression.type", "gzip"),
9425            ("batch.size", "1024"),
9426        ]))
9427        .expect("valid config");
9428        let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
9429            .expect("the full-strategy fallback supports compression");
9430        assert!(producer.chunked_accumulator.is_none());
9431    }
9432
9433    /// KAFKA-2121 for the KIP-1332 check: the compression `ConfigException` is raised inside
9434    /// `build_accumulator`, after a custom partitioner is configured, so the failed construction
9435    /// closes it (`KafkaProducer#KafkaProducer` -> `KafkaProducer`).
9436    #[cfg(feature = "lz4")]
9437    #[test]
9438    fn test_partitioner_closed_when_incremental_compression_is_rejected() {
9439        let _guard = MockPartitioner::lock_counters();
9440        MockPartitioner::reset_counters();
9441
9442        let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap();
9443        runtime.block_on(async {
9444            let partitioner: Box<dyn Partitioner<String, String>> = Box::new(MockPartitioner::new());
9445            let config = ProducerConfig::new(&guard_props(&[
9446                ("buffer.memory.allocation.strategy", "incremental"),
9447                ("compression.type", "lz4"),
9448            ]))
9449            .expect("valid config")
9450            .set_partitioner(partitioner);
9451            let error = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
9452                .expect_err("incremental with compression must fail construction");
9453            assert_eq!(error.message(), "Failed to construct kafka producer");
9454
9455            assert_eq!(1, MockPartitioner::init_count().load(Ordering::SeqCst));
9456            assert_eq!(1, MockPartitioner::close_count().load(Ordering::SeqCst));
9457        });
9458
9459        MockPartitioner::reset_counters();
9460    }
9461
9462    /// Java wraps the whole constructor in `catch (Throwable t)` and rethrows
9463    /// `new KafkaException("Failed to construct kafka producer", t)`
9464    /// (`KafkaProducer#KafkaProducer`), so a caller has one class and one message to
9465    /// guard construction with whatever went wrong inside. Every failure used to
9466    /// escape raw, and one of them (`Error::local_illegal_argument` from the channel
9467    /// builder) answered `false` to `is_kafka_error()`.
9468    #[test]
9469    fn construction_failures_are_wrapped_as_a_kafka_error() {
9470        let props = HashMap::from([("bootstrap.servers".to_string(), "not-a-host-port".to_string())]);
9471        let config = ProducerConfig::new(&props).expect("the config itself parses");
9472        let error = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
9473            .expect_err("an unparseable bootstrap.servers entry must fail construction");
9474
9475        assert_eq!(error.message(), "Failed to construct kafka producer");
9476        // Java's replacement is a bare `KafkaException`.
9477        assert!(error.is_kafka_error(), "Java's replacement is a Kafka error: {error:?}");
9478        assert!(!error.is_api_error(), "a bare Kafka error is not an API error: {error:?}");
9479        // The real cause is carried, not stringified into the message.
9480        assert!(
9481            error.source().is_some(),
9482            "the underlying failure must be the wrapper's cause, not lost"
9483        );
9484        // The cause is `parseAndValidateAddresses`' `ConfigException`, with its
9485        // single-message text.
9486        let cause: &Error = std::error::Error::source(&error)
9487            .and_then(|e| e.downcast_ref::<Error>())
9488            .expect("the cause is a crate Error");
9489        assert!(matches!(cause, Error::Config(_)), "got {cause:?}");
9490        assert_eq!(cause.message(), "Invalid url in bootstrap.servers: not-a-host-port");
9491    }
9492
9493    /// Translated from `KafkaProducerTest.testProducerBootstrapResolutionExceptionPropagated`
9494    /// (KIP-909): with a positive `bootstrap.resolve.timeout.ms` and an
9495    /// unresolvable bootstrap host, `partitionsFor` fails with the
9496    /// `BootstrapResolutionException` once the budget runs out (the metadata
9497    /// wait is woken by it, not left to `max.block.ms`), and so does every later
9498    /// call.
9499    #[tokio::test(flavor = "multi_thread")]
9500    #[doc(
9501        alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testProducerBootstrapResolutionExceptionPropagated"
9502    )]
9503    async fn test_producer_bootstrap_resolution_error_propagated() {
9504        let props = HashMap::from([
9505            (
9506                ProducerConfig::BOOTSTRAP_SERVERS_CONFIG.to_string(),
9507                "unresolvable.invalid:9092".to_string(),
9508            ),
9509            (
9510                ProducerConfig::BOOTSTRAP_RESOLVE_TIMEOUT_MS_CONFIG.to_string(),
9511                "3000".to_string(),
9512            ),
9513        ]);
9514        let producer = KafkaProducer::<String, String>::new(
9515            ProducerConfig::new(&props).unwrap(),
9516            StringSerializer,
9517            StringSerializer,
9518        )
9519        .expect("a positive timeout defers resolution, so construction succeeds");
9520
9521        let expected = "Failed to resolve bootstrap servers after 3000ms. \
9522                        Please check your bootstrap.servers configuration and DNS settings.";
9523        let start = std::time::Instant::now();
9524        let max_wait = Duration::from_millis(15000);
9525        loop {
9526            assert!(
9527                start.elapsed() < max_wait,
9528                "Expected the BootstrapResolutionError within {}ms",
9529                max_wait.as_millis()
9530            );
9531            match tokio::time::timeout(max_wait, producer.partitions_for("test-topic")).await {
9532                Ok(Err(Error::BootstrapResolution(e))) => {
9533                    assert_eq!(e.message(), expected);
9534                    break;
9535                },
9536                Ok(_) => {},
9537                Err(_) => panic!("partitions_for must be woken by the bootstrap failure"),
9538            }
9539        }
9540
9541        // After the first failure, any further API call must also return it. This guards against
9542        // accidentally clearing the bootstrap error from the metadata layer.
9543        match producer.partitions_for("test-topic").await {
9544            Err(Error::BootstrapResolution(e)) => assert_eq!(e.message(), expected),
9545            other => panic!("expected the bootstrap failure, got {other:?}"),
9546        }
9547        producer.close_with_timeout(Duration::ZERO).await.unwrap();
9548    }
9549
9550    /// Translated from
9551    /// `KafkaProducerTest.testProducerSendOffsetsToTransactionBootstrapResolutionExceptionPropagated`
9552    /// (KIP-909 + KIP-1319, with 0720ba1141's group-metadata mock): sendOffsetsToTransaction became
9553    /// metadata-aware in KIP-1319, so it can also block on bootstrap and must surface the
9554    /// `BootstrapResolutionException`. The first call waits on the refresh and is woken by the
9555    /// failure; the second finds the topic already tracked and hits the no-refresh branch's check.
9556    #[tokio::test(flavor = "multi_thread")]
9557    #[doc(
9558        alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testProducerSendOffsetsToTransactionBootstrapResolutionExceptionPropagated"
9559    )]
9560    async fn test_producer_send_offsets_to_transaction_bootstrap_resolution_error_propagated() {
9561        let props = HashMap::from([
9562            (
9563                ProducerConfig::BOOTSTRAP_SERVERS_CONFIG.to_string(),
9564                "unresolvable.invalid:9092".to_string(),
9565            ),
9566            (
9567                ProducerConfig::BOOTSTRAP_RESOLVE_TIMEOUT_MS_CONFIG.to_string(),
9568                "3000".to_string(),
9569            ),
9570            (ProducerConfig::TRANSACTIONAL_ID_CONFIG.to_string(), "test-tx-id".to_string()),
9571        ]);
9572        let producer = KafkaProducer::<String, String>::new(
9573            ProducerConfig::new(&props).unwrap(),
9574            StringSerializer,
9575            StringSerializer,
9576        )
9577        .expect("a positive timeout defers resolution, so construction succeeds");
9578
9579        let group_metadata = ConsumerGroupMetadataImpl::new("test-group");
9580        let offsets = || {
9581            HashMap::from([(
9582                TopicPartition::new("test-topic".to_string(), 0),
9583                OffsetAndMetadata::new(0).expect("a non-negative offset"),
9584            )])
9585        };
9586        let expected = "Failed to resolve bootstrap servers after 3000ms. \
9587                        Please check your bootstrap.servers configuration and DNS settings.";
9588        for attempt in 0..2 {
9589            match tokio::time::timeout(
9590                Duration::from_secs(15),
9591                producer.send_offsets_to_transaction(offsets(), &group_metadata),
9592            )
9593            .await
9594            {
9595                Ok(Err(Error::BootstrapResolution(e))) => assert_eq!(e.message(), expected, "attempt {attempt}"),
9596                Ok(other) => panic!("attempt {attempt}: expected the bootstrap failure, got {other:?}"),
9597                Err(_) => {
9598                    panic!("attempt {attempt}: send_offsets_to_transaction must be woken by the bootstrap failure")
9599                },
9600            }
9601        }
9602        producer.close_with_timeout(Duration::ZERO).await.unwrap();
9603    }
9604
9605    /// Critic 92, Issue 2: the C / Python producer destroy path closes the
9606    /// producer and drops the runtime it built. With a slow bootstrap lookup in
9607    /// flight that now returns within Java's bound; with the lookup on that
9608    /// runtime's blocking pool the drop waited for every remaining lookup.
9609    #[test]
9610    fn test_close_and_runtime_drop_are_bounded_while_a_bootstrap_resolution_is_in_flight() {
9611        let servers: Vec<String> = (0..3)
9612            .map(|i| format!("host{i}{}:9092", crate::ClientUtils::SLOW_TEST_HOST_SUFFIX))
9613            .collect();
9614        let props = HashMap::from([
9615            (ProducerConfig::BOOTSTRAP_SERVERS_CONFIG.to_string(), servers.join(",")),
9616            (
9617                ProducerConfig::BOOTSTRAP_RESOLVE_TIMEOUT_MS_CONFIG.to_string(),
9618                "120000".to_string(),
9619            ),
9620        ]);
9621        let runtime = tokio::runtime::Builder::new_multi_thread().enable_all().build().unwrap();
9622        let producer = runtime.block_on(async {
9623            KafkaProducer::<String, String>::new(
9624                ProducerConfig::new(&props).unwrap(),
9625                StringSerializer,
9626                StringSerializer,
9627            )
9628            .expect("a positive timeout defers resolution, so construction succeeds")
9629        });
9630        let started = std::time::Instant::now();
9631        runtime.block_on(async {
9632            // Let the sender poll so the resolution is surely in flight.
9633            tokio::time::sleep(Duration::from_millis(100)).await;
9634            producer.close_with_timeout(Duration::ZERO).await.unwrap();
9635        });
9636        drop(producer);
9637        drop(runtime);
9638        let elapsed = started.elapsed();
9639        assert!(elapsed < Duration::from_secs(4), "close + runtime drop took {elapsed:?}");
9640    }
9641
9642    /// Translated from
9643    /// `KafkaProducerTest.testProducerConstructorFailsWithConfigExceptionOnUnresolvableBootstrapWhenTimeoutZero`
9644    /// (KAFKA-20939): the default `bootstrap.resolve.timeout.ms=0` resolves DNS
9645    /// synchronously in the constructor, so an unresolvable host fails
9646    /// construction with the `ConfigException` wrapped in the constructor's
9647    /// `KafkaException`.
9648    #[test]
9649    #[doc(
9650        alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testProducerConstructorFailsWithConfigExceptionOnUnresolvableBootstrapWhenTimeoutZero"
9651    )]
9652    fn test_producer_constructor_fails_with_config_error_on_unresolvable_bootstrap_when_timeout_zero() {
9653        let props = HashMap::from([(
9654            ProducerConfig::BOOTSTRAP_SERVERS_CONFIG.to_string(),
9655            "unresolvable.invalid:9092".to_string(),
9656        )]);
9657        let config = ProducerConfig::new(&props).unwrap();
9658        assert_eq!(config.bootstrap_resolve_timeout_ms, 0);
9659        let error = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
9660            .expect_err("construction must fail");
9661        assert_eq!(error.message(), "Failed to construct kafka producer");
9662        let cause: &Error = std::error::Error::source(&error)
9663            .and_then(|e| e.downcast_ref::<Error>())
9664            .expect("the cause is a crate Error");
9665        assert!(matches!(cause, Error::Config(_)), "got {cause:?}");
9666        assert_eq!(cause.message(), "No resolvable bootstrap urls given in bootstrap.servers");
9667    }
9668
9669    /// `doSend`'s inner `catch (KafkaException e)` around `waitOnMetadata`
9670    /// (`KafkaProducer#doSend`) relabels a `close()` racing an in-flight send:
9671    ///
9672    /// ```java
9673    /// if (metadata.isClosed())
9674    ///     throw new KafkaException("Producer closed while send in progress", e);
9675    /// ```
9676    ///
9677    /// Rust had neither layer: `await_update` ignored `is_closed()` and `do_send`
9678    /// had no counterpart to this catch, so a caller shutting down while a first
9679    /// send resolved metadata saw a stall followed by a misleading retriable
9680    /// timeout.
9681    #[tokio::test]
9682    async fn closing_the_metadata_during_a_send_reports_the_close_not_a_timeout() {
9683        // A producer whose metadata knows nothing, so `send` has to wait for it.
9684        let metadata = Arc::new(ProducerMetadata::new(
9685            100,
9686            1000,
9687            300_000,
9688            300_000,
9689            ClusterResourceListeners::new(),
9690        ));
9691        let accumulator = create_accumulator();
9692        let config = ProducerConfig { max_block_ms: 30_000, ..ProducerConfig::defaults() };
9693        let producer = create_producer_with_config(config, Arc::clone(&metadata), accumulator);
9694
9695        metadata.close();
9696
9697        let started = std::time::Instant::now();
9698        let record = ProducerRecord::with_key(TOPIC.to_string(), Some("k".to_string()), Some("v".to_string()));
9699        let error = producer
9700            .send(record)
9701            .await
9702            .expect_err("a bare Kafka error is not an API error, so send() returns Err");
9703
9704        assert_eq!(error.message(), "Producer closed while send in progress");
9705        assert!(error.is_kafka_error(), "got {error:?}");
9706        assert!(!error.is_api_error(), "got {error:?}");
9707        assert!(
9708            !error.is_timeout_error(),
9709            "the close must not be reported as a timeout: {error:?}"
9710        );
9711        assert_eq!(
9712            error.source().expect("the awaitUpdate error is the cause").message(),
9713            "Requested metadata update after close"
9714        );
9715        assert!(
9716            started.elapsed() < std::time::Duration::from_secs(5),
9717            "it must fail fast, not wait out max.block.ms"
9718        );
9719    }
9720
9721    /// The default configuration must construct. `#[tokio::test]`: a successful
9722    /// `new` spawns the Sender task and so needs a runtime. The spawned task
9723    /// attempts to reach localhost:9999, fails harmlessly, and is dropped with the
9724    /// test.
9725    #[tokio::test]
9726    async fn test_guard_allows_default_config() {
9727        from_guard_props(&guard_props(&[])).expect("default config must still construct");
9728    }
9729
9730    /// Explicit `enable.idempotence=true` is accepted as of Phase 4, and the
9731    /// producer really is idempotent: it holds a `TransactionManager`.
9732    #[tokio::test]
9733    async fn test_explicit_enable_idempotence_builds_a_transaction_manager() {
9734        let props = guard_props(&[("enable.idempotence", "true")]);
9735        let config = ProducerConfig::new(&props).expect("valid config");
9736        let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
9737            .expect("explicit idempotence is supported as of Phase 4");
9738        let transaction_manager = producer
9739            .transaction_manager
9740            .as_ref()
9741            .expect("configureTransactionState builds a manager when enable.idempotence is true");
9742        let manager = transaction_manager.lock().unwrap();
9743        assert!(!manager.is_transactional());
9744        assert!(
9745            !manager.has_producer_id(),
9746            "the producer id is acquired asynchronously by the Sender task"
9747        );
9748    }
9749
9750    /// `enable.idempotence=false` builds no manager at all, mirroring Java's `null`
9751    /// return from `configureTransactionState` (`KafkaProducer#configureTransactionState`).
9752    #[tokio::test]
9753    async fn test_disabled_idempotence_builds_no_transaction_manager() {
9754        let props = guard_props(&[("enable.idempotence", "false")]);
9755        let config = ProducerConfig::new(&props).expect("valid config");
9756        let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
9757            .expect("disabling idempotence is allowed");
9758        assert!(producer.transaction_manager.is_none());
9759    }
9760
9761    /// `transactional.id` is accepted as of Phase 6, and the producer really is
9762    /// transactional: `configureTransactionState` passes the id through to the
9763    /// manager (`KafkaProducer#configureTransactionState`) and `isTransactional()` reports it.
9764    ///
9765    /// This replaces the Phase-1 `test_guard_rejects_transactional_id`, whose whole
9766    /// subject — the `new` guard of PLAN §7.1 — is what this phase deleted.
9767    #[tokio::test]
9768    async fn test_transactional_id_builds_a_transactional_manager() {
9769        let props = guard_props(&[("transactional.id", "my-txn")]);
9770        let config = ProducerConfig::new(&props).expect("valid config");
9771        let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
9772            .expect("transactional.id is supported as of Phase 6");
9773        let transaction_manager = producer
9774            .transaction_manager
9775            .as_ref()
9776            .expect("configureTransactionState builds a manager when transactional.id is set");
9777        let manager = transaction_manager.lock().unwrap();
9778        assert!(manager.is_transactional());
9779        assert_eq!(manager.transactional_id(), Some("my-txn"));
9780    }
9781
9782    /// A producer with no `transactional.id` rejects every transactional method
9783    /// with Java's `throwIfNoTransactionManager` message — but only when there is
9784    /// no manager at all, i.e. `enable.idempotence=false`. An *idempotent* producer
9785    /// has a manager and is rejected one level down; the next test covers that.
9786    ///
9787    /// Java's message is built at `KafkaProducer`.
9788    #[tokio::test]
9789    async fn test_transactional_methods_without_a_manager() {
9790        let props = guard_props(&[("enable.idempotence", "false")]);
9791        let config = ProducerConfig::new(&props).expect("valid config");
9792        let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
9793            .expect("disabling idempotence is allowed");
9794
9795        const EXPECTED: &str = "Cannot use transactional methods without enabling transactions \
9796                                by setting the transactional.id configuration property";
9797
9798        let expect_no_manager = |error: Error, method: &str| {
9799            assert_eq!(error.message(), EXPECTED, "{} reported the wrong error", method);
9800        };
9801        expect_no_manager(producer.init_transactions().await.expect_err("no manager"), "init_transactions");
9802        expect_no_manager(producer.begin_transaction().expect_err("no manager"), "begin_transaction");
9803        expect_no_manager(
9804            producer.commit_transaction().await.expect_err("no manager"),
9805            "commit_transaction",
9806        );
9807        expect_no_manager(producer.abort_transaction().await.expect_err("no manager"), "abort_transaction");
9808        let group_metadata = ConsumerGroupMetadataImpl::new("group");
9809        expect_no_manager(
9810            producer
9811                .send_offsets_to_transaction(
9812                    HashMap::from([(
9813                        TopicPartition::new(TOPIC.to_string(), 0),
9814                        OffsetAndMetadata::new(1).expect("a non-negative offset"),
9815                    )]),
9816                    &group_metadata,
9817                )
9818                .await
9819                .expect_err("no manager"),
9820            "send_offsets_to_transaction",
9821        );
9822    }
9823
9824    /// An idempotent (non-transactional) producer *does* hold a manager, so Java's
9825    /// `throwIfNoTransactionManager` passes and the rejection comes from
9826    /// `TransactionManager.ensureTransactional()`
9827    /// with a different message. Pinning both messages is what proves
9828    /// [`KafkaProducer::transaction_manager_or_error`] does not additionally test
9829    /// `is_transactional()`.
9830    #[tokio::test]
9831    async fn test_transactional_methods_on_an_idempotent_producer() {
9832        let props = guard_props(&[("enable.idempotence", "true")]);
9833        let config = ProducerConfig::new(&props).expect("valid config");
9834        let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
9835            .expect("explicit idempotence is supported as of Phase 4");
9836
9837        const EXPECTED: &str = "Transactional method invoked on a non-transactional producer.";
9838        assert_eq!(
9839            producer.init_transactions().await.expect_err("not transactional").message(),
9840            EXPECTED
9841        );
9842        assert_eq!(producer.begin_transaction().expect_err("not transactional").message(), EXPECTED);
9843        assert_eq!(
9844            producer.commit_transaction().await.expect_err("not transactional").message(),
9845            EXPECTED
9846        );
9847        assert_eq!(
9848            producer.abort_transaction().await.expect_err("not transactional").message(),
9849            EXPECTED
9850        );
9851    }
9852
9853    /// `metadata.cluster.check.enable` (KIP-1242) reaches the producer's
9854    /// `NetworkClient`, as `ClientUtils.createNetworkClient` passes it
9855    /// (`ClientUtils#createNetworkClient`). Goes through the REAL constructor (DoD #12)
9856    /// against an in-process fake broker and observes the wire: the bootstrap
9857    /// connection's ApiVersions carries no cluster id (it is not known yet), and
9858    /// the connection to the partition leader (node 0, learned from the Metadata
9859    /// response) carries `ClusterId`/`NodeId` only when the check is enabled —
9860    /// the default. Java's broker-backed analogue is
9861    /// `ClientRebootstrapTest.testRebootstrapOnMetadataClusterCheckFail`.
9862    #[tokio::test(flavor = "multi_thread")]
9863    async fn metadata_cluster_check_enable_reaches_the_network_client() {
9864        use crate::producer::Producer;
9865        use crate::test_fake_broker::{FAKE_BROKER_NODE_ID, FakeBroker, ReceivedApiVersions};
9866
9867        const TOPIC: &str = "cluster-check-topic";
9868        for (configured, check) in [(None, true), (Some("false"), false), (Some("true"), true)] {
9869            let broker = FakeBroker::start("fake-cluster", TOPIC).await;
9870            // Non-idempotent, so the Sender does not first wait on an
9871            // InitProducerId the fake never answers.
9872            let mut props = HashMap::from([
9873                ("bootstrap.servers".to_string(), broker.bootstrap()),
9874                ("enable.idempotence".to_string(), "false".to_string()),
9875            ]);
9876            if let Some(value) = configured {
9877                props.insert("metadata.cluster.check.enable".to_string(), value.to_string());
9878            }
9879            let config = ProducerConfig::new(&props).expect("valid config");
9880            let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
9881                .expect("constructs against the fake broker");
9882            // The send waits for the topic's metadata, then the batch drains to
9883            // the leader, which opens the second connection. The Produce request
9884            // is never answered; only the ApiVersions requests matter.
9885            let _pending = producer
9886                .send(ProducerRecord::new(TOPIC.to_string(), Some("value".to_string())))
9887                .await
9888                .expect("metadata arrives from the fake broker");
9889            let seen = broker.await_api_versions_requests(2, Duration::from_secs(30)).await;
9890            assert_eq!(2, seen.len(), "metadata.cluster.check.enable = {configured:?}: {seen:?}");
9891            let unchecked = ReceivedApiVersions { version: 5, cluster_id: None, node_id: -1 };
9892            assert_eq!(unchecked, seen[0], "bootstrap connection, check = {check}");
9893            let expected_leader = if check {
9894                ReceivedApiVersions {
9895                    version: 5,
9896                    cluster_id: Some("fake-cluster".to_string()),
9897                    node_id: FAKE_BROKER_NODE_ID,
9898                }
9899            } else {
9900                unchecked
9901            };
9902            assert_eq!(expected_leader, seen[1], "leader connection, check = {check}");
9903            producer.close_with_timeout(Duration::ZERO).await.expect("close");
9904        }
9905    }
9906
9907    /// A closed and dropped producer frees its buffer pool and metadata. The
9908    /// accumulator's gauges capture the pool and the Sender's `metadata-age`
9909    /// gauge the metadata; before the registry's "count" gauge and the
9910    /// sensors' registry back-reference were weak, the registry kept both
9911    /// alive (the pool with its cached free-list buffers) for the life of the
9912    /// process. Java's garbage collector frees the same graph.
9913    #[tokio::test(flavor = "multi_thread")]
9914    async fn test_closed_producer_is_freed() {
9915        let config = ProducerConfig::new(&guard_props(&[])).expect("valid config");
9916        let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
9917            .expect("the producer constructs");
9918        let pool = Arc::downgrade(producer.accumulator.buffer_pool_for_test());
9919        let metadata = Arc::downgrade(&producer.metadata);
9920        let metrics = Arc::downgrade(&producer.metrics);
9921        producer.close().await.expect("close");
9922        drop(producer);
9923
9924        // Spawned tasks holding a clone end shortly after the close.
9925        let deadline = std::time::Instant::now() + Duration::from_secs(5);
9926        while (pool.strong_count() > 0 || metadata.strong_count() > 0 || metrics.strong_count() > 0)
9927            && std::time::Instant::now() < deadline
9928        {
9929            tokio::time::sleep(Duration::from_millis(10)).await;
9930        }
9931        assert_eq!(
9932            (0, 0, 0),
9933            (pool.strong_count(), metadata.strong_count(), metrics.strong_count()),
9934            "BufferPool / ProducerMetadata / Metrics still referenced"
9935        );
9936    }
9937
9938    /// A producer dropped without `close` still delivers the record it holds, as
9939    /// Java's does: Java's I/O thread is a daemon (built in the `KafkaProducer` constructor) that
9940    /// keeps sending after nothing references the producer. Dropping it used to
9941    /// force-close the sender, which failed the record with "Producer is closed
9942    /// forcefully.".
9943    #[tokio::test(flavor = "multi_thread")]
9944    async fn a_producer_dropped_without_close_delivers_the_record_it_holds() {
9945        use crate::producer::Producer;
9946        use crate::test_fake_broker::FakeBroker;
9947
9948        const TOPIC: &str = "dropped-producer-topic";
9949        let broker = FakeBroker::start_acknowledging_produce("fake-cluster", TOPIC).await;
9950        let props = HashMap::from([
9951            ("bootstrap.servers".to_string(), broker.bootstrap()),
9952            // Non-idempotent, so the Sender does not first wait on an
9953            // InitProducerId the fake never answers.
9954            ("enable.idempotence".to_string(), "false".to_string()),
9955            // The record stays in the accumulator until the producer is dropped.
9956            ("linger.ms".to_string(), "60000".to_string()),
9957        ]);
9958        let config = ProducerConfig::new(&props).expect("valid config");
9959        let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
9960            .expect("constructs against the fake broker");
9961        let future = producer
9962            .send(ProducerRecord::new(TOPIC.to_string(), Some("value".to_string())))
9963            .await
9964            .expect("metadata arrives from the fake broker");
9965        assert!(!future.is_done(), "the record must still be held when the producer is dropped");
9966
9967        drop(producer);
9968
9969        let metadata = future
9970            .get_with_timeout(Duration::from_secs(30))
9971            .await
9972            .expect("the record must be delivered after the drop");
9973        assert_eq!(0, metadata.offset());
9974    }
9975
9976    /// A broker with `connections.max.reauth.ms` set gives each SASL session a
9977    /// lifetime and closes a connection whose session has expired when it next
9978    /// receives a request other than a re-authentication. Java's client
9979    /// re-authenticates the connection before then (KIP-368), so its requests
9980    /// keep flowing over the same connection. This client used to ignore the
9981    /// lifetime: the broker closed its connection every lifetime, and the
9982    /// requests in flight failed and were retried.
9983    #[tokio::test(flavor = "multi_thread")]
9984    async fn a_sasl_producer_reauthenticates_before_its_session_expires() {
9985        use crate::producer::Producer;
9986        use crate::test_fake_broker::FakeBroker;
9987
9988        const TOPIC: &str = "reauthentication-topic";
9989        const SESSION_LIFETIME_MS: u64 = 500;
9990        let broker = FakeBroker::start_acknowledging_produce("fake-cluster", TOPIC).await;
9991        broker.set_sasl_session_lifetime_ms(SESSION_LIFETIME_MS as i64);
9992        let props = HashMap::from([
9993            ("bootstrap.servers".to_string(), broker.bootstrap()),
9994            // Non-idempotent, so the Sender does not first wait on an
9995            // InitProducerId the fake never answers.
9996            ("enable.idempotence".to_string(), "false".to_string()),
9997            // A record whose connection is closed under it fails instead of
9998            // being retried, so a lost connection shows up as a failed send too.
9999            ("retries".to_string(), "0".to_string()),
10000            ("security.protocol".to_string(), "SASL_PLAINTEXT".to_string()),
10001            ("sasl.mechanism".to_string(), "PLAIN".to_string()),
10002            (
10003                "sasl.jaas.config".to_string(),
10004                "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"u\" password=\"p\";"
10005                    .to_string(),
10006            ),
10007        ]);
10008        let config = ProducerConfig::new(&props).expect("valid config");
10009        let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
10010            .expect("constructs against the fake broker");
10011
10012        // One record at a time for four session lifetimes.
10013        let deadline = std::time::Instant::now() + Duration::from_millis(4 * SESSION_LIFETIME_MS);
10014        let mut sent = 0;
10015        while std::time::Instant::now() < deadline {
10016            producer
10017                .send(ProducerRecord::new(TOPIC.to_string(), Some(format!("value-{sent}"))))
10018                .await
10019                .expect("metadata arrives from the fake broker")
10020                .get_with_timeout(Duration::from_secs(10))
10021                .await
10022                .unwrap_or_else(|e| panic!("record {sent} must be acknowledged: {e:?}"));
10023            sent += 1;
10024            tokio::time::sleep(Duration::from_millis(20)).await;
10025        }
10026        producer.close().await.expect("close");
10027
10028        assert_eq!(
10029            0,
10030            broker.expired_connections_killed(),
10031            "the broker had to close a connection whose session expired; {sent} records sent"
10032        );
10033        // The bootstrap and the leader connection each authenticated once; the
10034        // leader connection, used throughout, re-authenticated about once a
10035        // lifetime.
10036        let authentications = broker.sasl_authentications();
10037        assert!(
10038            authentications >= 2 + 3,
10039            "{authentications} SASL authentications in four session lifetimes"
10040        );
10041    }
10042
10043    /// As [`a_sasl_producer_reauthenticates_before_its_session_expires`], with
10044    /// up to five requests in flight on the leader's connection when its
10045    /// session falls due, so responses to them arrive while it re-authenticates.
10046    /// The authenticator sets each aside and the selector hands it to the
10047    /// network client, at most one per channel per pass, as Java's does once
10048    /// per `poll()`.
10049    #[tokio::test(flavor = "multi_thread")]
10050    async fn a_sasl_producer_with_requests_in_flight_reauthenticates() {
10051        use crate::producer::Producer;
10052        use crate::test_fake_broker::FakeBroker;
10053
10054        const TOPIC: &str = "reauthentication-in-flight-topic";
10055        const SESSION_LIFETIME_MS: u64 = 500;
10056        let broker = FakeBroker::start_acknowledging_produce("fake-cluster", TOPIC).await;
10057        broker.set_sasl_session_lifetime_ms(SESSION_LIFETIME_MS as i64);
10058        let props = HashMap::from([
10059            ("bootstrap.servers".to_string(), broker.bootstrap()),
10060            ("enable.idempotence".to_string(), "false".to_string()),
10061            ("retries".to_string(), "0".to_string()),
10062            ("linger.ms".to_string(), "0".to_string()),
10063            ("batch.size".to_string(), "16".to_string()),
10064            ("max.in.flight.requests.per.connection".to_string(), "5".to_string()),
10065            ("security.protocol".to_string(), "SASL_PLAINTEXT".to_string()),
10066            ("sasl.mechanism".to_string(), "PLAIN".to_string()),
10067            (
10068                "sasl.jaas.config".to_string(),
10069                "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"u\" password=\"p\";"
10070                    .to_string(),
10071            ),
10072        ]);
10073        let config = ProducerConfig::new(&props).expect("valid config");
10074        let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
10075            .expect("constructs against the fake broker");
10076
10077        let deadline = std::time::Instant::now() + Duration::from_millis(4 * SESSION_LIFETIME_MS);
10078        let mut sent = 0;
10079        while std::time::Instant::now() < deadline {
10080            // A burst of one-record batches, each its own request.
10081            let mut futures = Vec::new();
10082            for _ in 0..20 {
10083                futures.push(
10084                    producer
10085                        .send(ProducerRecord::new(TOPIC.to_string(), Some(format!("value-{sent}"))))
10086                        .await
10087                        .expect("metadata arrives from the fake broker"),
10088                );
10089                sent += 1;
10090            }
10091            for future in futures {
10092                future
10093                    .get_with_timeout(Duration::from_secs(10))
10094                    .await
10095                    .unwrap_or_else(|e| panic!("every record must be acknowledged: {e:?}"));
10096            }
10097        }
10098        producer.close().await.expect("close");
10099
10100        assert_eq!(0, broker.expired_connections_killed(), "{sent} records sent");
10101        assert!(broker.sasl_authentications() >= 2 + 3, "{}", broker.sasl_authentications());
10102    }
10103
10104    /// `buffer.memory=0` builds a producer, as Java's `atLeast(0)` validator
10105    /// and `BufferPool` constructor allow, and each send fails in
10106    /// `KafkaProducer#ensureValidRecordSize` with Java's
10107    /// message, under both allocation strategies. Construction used to panic
10108    /// with "Buffer pool memory must be positive".
10109    #[tokio::test(flavor = "multi_thread")]
10110    async fn a_producer_with_no_buffer_memory_fails_each_send_as_java_does() {
10111        use crate::producer::Producer;
10112        use crate::test_fake_broker::FakeBroker;
10113
10114        const TOPIC: &str = "no-buffer-memory-topic";
10115        let broker = FakeBroker::start("fake-cluster", TOPIC).await;
10116        for strategy in [
10117            ProducerConfig::BUFFER_MEMORY_ALLOCATION_STRATEGY_FULL,
10118            ProducerConfig::BUFFER_MEMORY_ALLOCATION_STRATEGY_INCREMENTAL,
10119        ] {
10120            let props = HashMap::from([
10121                ("bootstrap.servers".to_string(), broker.bootstrap()),
10122                (ProducerConfig::BUFFER_MEMORY_CONFIG.to_string(), "0".to_string()),
10123                (
10124                    ProducerConfig::BUFFER_MEMORY_ALLOCATION_STRATEGY_CONFIG.to_string(),
10125                    strategy.to_string(),
10126                ),
10127                // Non-idempotent, so the Sender does not first wait on an
10128                // InitProducerId the fake never answers.
10129                ("enable.idempotence".to_string(), "false".to_string()),
10130            ]);
10131            let config = ProducerConfig::new(&props).expect("buffer.memory=0 is a valid config");
10132            let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
10133                .expect("a producer with no buffer memory constructs");
10134            assert_eq!(
10135                strategy == ProducerConfig::BUFFER_MEMORY_ALLOCATION_STRATEGY_INCREMENTAL,
10136                producer.chunked_accumulator.is_some(),
10137                "{strategy}: the accumulator this strategy selects"
10138            );
10139            let future = producer
10140                .send(ProducerRecord::new(TOPIC.to_string(), Some("value".to_string())))
10141                .await
10142                .expect("metadata arrives from the fake broker");
10143            let err = future.get().await.expect_err("no record fits in no buffer memory");
10144            assert!(matches!(err, Error::RecordTooLarge(_)), "{strategy}: {err:?}");
10145            let size = AbstractRecords::estimate_size_in_bytes_upper_bound(
10146                RecordBatch::CURRENT_MAGIC_VALUE,
10147                CompressionType::None,
10148                None,
10149                Some(b"value"),
10150                &[],
10151            );
10152            assert_eq!(
10153                format!(
10154                    "The message is {size} bytes when serialized which is larger than the total memory buffer you \
10155                     have configured with the buffer.memory configuration."
10156                ),
10157                err.message(),
10158                "{strategy}"
10159            );
10160            producer.close_with_timeout(Duration::ZERO).await.expect("close");
10161        }
10162    }
10163}