confluent_kafka/producer/kafka_producer.rs
1// Copyright 2025 Confluent Inc.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15#![cfg_attr(not(test), expect(dead_code))]
16//! A Kafka client that publishes records to the Kafka cluster.
17//!
18//! Translated from `org.apache.kafka.clients.producer.KafkaProducer`.
19//!
20//! The producer is thread-safe and sharing a single producer instance across
21//! threads will generally be faster than having multiple instances.
22//!
23//! Transactional methods are translated: see [`KafkaProducer::init_transactions`]
24//! and its four siblings.
25
26use crate::common::requests::TxnOffsetCommitRequest;
27use std::collections::{HashMap, HashSet};
28use std::sync::atomic::{AtomicBool, Ordering};
29use std::sync::{Arc, Mutex};
30use std::time::Duration;
31
32use tokio::sync::Notify;
33use tokio::task::JoinHandle;
34
35use crate::ClientUtils;
36use crate::KafkaClient;
37use crate::Metadata;
38use crate::NetworkClient;
39use crate::common::Cluster;
40use crate::common::Error;
41use crate::common::KafkaFuture;
42use crate::common::PartitionInfo;
43use crate::common::TopicPartition;
44use crate::common::compress::Compression;
45use crate::common::errors::TimeoutError;
46use crate::common::header::Headers;
47use crate::common::header::RecordHeader;
48use crate::common::header::RecordHeaders;
49use crate::common::internals::ClusterResourceListeners;
50use crate::common::metrics::{MetricConfig, Metrics, MetricsView, RecordingLevel};
51use crate::common::network::ChannelBuilders;
52use crate::common::network::Selector;
53use crate::common::record::internal::AbstractRecords;
54use crate::common::record::internal::CompressionType;
55use crate::common::record::internal::RecordBatch;
56use crate::common::serialization::{DynSerializer, Serializer};
57use crate::common::utils::internals::LogContext;
58use crate::common::utils::{SystemTime, Time, Utils};
59use crate::consumer::OffsetAndMetadata;
60use crate::consumer::internals::panic_payload_message;
61use crate::consumer::{ConsumerGroupMetadata, ConsumerGroupMetadataImpl};
62use crate::producer::Callback;
63use crate::producer::Partitioner;
64use crate::producer::Producer;
65use crate::producer::ProducerConfig;
66use crate::producer::ProducerRecord;
67use crate::producer::RecordMetadata;
68use crate::producer::internals::BufferPool;
69use crate::producer::internals::Caller;
70use crate::producer::internals::ChunkedRecordAccumulator;
71use crate::producer::internals::FutureRecordMetadata;
72use crate::producer::internals::KafkaProducerMetrics;
73use crate::producer::internals::PendingRequests;
74use crate::producer::internals::ProducerMetadata;
75use crate::producer::internals::ProducerMetrics;
76use crate::producer::internals::Sender;
77use crate::producer::internals::SenderMetricsRegistry;
78use crate::producer::internals::SenderStatics;
79use crate::producer::internals::TransactionManager;
80use crate::producer::internals::buffer_pool::AllocationMode;
81use crate::producer::internals::{BuiltInPartitioner, KeyHasher, PartitionerConfig, RecordAccumulator};
82use crate::{ApiVersions, DefaultHostResolver};
83use crate::{kafka_debug, kafka_error, kafka_info, kafka_trace, kafka_warn};
84use futures_util::FutureExt;
85use std::panic::AssertUnwindSafe;
86
87/// Metadata and time spent waiting for it.
88#[derive(Debug)]
89#[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer$ClusterAndWaitTime")]
90struct ClusterAndWaitTime {
91 /// The cluster metadata.
92 cluster: Arc<Cluster>,
93 /// Time in ms spent waiting for metadata.
94 waited_on_metadata_ms: i64,
95}
96
97/// Whether a send may block before the record reaches the accumulator.
98///
99/// Java's `KafkaProducer.send` blocks for up to `max.block.ms` at two points
100/// before `RecordAccumulator.append` succeeds: `waitOnMetadata`
101/// (`KafkaProducer`) and `BufferPool.allocate`. This enum has no Java
102/// counterpart (DoD #7): it exists so the crate-internal
103/// [`KafkaProducer::try_send`] can share the whole `send` path and differ from
104/// it only at those two points.
105#[derive(Clone, Copy, Debug, PartialEq, Eq)]
106enum SendWait {
107 /// Block for up to `max.block.ms`, as Java does.
108 MaxBlock,
109 /// Never block: a wait at either point is reported as
110 /// [`TrySendOutcome::WouldBlock`], with the callback un-invoked.
111 Never,
112}
113
114/// The outcome of a non-blocking [`KafkaProducer::try_send`].
115///
116/// There is no Java counterpart (DoD #7). Java's `KafkaProducer.send` always
117/// blocks for up to `max.block.ms` at its two pre-append wait points —
118/// `waitOnMetadata` (`KafkaProducer`) and `BufferPool.allocate` inside
119/// `RecordAccumulator.append` — and a caller who cannot afford to block has no
120/// way to ask for anything else. The C FFI's non-blocking send needs exactly
121/// that: it appends on the caller's own thread when the producer is ready and
122/// must fall back to its submission queue, rather than stall that thread,
123/// whenever the producer would have to wait. This enum is how the producer
124/// reports which of the two happened. Crate-internal: the public surface stays
125/// Java's `send`.
126pub(crate) enum TrySendOutcome {
127 /// The record was handed to the producer and resolved the way `send` would
128 /// have resolved it: appended to a batch, or rejected with an `ApiException`
129 /// (a record too large, say) — in which case the future is already failed
130 /// and the callback has already fired, exactly as on `send`.
131 Accepted(KafkaFuture<RecordMetadata>),
132 /// The producer would have had to block — on metadata for the topic, or on
133 /// buffer memory — so it did nothing: no batch was touched, no future was
134 /// created and the callback was **not** invoked. The callback is handed back
135 /// so the caller can retry the record with `send` once it can afford to
136 /// wait.
137 WouldBlock(Option<Box<dyn Callback>>),
138}
139
140impl TrySendOutcome {
141 /// The blocking path's view of an outcome.
142 ///
143 /// [`SendWait::MaxBlock`] never produces [`WouldBlock`](Self::WouldBlock), so
144 /// the second arm is unreachable by construction; it is an `Err` rather than
145 /// a `panic!` per CLAUDE.md §12.1.
146 fn into_accepted_future(self) -> Result<KafkaFuture<RecordMetadata>, Error> {
147 match self {
148 TrySendOutcome::Accepted(future) => Ok(future),
149 TrySendOutcome::WouldBlock(_) => {
150 Err(Error::local_illegal_state("A blocking send reported that it would block"))
151 },
152 }
153 }
154}
155
156impl std::fmt::Debug for TrySendOutcome {
157 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
158 match self {
159 TrySendOutcome::Accepted(future) => f.debug_tuple("Accepted").field(future).finish(),
160 // A `Box<dyn Callback>` is not `Debug`, so only its presence is printable.
161 TrySendOutcome::WouldBlock(callback) => f
162 .debug_tuple("WouldBlock")
163 .field(&callback.as_ref().map(|_| "callback"))
164 .finish(),
165 }
166 }
167}
168
169/// A Kafka client that publishes records to the Kafka cluster.
170///
171/// The producer is thread-safe and sharing a single producer instance across
172/// tasks will generally be faster than having multiple instances.
173///
174/// The producer consists of a pool of buffer space that holds records that
175/// haven't yet been transmitted to the server, as well as a background I/O
176/// task that is responsible for turning these records into requests and
177/// transmitting them to the cluster. Failure to close the producer after use
178/// will leak these resources.
179///
180/// The [`send`](KafkaProducer::send) method is asynchronous. When called, it
181/// adds the record to a buffer of pending record sends and immediately returns.
182/// This allows the producer to batch together individual records for efficiency.
183#[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer")]
184pub struct KafkaProducer<K, V> {
185 /// The client ID used for this producer.
186 client_id: String,
187 /// The key serializer.
188 key_serializer: Box<dyn DynSerializer<K>>,
189 /// The value serializer.
190 value_serializer: Box<dyn DynSerializer<V>>,
191 /// Set by the first `close`, which closes both serializers: `close` is
192 /// idempotent, and a serializer's `close` runs at most once.
193 serializers_closed: AtomicBool,
194 /// The maximum size of a request in bytes.
195 max_request_size: i32,
196 /// The total memory size for the buffer pool.
197 total_memory_size: i64,
198 /// The record accumulator that batches records.
199 accumulator: Arc<RecordAccumulator>,
200 /// The incremental-strategy accumulator (KIP-1332), when `buffer.memory.allocation.strategy`
201 /// is `incremental` and `batch.size` admits it; its base is [`Self::accumulator`].
202 ///
203 /// Java holds one `accumulator` field typed `RecordAccumulator` and lets virtual dispatch pick
204 /// `ChunkedRecordAccumulator.append`. Rust composes the chunked accumulator over the base
205 /// (PLAN §2.3), so the one call site that differs — `append` on the send path — dispatches on
206 /// this field, and every other use keeps the shared base.
207 chunked_accumulator: Option<ChunkedRecordAccumulator>,
208 /// The producer metadata.
209 metadata: Arc<ProducerMetadata>,
210 /// All the state related to transactions, in particular the producer id,
211 /// producer epoch, and sequence numbers; `None` when idempotence is disabled.
212 ///
213 /// Translated from `KafkaProducer#transactionManager`, which is
214 /// nullable — hence [`Option`].
215 ///
216 /// Shared with the [`Sender`] task and the [`RecordAccumulator`] behind a
217 /// `std::sync::Mutex` (`.claude/rules/producer-transactions.md` §2 and
218 /// PLAN §6.3): the Sender is moved into a `tokio::task::spawn`, so this
219 /// struct cannot reach it any other way. No guard is ever held across an
220 /// `.await` (rules §4).
221 transaction_manager: Option<Arc<Mutex<TransactionManager>>>,
222 /// The queue of transactional requests waiting for the [`Sender`] to send them.
223 ///
224 /// Translated from `TransactionManager.pendingRequests`
225 /// (`TransactionManager#pendingRequests`). It lives outside the manager and is shared
226 /// with the `Sender` — see [`Sender::pending_requests`] for the full rationale
227 /// and for the lock order (`pending_requests` → `transaction_manager`) every
228 /// site below observes.
229 ///
230 /// Java's four transactional entry points on this class all enqueue into it from
231 /// the application thread, which is why the producer needs a handle at all. Cited
232 /// at the `transactionManager.<m>(..)` call statement in each, so the line and the
233 /// method cannot drift apart:
234 ///
235 /// | method | call statement | line |
236 /// |---|---|---|
237 /// | `initTransactions` | `initializeTransactions(false)` | 652 |
238 /// | `sendOffsetsToTransaction` | `sendOffsetsToTransaction(..)` | 740 |
239 /// | `commitTransaction` | `beginCommit()` | 783 |
240 /// | `abortTransaction` | `beginAbort()` | 818 |
241 ///
242 /// [`Sender::pending_requests`]: crate::producer::internals::Sender
243 pending_requests: Arc<Mutex<PendingRequests>>,
244 /// The compression type for records.
245 compression_type: CompressionType,
246 /// The maximum time to block on send/partitionsFor.
247 max_block_ms: i64,
248 /// The custom partitioner, if one is configured; `None` uses the built-in
249 /// default partitioner (the keyed CRC-32 / murmur2 path plus adaptive
250 /// partitioning).
251 ///
252 /// Translated from `KafkaProducer#partitionerPlugin`, a
253 /// `Plugin<Partitioner>`. The `Plugin<>` wrapper is `Monitorable`/metrics
254 /// plumbing (KIP-877) with no Rust counterpart in this milestone, so the
255 /// bare [`Partitioner`] trait object is held directly. Nullable in Java
256 /// (built-in partitioning when unset) — hence [`Option`]. Resolved from
257 /// `partitioner.type` via
258 /// [`ProducerConfig::resolve_partitioner`](crate::producer::ProducerConfig)
259 /// or set with
260 /// [`ProducerConfig::set_partitioner`](crate::producer::ProducerConfig::set_partitioner).
261 partitioner: Option<Box<dyn Partitioner<K, V>>>,
262 /// Whether to ignore keys for partitioning.
263 partitioner_ignore_keys: bool,
264 /// Which hash the keyed partition path uses, resolved from
265 /// `partitioner.type` (default [`KeyHasher::Crc32`], librdkafka
266 /// `consistent_random` parity). Copy-cheap; consulted once per keyed record.
267 key_hasher: KeyHasher,
268 /// Whether the sender task is still running.
269 running: Arc<AtomicBool>,
270 /// Whether the caller wants to force-close.
271 force_close: Arc<AtomicBool>,
272 /// Wakeup notification for the sender task.
273 wakeup: Arc<Notify>,
274 /// Handle to the sender background task.
275 /// Wrapped in `Mutex<Option<_>>` so `close_with_timeout` can take ownership
276 /// and `.await` it even though we only have `&self` (not `&mut self`).
277 sender_handle: Mutex<Option<JoinHandle<()>>>,
278 /// Java: `private final Time time`.
279 time: Arc<dyn Time>,
280 /// The metrics registry owned by this producer.
281 ///
282 /// Translated from Java's `Metrics metrics` field. Shared into
283 /// `producer_metrics`; exposed via [`metrics()`](Producer::metrics). Later
284 /// phases wire the Sender / BufferPool / RecordAccumulator sensors against
285 /// this same registry.
286 metrics: Arc<Metrics>,
287 /// Producer-level latency metrics (flush, metadata-wait, txn timings).
288 ///
289 /// Translated from Java's `KafkaProducerMetrics producerMetrics` field.
290 producer_metrics: KafkaProducerMetrics,
291 /// Contextual log message prefix.
292 ///
293 /// Translated from Java's `LogContext logContext` field in `KafkaProducer`.
294 log_context: LogContext,
295}
296
297impl<K, V> std::fmt::Debug for KafkaProducer<K, V> {
298 // The client id only, never the configuration.
299 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
300 f.debug_struct("KafkaProducer")
301 .field("client_id", &self.client_id)
302 .finish_non_exhaustive()
303 }
304}
305
306/// Parameters for [`KafkaProducer::with_options`].
307///
308/// This struct has **no Java counterpart** (DoD #7). It exists solely to satisfy
309/// CLAUDE.md §2's cap on derived overload names: Java's widest `KafkaProducer`
310/// constructor (`KafkaProducer#KafkaProducer`, marked `// visible for testing`)
311/// differs from the constructor group's parameter-name intersection —
312/// `{config, keySerializer, valueSerializer}`, which [`KafkaProducer::new`] owns
313/// — by ten parameters, far past the cap of three. So the derived name collapses
314/// to `with_options` and this struct becomes the method's *only* parameter,
315/// carrying every parameter including the intersection.
316///
317/// `config` is borrowed, not owned, exactly as the Java constructor borrows it:
318/// the producer reads it and stores only derived values.
319///
320/// Construct it with [`KafkaProducerOptionsBuilder::new`];
321/// [`KafkaProducerOptionsBuilder::build`] validates the mandatory parameters.
322#[non_exhaustive]
323pub(crate) struct KafkaProducerOptions<'a, K, V> {
324 /// The producer configuration. Java's `config`.
325 pub config: &'a ProducerConfig,
326 /// The key serializer. Java's `keySerializer`.
327 pub key_serializer: Box<dyn DynSerializer<K>>,
328 /// The value serializer. Java's `valueSerializer`.
329 pub value_serializer: Box<dyn DynSerializer<V>>,
330 /// The producer metadata. Java's `metadata`.
331 pub metadata: Arc<ProducerMetadata>,
332 /// The record accumulator. Java's `accumulator`. A producer built through these options
333 /// uses the full strategy: no caller (all of them tests) hands it a
334 /// `ChunkedRecordAccumulator`.
335 pub accumulator: Arc<RecordAccumulator>,
336 /// Whether the sender task is running. Part of Rust's decomposition of
337 /// Java's `Sender sender` / `Sender.SenderThread ioThread` pair; defaults to
338 /// `true`, the value [`KafkaProducer::with_client_options`] supplies itself.
339 pub running: Arc<AtomicBool>,
340 /// Whether force-close has been requested. Rust-side sender lifecycle, as
341 /// for [`Self::running`]; defaults to `false`.
342 pub force_close: Arc<AtomicBool>,
343 /// Notification that wakes the sender task. Rust-side sender lifecycle, as
344 /// for [`Self::running`]. Defaults to a *fresh* [`Notify`], attached to no
345 /// client — unlike [`KafkaProducer::with_client_options`], which takes this
346 /// from the client it was handed. A caller needing the producer and a client
347 /// to share one must set it.
348 pub wakeup: Arc<Notify>,
349 /// Handle to the sender background task, or `None` when no task was spawned.
350 /// Rust's counterpart of Java's `ioThread`, which the test-only constructor also
351 /// (`KafkaProducer#KafkaProducer(ProducerConfig,LogContext,Metrics,Serializer,Serializer,ProducerMetadata,RecordAccumulator,TransactionManager,Sender,ProducerInterceptors,Partitioner,Time,SenderThread,Optional)`)
352 /// accepts as a pre-built value.
353 pub sender_handle: Option<JoinHandle<()>>,
354 /// The clock. Java's `time`.
355 pub time: Arc<dyn Time>,
356 /// The shared transaction state, or `None` when idempotence is disabled.
357 /// Java's `transactionManager`, which that constructor also accepts as `null`.
358 pub transaction_manager: Option<Arc<Mutex<TransactionManager>>>,
359 /// The transactional request queue this producer shares with the [`Sender`].
360 pub pending_requests: Arc<Mutex<PendingRequests>>,
361 /// The custom partitioner instance, or `None` for the built-in default.
362 /// Java's `partitioner`. Stored as-is and **not** configured, mirroring
363 /// `KafkaProducer#KafkaProducer`, which wraps a pre-built `Partitioner` without
364 /// calling `configure`.
365 pub partitioner: Option<Box<dyn Partitioner<K, V>>>,
366}
367
368/// Fluent builder for [`KafkaProducerOptions`].
369///
370/// Per CLAUDE.md §2 [`Self::new`] takes no parameters, every parameter has a
371/// fluent setter, and [`Self::build`] validates the mandatory ones — returning
372/// [`Error::LocalIllegalArgument`] if they were not set. Like
373/// [`KafkaProducerOptions`] it has no Java counterpart and exists solely to
374/// satisfy that naming rule (DoD #7).
375pub(crate) struct KafkaProducerOptionsBuilder<'a, K, V> {
376 config: Option<&'a ProducerConfig>,
377 key_serializer: Option<Box<dyn DynSerializer<K>>>,
378 value_serializer: Option<Box<dyn DynSerializer<V>>>,
379 metadata: Option<Arc<ProducerMetadata>>,
380 accumulator: Option<Arc<RecordAccumulator>>,
381 running: Option<Arc<AtomicBool>>,
382 force_close: Option<Arc<AtomicBool>>,
383 wakeup: Option<Arc<Notify>>,
384 sender_handle: Option<JoinHandle<()>>,
385 time: Option<Arc<dyn Time>>,
386 transaction_manager: Option<Arc<Mutex<TransactionManager>>>,
387 pending_requests: Option<Arc<Mutex<PendingRequests>>>,
388 partitioner: Option<Box<dyn Partitioner<K, V>>>,
389}
390
391impl<K, V> Default for KafkaProducerOptionsBuilder<'_, K, V> {
392 fn default() -> Self {
393 Self::new()
394 }
395}
396
397impl<'a, K, V> KafkaProducerOptionsBuilder<'a, K, V> {
398 /// Creates a builder with every mandatory parameter unset and every other
399 /// parameter at the value the constructor supplies on its caller's behalf.
400 pub(crate) fn new() -> Self {
401 Self {
402 config: None,
403 key_serializer: None,
404 value_serializer: None,
405 metadata: None,
406 accumulator: None,
407 running: None,
408 force_close: None,
409 wakeup: None,
410 sender_handle: None,
411 time: None,
412 transaction_manager: None,
413 pending_requests: None,
414 partitioner: None,
415 }
416 }
417
418 /// Sets [`KafkaProducerOptions::config`], a mandatory parameter.
419 pub(crate) fn set_config(mut self, config: &'a ProducerConfig) -> Self {
420 self.config = Some(config);
421 self
422 }
423 /// Sets [`KafkaProducerOptions::key_serializer`], a mandatory parameter.
424 pub(crate) fn set_key_serializer(mut self, key_serializer: Box<dyn DynSerializer<K>>) -> Self {
425 self.key_serializer = Some(key_serializer);
426 self
427 }
428 /// Sets [`KafkaProducerOptions::value_serializer`], a mandatory parameter.
429 pub(crate) fn set_value_serializer(mut self, value_serializer: Box<dyn DynSerializer<V>>) -> Self {
430 self.value_serializer = Some(value_serializer);
431 self
432 }
433 /// Sets [`KafkaProducerOptions::metadata`], a mandatory parameter.
434 pub(crate) fn set_metadata(mut self, metadata: Arc<ProducerMetadata>) -> Self {
435 self.metadata = Some(metadata);
436 self
437 }
438 /// Sets [`KafkaProducerOptions::accumulator`], a mandatory parameter.
439 pub(crate) fn set_accumulator(mut self, accumulator: Arc<RecordAccumulator>) -> Self {
440 self.accumulator = Some(accumulator);
441 self
442 }
443 /// Sets [`KafkaProducerOptions::running`]; defaults to `true`.
444 pub(crate) fn set_running(mut self, running: Arc<AtomicBool>) -> Self {
445 self.running = Some(running);
446 self
447 }
448 /// Sets [`KafkaProducerOptions::force_close`]; defaults to `false`.
449 pub(crate) fn set_force_close(mut self, force_close: Arc<AtomicBool>) -> Self {
450 self.force_close = Some(force_close);
451 self
452 }
453 /// Sets [`KafkaProducerOptions::wakeup`]; defaults to a fresh [`Notify`].
454 pub(crate) fn set_wakeup(mut self, wakeup: Arc<Notify>) -> Self {
455 self.wakeup = Some(wakeup);
456 self
457 }
458 /// Sets [`KafkaProducerOptions::sender_handle`]; defaults to `None`.
459 pub(crate) fn set_sender_handle(mut self, sender_handle: Option<JoinHandle<()>>) -> Self {
460 self.sender_handle = sender_handle;
461 self
462 }
463 /// Sets [`KafkaProducerOptions::time`], a mandatory parameter.
464 pub(crate) fn set_time(mut self, time: Arc<dyn Time>) -> Self {
465 self.time = Some(time);
466 self
467 }
468 /// Sets [`KafkaProducerOptions::transaction_manager`]; defaults to `None`.
469 pub(crate) fn set_transaction_manager(
470 mut self,
471 transaction_manager: Option<Arc<Mutex<TransactionManager>>>,
472 ) -> Self {
473 self.transaction_manager = transaction_manager;
474 self
475 }
476 /// Sets [`KafkaProducerOptions::pending_requests`], a mandatory parameter.
477 pub(crate) fn set_pending_requests(mut self, pending_requests: Arc<Mutex<PendingRequests>>) -> Self {
478 self.pending_requests = Some(pending_requests);
479 self
480 }
481 /// Sets [`KafkaProducerOptions::partitioner`]; defaults to `None`.
482 pub(crate) fn set_partitioner(mut self, partitioner: Option<Box<dyn Partitioner<K, V>>>) -> Self {
483 self.partitioner = partitioner;
484 self
485 }
486
487 /// Returns the built options.
488 ///
489 /// Per CLAUDE.md §2 the mandatory parameters are validated here rather than
490 /// being named in the constructor, so a later Java version that makes one of
491 /// them optional changes the set this accepts instead of adding a second
492 /// constructor. Today there is one mandatory set: `config`,
493 /// `key_serializer`, `value_serializer`, `metadata`, `accumulator`,
494 /// `time` and `pending_requests`.
495 ///
496 /// The other six are not in it for two distinct reasons. `sender_handle`,
497 /// `transaction_manager` and `partitioner` are the parameters Java's test-only constructor
498 /// itself accepts as `null`. `running`, `force_close` and `wakeup` have no
499 /// Java counterpart at all — they are Rust's decomposition of Java's
500 /// `Sender` / `SenderThread` pair, and are values a constructor supplies on
501 /// the caller's behalf rather than values the caller must choose:
502 /// [`KafkaProducer::with_client_options`] builds `running` and `force_close`
503 /// with exactly these defaults itself, and derives `wakeup` from the client
504 /// it was handed.
505 ///
506 /// `pending_requests` is deliberately mandatory although a fresh queue would
507 /// be a plausible default: it is shared with the [`Sender`], and defaulting
508 /// it would silently hand back a producer whose transactional requests
509 /// nobody drains.
510 ///
511 /// # Errors
512 ///
513 /// Returns [`Error::LocalIllegalArgument`] naming the first parameter of the
514 /// mandatory set which was not given a setter call. Only presence is checked
515 /// here; semantic validation belongs to the method the options are passed to
516 /// (CLAUDE.md §2).
517 pub(crate) fn build(self) -> Result<KafkaProducerOptions<'a, K, V>, Error> {
518 Ok(KafkaProducerOptions {
519 config: self.config.ok_or_else(|| Self::missing("config"))?,
520 key_serializer: self.key_serializer.ok_or_else(|| Self::missing("key_serializer"))?,
521 value_serializer: self.value_serializer.ok_or_else(|| Self::missing("value_serializer"))?,
522 metadata: self.metadata.ok_or_else(|| Self::missing("metadata"))?,
523 accumulator: self.accumulator.ok_or_else(|| Self::missing("accumulator"))?,
524 running: self.running.unwrap_or_else(|| Arc::new(AtomicBool::new(true))),
525 force_close: self.force_close.unwrap_or_else(|| Arc::new(AtomicBool::new(false))),
526 wakeup: self.wakeup.unwrap_or_else(|| Arc::new(Notify::new())),
527 sender_handle: self.sender_handle,
528 time: self.time.ok_or_else(|| Self::missing("time"))?,
529 transaction_manager: self.transaction_manager,
530 pending_requests: self.pending_requests.ok_or_else(|| Self::missing("pending_requests"))?,
531 partitioner: self.partitioner,
532 })
533 }
534
535 /// Builds the [`Error::LocalIllegalArgument`] naming a mandatory parameter
536 /// [`Self::build`] found unset.
537 fn missing(parameter: &str) -> Error {
538 Error::local_illegal_argument(format!(
539 "KafkaProducerOptionsBuilder::build: mandatory parameter `{parameter}` was not set"
540 ))
541 }
542}
543
544/// Parameters for [`KafkaProducer::with_client_options`].
545///
546/// This struct has **no Java counterpart** (DoD #7), and exists for the same
547/// reason as [`KafkaProducerOptions`]: Java's other `// visible for testing`
548/// constructor (`KafkaProducer#KafkaProducer`) differs from the group's
549/// parameter-name intersection by ten parameters, past CLAUDE.md §2's cap of
550/// three, so it is served by an options struct rather than by a name listing
551/// every parameter.
552///
553/// Note the method it parameterizes is `with_client_options`, not the bare
554/// `with_options` §2's rule would literally derive: both widest Java
555/// constructors collapse to the same derived name, so the one discriminating
556/// parameter that distinguishes this overload from the test-only one — Java's
557/// `kafkaClient` — is kept in the name.
558#[non_exhaustive]
559pub(crate) struct KafkaProducerClientOptions<'a, K, V, C> {
560 /// The producer configuration. Java's `config`.
561 pub config: &'a ProducerConfig,
562 /// The key serializer. Java's `keySerializer`.
563 pub key_serializer: Box<dyn DynSerializer<K>>,
564 /// The value serializer. Java's `valueSerializer`.
565 pub value_serializer: Box<dyn DynSerializer<V>>,
566 /// The producer metadata. Java's `metadata`.
567 pub metadata: Arc<ProducerMetadata>,
568 /// The record accumulator. Java has no `accumulator` parameter on `KafkaProducer#KafkaProducer(ProducerConfig,Serializer,Serializer,ProducerMetadata,KafkaClient,ProducerInterceptors,ApiVersions,Time)`;
569 /// Rust builds it in [`KafkaProducer::new_inner`] and injects it here.
570 pub accumulator: Arc<RecordAccumulator>,
571 /// The incremental-strategy accumulator over `accumulator`, or `None` for the full strategy
572 /// (see `KafkaProducer::chunked_accumulator`).
573 pub chunked_accumulator: Option<ChunkedRecordAccumulator>,
574 /// The network client the sender task drives. Java's `kafkaClient`.
575 pub client: C,
576 /// The clock. Java's `time`.
577 pub time: Arc<dyn Time>,
578 /// The producer's metrics registry.
579 pub metrics: Arc<Metrics>,
580 /// The producer-level metrics wrapper.
581 pub producer_metrics: KafkaProducerMetrics,
582 /// The sender-level metrics registry.
583 pub sender_metrics_registry: SenderMetricsRegistry,
584 /// The shared transaction state, or `None` when idempotence is disabled.
585 pub transaction_manager: Option<Arc<Mutex<TransactionManager>>>,
586 /// The transactional request queue this producer shares with the [`Sender`].
587 pub pending_requests: Arc<Mutex<PendingRequests>>,
588 /// The custom partitioner instance (already `configure`d by the caller), or
589 /// `None` for the built-in default partitioner. Callers that resolve a
590 /// partitioner also gate adaptive partitioning on its absence before
591 /// building the `accumulator` they pass in (see `KafkaProducer::new`).
592 pub partitioner: Option<Box<dyn Partitioner<K, V>>>,
593}
594
595/// Fluent builder for [`KafkaProducerClientOptions`].
596///
597/// Per CLAUDE.md §2 [`Self::new`] takes no parameters, every parameter has a
598/// fluent setter, and [`Self::build`] validates the mandatory ones — returning
599/// [`Error::LocalIllegalArgument`] if they were not set.
600pub(crate) struct KafkaProducerClientOptionsBuilder<'a, K, V, C> {
601 config: Option<&'a ProducerConfig>,
602 key_serializer: Option<Box<dyn DynSerializer<K>>>,
603 value_serializer: Option<Box<dyn DynSerializer<V>>>,
604 metadata: Option<Arc<ProducerMetadata>>,
605 accumulator: Option<Arc<RecordAccumulator>>,
606 chunked_accumulator: Option<ChunkedRecordAccumulator>,
607 client: Option<C>,
608 time: Option<Arc<dyn Time>>,
609 metrics: Option<Arc<Metrics>>,
610 producer_metrics: Option<KafkaProducerMetrics>,
611 sender_metrics_registry: Option<SenderMetricsRegistry>,
612 transaction_manager: Option<Arc<Mutex<TransactionManager>>>,
613 pending_requests: Option<Arc<Mutex<PendingRequests>>>,
614 partitioner: Option<Box<dyn Partitioner<K, V>>>,
615}
616
617impl<K, V, C> Default for KafkaProducerClientOptionsBuilder<'_, K, V, C> {
618 fn default() -> Self {
619 Self::new()
620 }
621}
622
623impl<'a, K, V, C> KafkaProducerClientOptionsBuilder<'a, K, V, C> {
624 /// Creates a builder with every mandatory parameter unset and every other
625 /// parameter at the value the constructor supplies on its caller's behalf.
626 pub(crate) fn new() -> Self {
627 Self {
628 config: None,
629 key_serializer: None,
630 value_serializer: None,
631 metadata: None,
632 accumulator: None,
633 chunked_accumulator: None,
634 client: None,
635 time: None,
636 metrics: None,
637 producer_metrics: None,
638 sender_metrics_registry: None,
639 transaction_manager: None,
640 pending_requests: None,
641 partitioner: None,
642 }
643 }
644
645 /// Sets [`KafkaProducerClientOptions::config`], a mandatory parameter.
646 pub(crate) fn set_config(mut self, config: &'a ProducerConfig) -> Self {
647 self.config = Some(config);
648 self
649 }
650 /// Sets [`KafkaProducerClientOptions::key_serializer`], a mandatory parameter.
651 pub(crate) fn set_key_serializer(mut self, key_serializer: Box<dyn DynSerializer<K>>) -> Self {
652 self.key_serializer = Some(key_serializer);
653 self
654 }
655 /// Sets [`KafkaProducerClientOptions::value_serializer`], a mandatory parameter.
656 pub(crate) fn set_value_serializer(mut self, value_serializer: Box<dyn DynSerializer<V>>) -> Self {
657 self.value_serializer = Some(value_serializer);
658 self
659 }
660 /// Sets [`KafkaProducerClientOptions::metadata`], a mandatory parameter.
661 pub(crate) fn set_metadata(mut self, metadata: Arc<ProducerMetadata>) -> Self {
662 self.metadata = Some(metadata);
663 self
664 }
665 /// Sets [`KafkaProducerClientOptions::accumulator`], a mandatory parameter.
666 pub(crate) fn set_accumulator(mut self, accumulator: Arc<RecordAccumulator>) -> Self {
667 self.accumulator = Some(accumulator);
668 self
669 }
670 /// Sets [`KafkaProducerClientOptions::chunked_accumulator`]; `None` (the full strategy) if
671 /// unset.
672 pub(crate) fn set_chunked_accumulator(mut self, chunked_accumulator: Option<ChunkedRecordAccumulator>) -> Self {
673 self.chunked_accumulator = chunked_accumulator;
674 self
675 }
676 /// Sets [`KafkaProducerClientOptions::client`], a mandatory parameter.
677 pub(crate) fn set_client(mut self, client: C) -> Self {
678 self.client = Some(client);
679 self
680 }
681 /// Sets [`KafkaProducerClientOptions::time`], a mandatory parameter.
682 pub(crate) fn set_time(mut self, time: Arc<dyn Time>) -> Self {
683 self.time = Some(time);
684 self
685 }
686 /// Sets [`KafkaProducerClientOptions::metrics`], a mandatory parameter.
687 pub(crate) fn set_metrics(mut self, metrics: Arc<Metrics>) -> Self {
688 self.metrics = Some(metrics);
689 self
690 }
691 /// Sets [`KafkaProducerClientOptions::producer_metrics`], a mandatory parameter.
692 pub(crate) fn set_producer_metrics(mut self, producer_metrics: KafkaProducerMetrics) -> Self {
693 self.producer_metrics = Some(producer_metrics);
694 self
695 }
696 /// Sets [`KafkaProducerClientOptions::sender_metrics_registry`], a mandatory parameter.
697 pub(crate) fn set_sender_metrics_registry(mut self, sender_metrics_registry: SenderMetricsRegistry) -> Self {
698 self.sender_metrics_registry = Some(sender_metrics_registry);
699 self
700 }
701 /// Sets [`KafkaProducerClientOptions::transaction_manager`]; defaults to `None`.
702 pub(crate) fn set_transaction_manager(
703 mut self,
704 transaction_manager: Option<Arc<Mutex<TransactionManager>>>,
705 ) -> Self {
706 self.transaction_manager = transaction_manager;
707 self
708 }
709 /// Sets [`KafkaProducerClientOptions::pending_requests`], a mandatory parameter.
710 pub(crate) fn set_pending_requests(mut self, pending_requests: Arc<Mutex<PendingRequests>>) -> Self {
711 self.pending_requests = Some(pending_requests);
712 self
713 }
714 /// Sets [`KafkaProducerClientOptions::partitioner`]; defaults to `None`.
715 pub(crate) fn set_partitioner(mut self, partitioner: Option<Box<dyn Partitioner<K, V>>>) -> Self {
716 self.partitioner = partitioner;
717 self
718 }
719
720 /// Returns the built options.
721 ///
722 /// Per CLAUDE.md §2 the mandatory parameters are validated here rather than
723 /// being named in the constructor. Every parameter is mandatory except
724 /// `transaction_manager` and `partitioner`, the two that Java's two
725 /// package-private constructors also accept as `null`.
726 ///
727 /// # Errors
728 ///
729 /// Returns [`Error::LocalIllegalArgument`] naming the first parameter of the
730 /// mandatory set which was not given a setter call. Only presence is checked
731 /// here; semantic validation belongs to the method the options are passed to
732 /// (CLAUDE.md §2).
733 pub(crate) fn build(self) -> Result<KafkaProducerClientOptions<'a, K, V, C>, Error> {
734 Ok(KafkaProducerClientOptions {
735 config: self.config.ok_or_else(|| Self::missing("config"))?,
736 key_serializer: self.key_serializer.ok_or_else(|| Self::missing("key_serializer"))?,
737 value_serializer: self.value_serializer.ok_or_else(|| Self::missing("value_serializer"))?,
738 metadata: self.metadata.ok_or_else(|| Self::missing("metadata"))?,
739 accumulator: self.accumulator.ok_or_else(|| Self::missing("accumulator"))?,
740 chunked_accumulator: self.chunked_accumulator,
741 client: self.client.ok_or_else(|| Self::missing("client"))?,
742 time: self.time.ok_or_else(|| Self::missing("time"))?,
743 metrics: self.metrics.ok_or_else(|| Self::missing("metrics"))?,
744 producer_metrics: self.producer_metrics.ok_or_else(|| Self::missing("producer_metrics"))?,
745 sender_metrics_registry: self
746 .sender_metrics_registry
747 .ok_or_else(|| Self::missing("sender_metrics_registry"))?,
748 transaction_manager: self.transaction_manager,
749 pending_requests: self.pending_requests.ok_or_else(|| Self::missing("pending_requests"))?,
750 partitioner: self.partitioner,
751 })
752 }
753
754 /// Builds the [`Error::LocalIllegalArgument`] naming a mandatory parameter
755 /// [`Self::build`] found unset.
756 fn missing(parameter: &str) -> Error {
757 Error::local_illegal_argument(format!(
758 "KafkaProducerClientOptionsBuilder::build: mandatory parameter `{parameter}` was not set"
759 ))
760 }
761}
762
763impl<K, V> KafkaProducer<K, V> {
764 /// Network thread name prefix.
765 pub const NETWORK_THREAD_PREFIX: &str = "kafka-producer-network-thread";
766
767 /// Producer metric group name.
768 pub const PRODUCER_METRIC_GROUP_NAME: &str = "producer-metrics";
769
770 /// Timeout reason appended to the [`Error::Timeout`] message when
771 /// [`KafkaProducer::init_transactions`] does not complete within `max.block.ms`.
772 const INIT_TXN_TIMEOUT_MSG: &str = "InitTransactions timed out - did not complete coordinator discovery or \
773 receive the InitProducerId response within max.block.ms.";
774
775 /// Timeout reason appended when [`KafkaProducer::send_offsets_to_transaction`]
776 /// does not complete within `max.block.ms`.
777 const SEND_OFFSETS_TIMEOUT_MSG: &str = "SendOffsetsToTransaction timed out - did not reach the coordinator or \
778 receive the TxnOffsetCommit/AddOffsetsToTxn response within max.block.ms";
779
780 /// Timeout reason appended when [`KafkaProducer::commit_transaction`] does not
781 /// complete within `max.block.ms`.
782 const COMMIT_TXN_TIMEOUT_MSG: &str =
783 "CommitTransaction timed out - did not complete EndTxn with the transaction coordinator within max.block.ms";
784
785 /// Timeout reason appended when [`KafkaProducer::abort_transaction`] does not
786 /// complete within `max.block.ms`.
787 const ABORT_TXN_TIMEOUT_MSG: &str = "AbortTransaction timed out - did not complete EndTxn(abort) with the transaction coordinator within max.block.ms";
788
789 /// Creates a `KafkaProducer` from individual pre-built components.
790 ///
791 /// Translates Java's `KafkaProducer(ProducerConfig, LogContext, Metrics,
792 /// Serializer, Serializer, ProducerMetadata, RecordAccumulator,
793 /// TransactionManager, Sender, ProducerInterceptors, Partitioner, Time,
794 /// Sender.SenderThread, Optional<ClientTelemetryReporter>)`
795 /// (`KafkaProducer#KafkaProducer`), which Java marks `// visible for testing`.
796 /// This is the injection seam tests use when they supply every collaborator
797 /// themselves; [`Self::new`] is the user-facing constructor.
798 ///
799 /// `pub(crate)`, matching Java's package-private visibility: the parameters
800 /// name `ProducerMetadata`, `RecordAccumulator`, `TransactionManager` and
801 /// `PendingRequests`, all `pub(crate)` under `producer::internals`, so an
802 /// external caller could never have named or constructed them even while
803 /// this method was nominally `pub`.
804 ///
805 /// # Arguments
806 ///
807 /// * `options` - Every parameter, built through
808 /// [`KafkaProducerOptionsBuilder`]. [`KafkaProducerOptions`] is this
809 /// method's only parameter because the derived name would list ten
810 /// parameters, past CLAUDE.md §2's cap of three.
811 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#KafkaProducer")]
812 pub(crate) fn with_options(options: KafkaProducerOptions<'_, K, V>) -> Self {
813 let KafkaProducerOptions {
814 config,
815 key_serializer,
816 value_serializer,
817 metadata,
818 accumulator,
819 running,
820 force_close,
821 wakeup,
822 sender_handle,
823 time,
824 transaction_manager,
825 pending_requests,
826 partitioner,
827 } = options;
828 let log_context = LogContext::new(format!("[Producer clientId={}] ", config.client_id));
829 let (metrics, producer_metrics) = Self::create_metrics(config);
830 Self {
831 client_id: config.client_id.clone(),
832 key_serializer,
833 value_serializer,
834 serializers_closed: AtomicBool::new(false),
835 max_request_size: config.max_request_size,
836 total_memory_size: config.buffer_memory,
837 accumulator,
838 chunked_accumulator: None,
839 metadata,
840 transaction_manager,
841 pending_requests,
842 compression_type: config.compression_type,
843 max_block_ms: config.max_block_ms,
844 partitioner,
845 partitioner_ignore_keys: config.partitioner_ignore_keys,
846 key_hasher: config.key_hasher(),
847 running,
848 force_close,
849 wakeup,
850 sender_handle: Mutex::new(sender_handle),
851 time,
852 metrics,
853 producer_metrics,
854 log_context,
855 }
856 }
857
858 // Java's no-serializer constructors, `KafkaProducer(Map)` and
859 // `KafkaProducer(Properties)`, are deliberately not translated: they delegate
860 // to `this(configs, null, null)` so that the private constructor can fill a
861 // `null` serializer in reflectively from the `key.serializer` /
862 // `value.serializer` `Type.CLASS` entries. Rust has no reflection, and unlike
863 // `partitioner.type` (mapped to built-in types by
864 // `ProducerConfig::resolve_partitioner`) a serializer is typed in the
865 // producer's own `K` / `V`, so no such mapping can be written for an arbitrary
866 // `K`. `ProducerConfig` therefore carries no serializer state, and a
867 // `key.serializer` entry is ignored as an unknown key.
868 //
869 // The Java group's parameter intersection is `{configs}`, whose overload is
870 // untranslatable, so the plain name `new` stays on this, the only
871 // general-purpose constructor (as `AsyncKafkaConsumer::new` does).
872 //
873 // It wires up all infrastructure components: it resolves the bootstrap
874 // addresses, creates `ProducerMetadata`, the channel builder, `Selector` and
875 // `NetworkClient`, the `BufferPool` and `RecordAccumulator`, and spawns the
876 // background sender task via the crate-internal `with_client_options`.
877 /// A producer is instantiated by providing a set of key-value pairs as
878 /// configuration, a key and a value [`Serializer`].
879 ///
880 /// Note: after creating a `KafkaProducer` you must always
881 /// [`close`](Producer::close) it to avoid resource leaks.
882 ///
883 /// # Arguments
884 ///
885 /// - `config` - The producer configs
886 /// - `key_serializer` - The serializer for key that implements [`Serializer`]. The
887 /// `configure` method won't be called in the producer when the serializer is passed in
888 /// directly.
889 /// - `value_serializer` - The serializer for value that implements [`Serializer`]. The
890 /// `configure` method won't be called in the producer when the serializer is passed in
891 /// directly.
892 ///
893 /// # Errors
894 ///
895 /// - `Error::is_kafka_error` if the producer cannot be constructed; the cause is its
896 /// source.
897 ///
898 /// # Examples
899 ///
900 /// ```no_run
901 /// use std::collections::HashMap;
902 /// use confluent_kafka::producer::KafkaProducer;
903 /// use confluent_kafka::producer::ProducerConfig;
904 /// use confluent_kafka::common::serialization::StringSerializer;
905 ///
906 /// let props = HashMap::from([
907 /// ("bootstrap.servers".to_string(), "localhost:9092".to_string()),
908 /// ]);
909 /// let config = ProducerConfig::new(&props).expect("Invalid config");
910 ///
911 /// let producer = KafkaProducer::<String, String>::new(
912 /// config,
913 /// StringSerializer::default(),
914 /// StringSerializer::default(),
915 /// ).expect("Failed to create producer");
916 /// ```
917 ///
918 /// # Translation notes
919 ///
920 /// Java's constructors without serializers are not translated: Rust cannot
921 /// instantiate the class named by `key.serializer`.
922 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#KafkaProducer(Map,Serializer,Serializer)")]
923 pub fn new(
924 config: ProducerConfig,
925 key_serializer: impl Serializer<K> + 'static,
926 value_serializer: impl Serializer<V> + 'static,
927 ) -> Result<Self, Error>
928 where
929 K: 'static,
930 V: 'static,
931 {
932 Self::with_boxed_serializers(config, Box::new(key_serializer), Box::new(value_serializer))
933 }
934
935 /// [`new`](Self::new) over serializers already boxed as `dyn`.
936 ///
937 /// For the C binding, whose serializers are trait objects before they reach
938 /// the producer: boxing them as `dyn` a second time would nest one
939 /// `InlineFuture` in another, which no longer fits its buffer, so every
940 /// record would allocate on the send path (CLAUDE.md §13).
941 pub(crate) fn with_boxed_serializers(
942 config: ProducerConfig,
943 key_serializer: Box<dyn DynSerializer<K>>,
944 value_serializer: Box<dyn DynSerializer<V>>,
945 ) -> Result<Self, Error>
946 where
947 K: 'static,
948 V: 'static,
949 {
950 // Java wraps the whole constructor body in `catch (Throwable t)` and
951 // relabels every failure (`KafkaProducer#KafkaProducer`):
952 //
953 // throw new KafkaException("Failed to construct kafka producer", t);
954 //
955 // so a caller has exactly one class to guard construction with, whatever
956 // went wrong inside. Without this, a bad `ssl.truststore.location` reached
957 // the caller as something for which `is_kafka_error()` is `false`.
958 //
959 // The `close(Duration.ofMillis(0), true)` half of Java's catch (KAFKA-2121)
960 // matters for the serializers and the partitioner only. No socket and no
961 // spawned task can leak: the sender task is spawned by the final, infallible
962 // step, and an unconnected `Selector` / `NetworkClient` dropped on an error
963 // owns no socket. But the serializers (`KafkaProducer`) and a
964 // configured custom partitioner (`KafkaProducer`) must be closed: `new_inner` closes
965 // the serializers on every failure (`CloseSerializersOnError`) and the
966 // partitioner on every fallible step after it is configured.
967 Self::new_inner(config, key_serializer, value_serializer)
968 .map_err(|e| Error::kafka_message_source("Failed to construct kafka producer", e))
969 }
970
971 /// The body of [`new`](Self::new) and [`with_boxed_serializers`](Self::with_boxed_serializers).
972 ///
973 /// The partitioner is resolved from `partitioner.type` (a built-in name, or
974 /// the partitioner set with
975 /// [`ProducerConfig::set_partitioner`](crate::producer::ProducerConfig::set_partitioner))
976 /// via `ProducerConfig::resolve_partitioner`. A resolved partitioner is
977 /// `configure`d exactly once here (`originals` + `client.id`, Java
978 /// `KafkaProducer#KafkaProducer`), and adaptive partitioning is gated on its
979 /// absence. Errors escape raw from here; `new` relabels them
980 /// (`KafkaProducer#KafkaProducer`).
981 fn new_inner(
982 mut config: ProducerConfig,
983 key_serializer: Box<dyn DynSerializer<K>>,
984 value_serializer: Box<dyn DynSerializer<V>>,
985 ) -> Result<Self, Error>
986 where
987 K: 'static,
988 V: 'static,
989 {
990 // Java's `ProducerConfig` is built with `doLog = true`, which logs
991 // every parsed value (`AbstractConfig.logAll`).
992 config.log_values();
993 let log_context = LogContext::new(format!("[Producer clientId={}] ", config.client_id));
994
995 kafka_trace!(log_context, "Starting the Kafka producer");
996
997 // From here on, a failed construction closes the serializers (Java's
998 // catch, `KafkaProducer#KafkaProducer`); see `CloseSerializersOnError`.
999 let serializers = CloseSerializersOnError::new(key_serializer, value_serializer, log_context.clone());
1000
1001 // 1. (KIP-909) Bootstrap address resolution moved below, to
1002 // `ClientUtils::maybe_bootstrap_metadata_synchronously` right after the
1003 // metadata is created, as in Java 4.4.
1004
1005 // 2. Validate delivery timeout configuration
1006 // Translated from KafkaProducer.configureDeliveryTimeout().
1007 let delivery_timeout_ms = Self::configure_delivery_timeout(&config, &log_context)?;
1008
1009 // The MILESTONE-11 GUARD that used to sit here is gone. Its idempotence arm
1010 // was removed in Phase 4, when `enable.idempotence` began to be honoured for
1011 // real; its transactional arm is removed here, now that
1012 // `init_transactions` / `begin_transaction` / `send_offsets_to_transaction` /
1013 // `commit_transaction` / `abort_transaction` are implemented. PLAN §7.1 named
1014 // this removal as an explicit Phase-6 deliverable, so nothing is left behind
1015 // (CLAUDE.md §7).
1016
1017 // 3. Derive compression from config
1018 // Translated from KafkaProducer.configureCompression().
1019 let compression = Self::configure_compression(&config)?;
1020
1021 // 4. The system clock, Java's `Time.SYSTEM`
1022 let time: Arc<dyn Time> = Arc::new(SystemTime);
1023
1024 // 5. Create ProducerMetadata. With `bootstrap.resolve.timeout.ms=0` (the
1025 // default) bootstrap it synchronously from `bootstrap.servers`, so an
1026 // unresolvable address fails construction (`ConfigException`); with a
1027 // positive value the NetworkClient resolves them asynchronously
1028 // (KIP-909, `KafkaProducer#KafkaProducer`).
1029 let metadata = Arc::new(ProducerMetadata::with_log_context(
1030 config.retry_backoff_ms,
1031 config.retry_backoff_max_ms,
1032 config.metadata_max_age_ms,
1033 config.metadata_max_idle_ms,
1034 ClusterResourceListeners::new(),
1035 log_context.clone(),
1036 ));
1037 ClientUtils::maybe_bootstrap_metadata_synchronously(
1038 config.bootstrap_resolve_timeout_ms,
1039 config.client_dns_lookup,
1040 &config.bootstrap_servers,
1041 &metadata,
1042 )?;
1043
1044 // 6. Get the shared Metadata Arc from ProducerMetadata so the NetworkClient
1045 // uses the same Metadata instance. This mirrors Java's inheritance where
1046 // ProducerMetadata extends Metadata.
1047 let shared_metadata = metadata.metadata_arc();
1048
1049 // 7. Create Selector + NetworkClient
1050 let channel_builder = ChannelBuilders::client_channel_builder(
1051 config.security_protocol,
1052 Some(&config.ssl_config),
1053 Some(&config.sasl_config),
1054 None,
1055 &config.client_id,
1056 log_context.clone(),
1057 )
1058 // Java's reachable failure here is `SslFactory.configure` throwing
1059 // `ConfigException` (`SslFactory#configure`) — the missing-argument
1060 // `IllegalArgumentException`s in `ChannelBuilders.create` cannot be reached
1061 // from a `ProducerConfig`, which always supplies both sub-configs. So the
1062 // class is `ConfigException`, inside the `KafkaException` hierarchy;
1063 // `illegal_argument` put it outside, where `is_kafka_error()` is `false`.
1064 .map_err(|e| Error::config_message(format!("Failed to create channel builder: {}", e)))?;
1065 let mut selector = Selector::with_defaults_and_log_context(
1066 config.connections_max_idle_ms,
1067 channel_builder,
1068 log_context.clone(),
1069 );
1070 selector.set_time(Arc::clone(&time));
1071 let api_versions = Arc::new(ApiVersions::new());
1072
1073 let mut client = NetworkClient::with_metadata_rebootstrap_trigger_ms(
1074 selector,
1075 shared_metadata,
1076 &config.client_id,
1077 config.max_in_flight_requests_per_connection as usize,
1078 config.reconnect_backoff_ms,
1079 config.reconnect_backoff_max_ms,
1080 config.send_buffer_bytes,
1081 config.receive_buffer_bytes,
1082 config.request_timeout_ms,
1083 config.socket_connection_setup_timeout_ms,
1084 config.socket_connection_setup_timeout_max_ms,
1085 true, // discover_broker_versions
1086 Arc::clone(&api_versions),
1087 DefaultHostResolver::new(),
1088 // `ClientUtils.createNetworkClient` reads
1089 // both from the config; the default strategy is `rebootstrap`.
1090 config.metadata_recovery_rebootstrap_trigger_ms,
1091 config.metadata_recovery_strategy,
1092 log_context.clone(),
1093 );
1094 // `ClientUtils.createNetworkClient` passes `metadata.cluster.check.enable`
1095 // to the `NetworkClient` constructor (`ClientUtils#createNetworkClient`, KIP-1242).
1096 client.set_metadata_cluster_check_enable(config.metadata_cluster_check_enable);
1097 // `ClientUtils.createNetworkClient` passes
1098 // `bootstrapConfiguration(config, bootstrap.servers)` to the
1099 // `NetworkClient` constructor (KIP-909).
1100 client.set_bootstrap_configuration(ClientUtils::bootstrap_configuration(
1101 config.bootstrap_resolve_timeout_ms,
1102 config.client_dns_lookup,
1103 config.retry_backoff_ms,
1104 &config.bootstrap_servers,
1105 )?);
1106
1107 // 8. Create the metrics registry. Java creates `this.metrics` early in
1108 // the constructor (`KafkaProducer#KafkaProducer`), before the
1109 // `RecordAccumulator`/`BufferPool` (`KafkaProducer#KafkaProducer`), because both are
1110 // handed the same `Metrics` instance.
1111 let (metrics, producer_metrics) = Self::create_metrics(&config);
1112
1113 // 9. Create the TransactionManager, before the accumulator and the Sender
1114 // because both of them need it (PLAN §6.3). Java's field assignment sits
1115 // at the same point in the constructor (`KafkaProducer#KafkaProducer`, ahead
1116 // of the `RecordAccumulator` at `KafkaProducer#KafkaProducer` and the `Sender` at `KafkaProducer#KafkaProducer`).
1117 let transaction_manager =
1118 Self::configure_transaction_state(&config, &api_versions, &metadata.metadata_arc(), &log_context);
1119
1120 // 9b. Resolve and configure the partitioner. Translated from
1121 // `KafkaProducer#KafkaProducer`: Java reflectively instantiates
1122 // `partitioner.class` and calls `partitioner.configure(originals +
1123 // {client.id -> clientId})`. Rust has no reflection, so the built-in
1124 // `partitioner.type` names resolve here, and a user-written partitioner
1125 // is the value set with `ProducerConfig::set_partitioner`. It is configured with the user
1126 // config map (`originals`) plus the resolved (possibly generated)
1127 // `client.id`, so a generated `producer-N` id is visible to `configure`
1128 // just as in Java. Resolved after metrics/`TransactionManager` creation
1129 // (matching Java's ordering) and before the `RecordAccumulator`, whose
1130 // adaptive-partitioning flag is gated on the partitioner's absence below.
1131 //
1132 // KAFKA-2121: Java's constructor `catch (Throwable)` calls
1133 // `close(Duration.ofMillis(0), true)`, which closes an already-configured
1134 // partitioner (`KafkaProducer#KafkaProducer` -> `KafkaProducer#close`).
1135 // `resolve_partitioner` fails only before resolving one (a partitioner
1136 // written for other record types), and `configure` is infallible. Every
1137 // fallible step *after* this point lives in `build_accumulator`, and its
1138 // error closes the partitioner before propagating, so a step added there
1139 // later is covered too.
1140 let mut partitioner = config.resolve_partitioner::<K, V>()?;
1141 if let Some(partitioner) = partitioner.as_mut() {
1142 let mut configs = config.originals();
1143 configs.insert(ProducerConfig::CLIENT_ID_CONFIG.to_string(), config.client_id.clone());
1144 partitioner.configure(&configs);
1145 }
1146
1147 let (accumulator, chunked_accumulator) = match Self::build_accumulator(
1148 &config,
1149 compression,
1150 delivery_timeout_ms,
1151 partitioner.is_none(),
1152 &metrics,
1153 &time,
1154 &transaction_manager,
1155 &log_context,
1156 ) {
1157 Ok(accumulator) => accumulator,
1158 Err(error) => {
1159 // Java's `close` closes the serializers, then the partitioner
1160 // (`KafkaProducer#close`).
1161 drop(serializers);
1162 if let Some(partitioner) = partitioner.as_ref() {
1163 partitioner.close();
1164 }
1165 return Err(error);
1166 },
1167 };
1168
1169 // 11. Wire the produce-throttle-time sensor into the network client.
1170 // Java creates the throttle sensor (`Sender.throttleTimeSensor(...)`)
1171 // and hands it to the `NetworkClient` at construction
1172 // (`KafkaProducer` via `ClientUtils.createNetworkClient`);
1173 // the client then records every response's throttle time into it. We
1174 // set it on the concrete `NetworkClient` here, before it moves into
1175 // the generic sender task.
1176 // Java: `new ProducerMetrics(this.metrics).senderMetrics`.
1177 let sender_metrics_registry = ProducerMetrics::new(Arc::clone(&metrics)).sender_metrics;
1178 let throttle_sensor = SenderStatics::throttle_time_sensor(&sender_metrics_registry)
1179 .expect("registering produce-throttle-time sensor");
1180 client.set_throttle_time_sensor(throttle_sensor);
1181 client.set_time(Arc::clone(&time));
1182
1183 // 12. Wire up the Sender and spawn the I/O background task. Every
1184 // fallible step is behind us.
1185 let (key_serializer, value_serializer) = serializers.disarm();
1186 // Java's `config.logUnused()` (`KafkaProducer#KafkaProducer`), once the
1187 // producer has read its settings.
1188 config.mark_read_keys_used();
1189 config.log_unused();
1190 Ok(Self::with_client_options(
1191 KafkaProducerClientOptionsBuilder::new()
1192 .set_config(&config)
1193 .set_key_serializer(key_serializer)
1194 .set_value_serializer(value_serializer)
1195 .set_metadata(metadata)
1196 .set_accumulator(accumulator)
1197 .set_chunked_accumulator(chunked_accumulator)
1198 .set_client(client)
1199 .set_time(time)
1200 .set_metrics(metrics)
1201 .set_producer_metrics(producer_metrics)
1202 .set_sender_metrics_registry(sender_metrics_registry)
1203 .set_transaction_manager(transaction_manager)
1204 .set_pending_requests(Arc::new(Mutex::new(PendingRequests::new())))
1205 .set_partitioner(partitioner)
1206 .build()
1207 .expect("KafkaProducerClientOptionsBuilder::build: every mandatory parameter is set above"),
1208 ))
1209 }
1210
1211 /// Step 10 of [`new_inner`](Self::new_inner): the `BufferPool` and the
1212 /// `RecordAccumulator` — a `ChunkedRecordAccumulator` for the incremental
1213 /// `buffer.memory.allocation.strategy` (KIP-1332) — i.e. every fallible constructor step that
1214 /// follows the custom partitioner's `configure`. `new_inner` closes the partitioner when this
1215 /// returns an error, as Java's constructor `catch` does (KAFKA-2121), so a fallible step added
1216 /// after the partitioner belongs here: the incremental strategy's compression
1217 /// `ConfigException` is one.
1218 ///
1219 /// Returns the base accumulator, which the `Sender` and the producer's flush / close paths
1220 /// share, and, for the incremental strategy, the chunked accumulator whose `append` the send
1221 /// path calls instead of the base one (PLAN §2.3).
1222 ///
1223 /// Translated from `KafkaProducer#KafkaProducer`.
1224 #[expect(clippy::too_many_arguments)]
1225 fn build_accumulator(
1226 config: &ProducerConfig,
1227 compression: Compression,
1228 delivery_timeout_ms: i32,
1229 no_custom_partitioner: bool,
1230 metrics: &Arc<Metrics>,
1231 time: &Arc<dyn Time>,
1232 transaction_manager: &Option<Arc<Mutex<TransactionManager>>>,
1233 log_context: &LogContext,
1234 ) -> Result<(Arc<RecordAccumulator>, Option<ChunkedRecordAccumulator>), Error> {
1235 // 10. Create BufferPool and RecordAccumulator, threading the shared
1236 // `Arc<Metrics>` and `time` into both (`KafkaProducer#KafkaProducer`
1237 // passes `metrics`/`time` to the `BufferPool` and `RecordAccumulator`).
1238 let partitioner_config = PartitionerConfig::new(
1239 // Java `KafkaProducer#KafkaProducer`: "There is no need to do work
1240 // required for adaptive partitioning, if we use a custom
1241 // partitioner." So adaptive partitioning is enabled only when no
1242 // custom partitioner is present AND the config opts in.
1243 no_custom_partitioner && config.partitioner_adaptive_partitioning_enable,
1244 config.partitioner_availability_timeout_ms,
1245 // KIP-1123 (`KafkaProducer#KafkaProducer`): passed through whether or
1246 // not a custom partitioner is set, as Java does, so a blank
1247 // `client.rack` with `partitioner.rack.aware=true` fails construction
1248 // either way.
1249 config.partitioner_rack_aware,
1250 &config.client_rack,
1251 )?;
1252 // As per Kafka producer configuration documentation batch.size may be set to 0
1253 // to explicitly disable batching which in practice actually means using a batch size of 1.
1254 let batch_size = config.batch_size.max(1);
1255 let allocation_strategy = config.buffer_memory_allocation_strategy.to_lowercase();
1256 let incremental = allocation_strategy == ProducerConfig::BUFFER_MEMORY_ALLOCATION_STRATEGY_INCREMENTAL;
1257 // Use the chunked path only when a batch is at least one full chunk
1258 // (batch.size >= CHUNK_SIZE). Below that, a batch can't fill even one chunk, so chunking
1259 // would over-reserve and the producer falls back to the full strategy instead.
1260 let use_incremental = incremental && batch_size >= ChunkedRecordAccumulator::CHUNK_SIZE;
1261 if incremental && !use_incremental {
1262 kafka_warn!(
1263 log_context,
1264 "Ignoring {}={} and falling back to {}: {} is {} bytes, below the {} byte chunk size, so a batch \
1265 cannot fill a single chunk.",
1266 ProducerConfig::BUFFER_MEMORY_ALLOCATION_STRATEGY_CONFIG,
1267 ProducerConfig::BUFFER_MEMORY_ALLOCATION_STRATEGY_INCREMENTAL,
1268 ProducerConfig::BUFFER_MEMORY_ALLOCATION_STRATEGY_FULL,
1269 ProducerConfig::BATCH_SIZE_CONFIG,
1270 batch_size,
1271 ChunkedRecordAccumulator::CHUNK_SIZE
1272 );
1273 }
1274 // The chunked path does not support compression yet (KAFKA-20579).
1275 if use_incremental && compression.compression_type() != CompressionType::None {
1276 return Err(Error::config_message(Self::incremental_compression_error_message()));
1277 }
1278 if use_incremental {
1279 let buffer_pool = Arc::new(BufferPool::with_allocation_mode(
1280 config.buffer_memory,
1281 ChunkedRecordAccumulator::CHUNK_SIZE as usize,
1282 Arc::clone(metrics),
1283 Arc::clone(time),
1284 Self::PRODUCER_METRIC_GROUP_NAME,
1285 AllocationMode::Incremental,
1286 ));
1287 let accumulator = ChunkedRecordAccumulator::new(
1288 batch_size,
1289 compression,
1290 Self::linger_ms(config),
1291 config.retry_backoff_ms,
1292 config.retry_backoff_max_ms,
1293 delivery_timeout_ms,
1294 partitioner_config,
1295 Arc::clone(metrics),
1296 Self::PRODUCER_METRIC_GROUP_NAME,
1297 Arc::clone(time),
1298 buffer_pool,
1299 transaction_manager.clone(),
1300 log_context.clone(),
1301 )?;
1302 return Ok((Arc::clone(accumulator.base()), Some(accumulator)));
1303 }
1304 let buffer_pool = Arc::new(BufferPool::with_allocation_mode(
1305 config.buffer_memory,
1306 batch_size as usize,
1307 Arc::clone(metrics),
1308 Arc::clone(time),
1309 Self::PRODUCER_METRIC_GROUP_NAME,
1310 AllocationMode::Full,
1311 ));
1312 let accumulator = Arc::new(RecordAccumulator::with_log_context(
1313 batch_size,
1314 compression,
1315 Self::linger_ms(config),
1316 config.retry_backoff_ms,
1317 config.retry_backoff_max_ms,
1318 delivery_timeout_ms,
1319 partitioner_config,
1320 Arc::clone(metrics),
1321 Self::PRODUCER_METRIC_GROUP_NAME,
1322 Arc::clone(time),
1323 buffer_pool,
1324 transaction_manager.clone(),
1325 log_context.clone(),
1326 ));
1327 Ok((accumulator, None))
1328 }
1329
1330 /// The `ConfigException` message for the incremental strategy combined with compression
1331 /// (`KafkaProducer#KafkaProducer`).
1332 fn incremental_compression_error_message() -> String {
1333 format!(
1334 "The {} {} does not support compression yet. {} must be set to none.",
1335 ProducerConfig::BUFFER_MEMORY_ALLOCATION_STRATEGY_INCREMENTAL,
1336 ProducerConfig::BUFFER_MEMORY_ALLOCATION_STRATEGY_CONFIG,
1337 ProducerConfig::COMPRESSION_TYPE_CONFIG
1338 )
1339 }
1340
1341 /// Builds the [`TransactionManager`] when idempotence is enabled.
1342 ///
1343 /// Translated from `KafkaProducer.configureTransactionState`
1344 /// (`KafkaProducer#configureTransactionState`).
1345 ///
1346 /// Java returns `null` when `enable.idempotence` is `false`; that is `None`
1347 /// here. Java's `else` branch only marks `transaction.timeout.ms` as consumed
1348 /// so `AbstractConfig` does not warn about it, which has no Rust analogue —
1349 /// `ProducerConfig` parses every key eagerly.
1350 ///
1351 /// Returns `None` when idempotence is disabled, mirroring Java's null
1352 /// `transactionManager`. It no longer returns a `Result`: the only error it
1353 /// ever carried was [`Self::new`]'s temporary guard on
1354 /// `transactional.id` (PLAN §7.1), which Phase 6 removed.
1355 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#configureTransactionState")]
1356 fn configure_transaction_state(
1357 config: &ProducerConfig,
1358 api_versions: &Arc<ApiVersions>,
1359 metadata: &Arc<Metadata>,
1360 log_context: &LogContext,
1361 ) -> Option<Arc<Mutex<TransactionManager>>> {
1362 if !config.enable_idempotence {
1363 return None;
1364 }
1365
1366 let transaction_manager = TransactionManager::new(
1367 log_context.clone(),
1368 config.transactional_id.clone(),
1369 config.transaction_timeout_ms,
1370 config.retry_backoff_ms,
1371 Arc::clone(api_versions),
1372 Arc::clone(metadata),
1373 config.two_phase_commit_enable,
1374 );
1375
1376 if transaction_manager.is_transactional() {
1377 kafka_info!(log_context, "Instantiated a transactional producer.");
1378 } else {
1379 kafka_info!(log_context, "Instantiated an idempotent producer.");
1380 }
1381
1382 Some(Arc::new(Mutex::new(transaction_manager)))
1383 }
1384
1385 /// Creates a `KafkaProducer` from pre-built collaborators and spawns the
1386 /// sender task.
1387 ///
1388 /// Translates Java's `KafkaProducer(ProducerConfig, Serializer, Serializer,
1389 /// ProducerMetadata, KafkaClient, ProducerInterceptors, ApiVersions, Time)`
1390 /// (`KafkaProducer#KafkaProducer`), which Java marks `// visible for testing`.
1391 /// [`Self::new`] is the user-facing constructor and the analogue of Java's
1392 /// public `KafkaProducer` constructor; this is the injection seam
1393 /// [`Self::new_inner`] delegates to, and the seam through which a mock
1394 /// [`KafkaClient`] enters.
1395 ///
1396 /// Marked `pub(crate)`, not `pub`: `metadata` and `accumulator` are `Arc`s of
1397 /// `ProducerMetadata` and `RecordAccumulator`, both `pub(crate)` under
1398 /// `producer::internals`, so an external caller could not have named or
1399 /// constructed them even when this method itself was `pub` — making it
1400 /// `pub(crate)` only makes that existing unreachability explicit, it does
1401 /// not remove any capability external callers actually had.
1402 ///
1403 /// (Through Phase 5 that unreachability was also the reason the temporary
1404 /// MILESTONE-11 GUARD in [`Self::new`] was not duplicated here. Phase 6
1405 /// removed the guard, so nothing turns on it any more; the visibility note above
1406 /// stands on its own.)
1407 ///
1408 /// # Type Parameters
1409 ///
1410 /// * `C` - The KafkaClient implementation type
1411 ///
1412 /// # Arguments
1413 ///
1414 /// * `options` - Every parameter, built through
1415 /// [`KafkaProducerClientOptionsBuilder`].
1416 /// [`KafkaProducerClientOptions`] is this method's only parameter because
1417 /// the derived name would list ten parameters, past CLAUDE.md §2's cap of
1418 /// three; see that struct for why `client` is nonetheless kept in the name.
1419 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#KafkaProducer")]
1420 pub(crate) fn with_client_options<C: KafkaClient + Send + 'static>(
1421 options: KafkaProducerClientOptions<'_, K, V, C>,
1422 ) -> Self {
1423 let KafkaProducerClientOptions {
1424 config,
1425 key_serializer,
1426 value_serializer,
1427 metadata,
1428 accumulator,
1429 chunked_accumulator,
1430 client,
1431 time,
1432 metrics,
1433 producer_metrics,
1434 sender_metrics_registry,
1435 transaction_manager,
1436 pending_requests,
1437 partitioner,
1438 } = options;
1439 let log_context = LogContext::new(format!("[Producer clientId={}] ", config.client_id));
1440 let running = Arc::new(AtomicBool::new(true));
1441 let force_close = Arc::new(AtomicBool::new(false));
1442 let wakeup = client.wakeup_notify();
1443
1444 let guarantee_message_order = config.max_in_flight_requests_per_connection == 1;
1445 let acks = config.acks;
1446 let retries = config.retries;
1447
1448 let mut sender = Sender::new(
1449 client,
1450 Arc::clone(&metadata),
1451 Arc::clone(&accumulator),
1452 guarantee_message_order,
1453 config.max_request_size,
1454 acks,
1455 retries,
1456 config.request_timeout_ms,
1457 config.retry_backoff_ms,
1458 sender_metrics_registry,
1459 Arc::clone(&running),
1460 Arc::clone(&force_close),
1461 Arc::clone(&time),
1462 transaction_manager.clone(),
1463 Arc::clone(&pending_requests),
1464 log_context.clone(),
1465 );
1466
1467 let io_thread_name = format!("{} | {}", Self::NETWORK_THREAD_PREFIX, config.client_id);
1468 let task_log_context = log_context.clone();
1469 let sender_handle = tokio::task::spawn(async move {
1470 kafka_debug!(task_log_context, "Starting {} I/O task", io_thread_name);
1471 sender.run().await;
1472 });
1473
1474 kafka_debug!(log_context, "Kafka producer started");
1475
1476 Self {
1477 client_id: config.client_id.clone(),
1478 key_serializer,
1479 value_serializer,
1480 serializers_closed: AtomicBool::new(false),
1481 max_request_size: config.max_request_size,
1482 total_memory_size: config.buffer_memory,
1483 accumulator,
1484 chunked_accumulator,
1485 metadata,
1486 transaction_manager,
1487 pending_requests,
1488 compression_type: config.compression_type,
1489 max_block_ms: config.max_block_ms,
1490 partitioner,
1491 partitioner_ignore_keys: config.partitioner_ignore_keys,
1492 key_hasher: config.key_hasher(),
1493 running,
1494 force_close,
1495 wakeup,
1496 sender_handle: Mutex::new(Some(sender_handle)),
1497 time,
1498 metrics,
1499 producer_metrics,
1500 log_context,
1501 }
1502 }
1503
1504 /// Create the producer's [`Metrics`] registry and [`KafkaProducerMetrics`].
1505 ///
1506 /// Translated from the metrics-setup block of Java's `KafkaProducer`
1507 /// constructor (`KafkaProducer#KafkaProducer`): a [`MetricConfig`] carrying
1508 /// `metrics.num.samples`, `metrics.sample.window.ms`,
1509 /// `metrics.recording.level` and a single `client-id` tag. Reporters and the
1510 /// JMX metrics context are N/A in Rust (consumer Phase M7 precedent); the
1511 /// registry is reporter-less but fully functional.
1512 fn create_metrics(config: &ProducerConfig) -> (Arc<Metrics>, KafkaProducerMetrics) {
1513 const CLIENT_ID_METRIC_TAG: &str = "client-id";
1514
1515 let mut tags = std::collections::BTreeMap::new();
1516 tags.insert(CLIENT_ID_METRIC_TAG.to_string(), config.client_id.clone());
1517
1518 let recording_level = RecordingLevel::for_name(&config.metrics_recording_level).unwrap_or(RecordingLevel::Info);
1519 let metric_config = MetricConfig::new()
1520 .set_samples(config.metrics_num_samples)
1521 .set_time_window_ms(config.metrics_sample_window_ms)
1522 .set_record_level(recording_level)
1523 .set_tags(tags);
1524
1525 let metrics = Arc::new(Metrics::with_default_config(Arc::new(metric_config)));
1526 let producer_metrics = KafkaProducerMetrics::new(Arc::clone(&metrics));
1527 (metrics, producer_metrics)
1528 }
1529
1530 /// Builds the compression codec from `compression.type` and the level of
1531 /// that codec: Java's `configureCompression`.
1532 ///
1533 /// # Errors
1534 ///
1535 /// The builder's error for a level outside the codec's range, which the
1536 /// `ConfigDef` validators already rule out for a configuration built by
1537 /// `ProducerConfig::new`.
1538 fn configure_compression(config: &ProducerConfig) -> Result<Compression, Error> {
1539 Ok(match config.compression_type {
1540 CompressionType::Gzip => Compression::gzip().level(config.compression_gzip_level)?.build(),
1541 CompressionType::Lz4 => Compression::lz4().level(config.compression_lz4_level)?.build(),
1542 CompressionType::Zstd => Compression::zstd().level(config.compression_zstd_level)?.build(),
1543 compression_type => Compression::of(compression_type).build(),
1544 })
1545 }
1546
1547 /// Returns `linger.ms` as the `i32` the accumulator works with.
1548 ///
1549 /// `linger.ms` is a long-typed config, so values above `i32::MAX` are clamped
1550 /// to `i32::MAX` rather than cast with `as i32`, which keeps only the low 32
1551 /// bits and can yield zero or a negative linger, making every batch ready
1552 /// immediately.
1553 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#lingerMs")]
1554 fn linger_ms(config: &ProducerConfig) -> i32 {
1555 config.linger_ms.min(i32::MAX as i64) as i32
1556 }
1557
1558 /// Validate and optionally adjust `delivery.timeout.ms` against
1559 /// `linger.ms + request.timeout.ms`.
1560 ///
1561 /// Translated from `KafkaProducer.configureDeliveryTimeout()`
1562 /// (`KafkaProducer#configureDeliveryTimeout`). Java's check has two arms and both are
1563 /// reproduced: an *explicitly supplied* inconsistent `delivery.timeout.ms` is
1564 /// a `ConfigException`, while an inconsistency that comes only from the
1565 /// default is clamped up to `linger.ms + request.timeout.ms` and warned about,
1566 /// for backward compatibility. `ProducerConfig::user_configured` stands for
1567 /// Java's `config.originals().containsKey(..)`.
1568 ///
1569 /// # Errors
1570 ///
1571 /// Returns [`Error::Config`] — Java's `ConfigException`, which is inside the
1572 /// `KafkaException` hierarchy — if the user explicitly set a
1573 /// `delivery.timeout.ms` smaller than `linger.ms + request.timeout.ms`.
1574 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#configureDeliveryTimeout")]
1575 fn configure_delivery_timeout(config: &ProducerConfig, log_context: &LogContext) -> Result<i32, Error> {
1576 let mut delivery_timeout_ms = config.delivery_timeout_ms;
1577 let linger_ms = Self::linger_ms(config);
1578 let request_timeout_ms = config.request_timeout_ms;
1579 let linger_and_request_timeout_ms = (linger_ms as i64 + request_timeout_ms as i64).min(i32::MAX as i64) as i32;
1580
1581 if delivery_timeout_ms < linger_and_request_timeout_ms {
1582 if config.user_configured(ProducerConfig::DELIVERY_TIMEOUT_MS_CONFIG) {
1583 // Java `KafkaProducer#configureDeliveryTimeout`: throw if the user explicitly set an inconsistent value.
1584 // The class is `ConfigException`, inside the `KafkaException`
1585 // hierarchy (`KafkaProducerTest.testDeliveryTimeoutAndLingerMsConfig`
1586 // asserts `KafkaException.class`); `illegal_argument` put it outside,
1587 // where `is_kafka_error()` answers `false`.
1588 return Err(Error::config_message(format!(
1589 "{} should be equal to or larger than {} + {}",
1590 ProducerConfig::DELIVERY_TIMEOUT_MS_CONFIG,
1591 ProducerConfig::LINGER_MS_CONFIG,
1592 ProducerConfig::REQUEST_TIMEOUT_MS_CONFIG,
1593 )));
1594 }
1595 // Java `KafkaProducer#configureDeliveryTimeout`: override the default for backward compatibility.
1596 delivery_timeout_ms = linger_and_request_timeout_ms;
1597 kafka_warn!(
1598 log_context,
1599 "{} should be equal to or larger than {} + {}. Setting it to {}.",
1600 ProducerConfig::DELIVERY_TIMEOUT_MS_CONFIG,
1601 ProducerConfig::LINGER_MS_CONFIG,
1602 ProducerConfig::REQUEST_TIMEOUT_MS_CONFIG,
1603 delivery_timeout_ms
1604 );
1605 }
1606 Ok(delivery_timeout_ms)
1607 }
1608
1609 /// Returns the client ID for this producer.
1610 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#getClientId")]
1611 pub(crate) fn client_id(&self) -> &str {
1612 &self.client_id
1613 }
1614
1615 /// Returns the current time in milliseconds from `time`.
1616 fn now_ms(&self) -> i64 {
1617 self.time.milliseconds()
1618 }
1619
1620 /// `max.block.ms` as a [`Duration`], for the four transactional methods that
1621 /// bound their wait with it (`result.await(maxBlockTimeMs, MILLISECONDS)`).
1622 ///
1623 /// Clamped at zero: a negative `max.block.ms` cannot be configured, and
1624 /// `Duration` has no negative representation.
1625 fn max_block_timeout(&self) -> Duration {
1626 Duration::from_millis(self.max_block_ms.max(0) as u64)
1627 }
1628
1629 /// Needs to be called before any other methods when the `transactional.id`
1630 /// is set in the configuration. This method does the following:
1631 ///
1632 /// 1. Ensures any transactions initiated by previous instances of the producer
1633 /// with the same `transactional.id` are completed. If the previous instance
1634 /// had failed with a transaction in progress, it will be aborted. If the
1635 /// last transaction had begun completion, but not yet finished, this method
1636 /// awaits its completion.
1637 /// 2. Gets the internal producer id and epoch, used in all future
1638 /// transactional messages issued by the producer.
1639 ///
1640 /// Note that this method will return a timeout error if the transactional
1641 /// state cannot be initialized before expiration of `max.block.ms`.
1642 /// Additionally, it will return an interrupt error if interrupted. It is safe
1643 /// to retry in either case, but once the transactional state has been
1644 /// successfully initialized, this method should no longer be used.
1645 ///
1646 /// # Errors
1647 ///
1648 /// - `Error::is_local_illegal_state_error` if no `transactional.id` has been configured, or
1649 /// the producer has been closed
1650 /// - `Error::is_unsupported_version_error` fatal error indicating the broker does not support
1651 /// transactions (i.e. if its version is lower than 0.11.0.0)
1652 /// - `Error::is_authorization_error` error indicating that the configured transactional.id is
1653 /// not authorized, or the idempotent producer id is unavailable. See the error for more
1654 /// details. User may retry this function call after fixing the permission.
1655 /// - `Error::is_kafka_error` if the producer has encountered a previous fatal error or for any
1656 /// other unexpected error
1657 /// - `Error::is_timeout_error` if the time taken for initialize the transaction has surpassed
1658 /// `max.block.ms`.
1659 /// - `Error::is_interrupt_error` if the task is interrupted while blocked
1660 ///
1661 /// # Translation notes
1662 ///
1663 /// A Rust task is never interrupted: dropping the future cancels the wait.
1664 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#initTransactions")]
1665 pub fn init_transactions(&self) -> impl Future<Output = Result<(), Error>> + Send {
1666 async move {
1667 let transaction_manager = self.transaction_manager_or_error()?;
1668 self.ensure_not_closed()?;
1669 // Java: `long now = time.nanoseconds()` (`KafkaProducer`), fed to
1670 // `producerMetrics.recordInit(time.nanoseconds() - now)` (`KafkaProducer`) after a
1671 // successful await. Timed with the injected `time`, as Java times with its
1672 // injected `Time` — the same clock `flush` and `waitOnMetadata` read.
1673 let start = self.time.nanoseconds();
1674 let result = {
1675 // `pending_requests` before the manager, per the field docs.
1676 let mut pending_requests = self.pending_requests.lock().unwrap();
1677 transaction_manager
1678 .lock()
1679 .unwrap()
1680 .initialize_transactions(false, &mut pending_requests)?
1681 };
1682 self.wakeup.notify_one();
1683 result
1684 .await_result_timeout(self.max_block_timeout(), Self::INIT_TXN_TIMEOUT_MSG)
1685 .await?;
1686 // Java records only after a successful await (`KafkaProducer#initTransactions`), so the `?` above must
1687 // stay ahead of both this and `maybeUpdateTransactionV2Enabled`.
1688 self.producer_metrics.record_init(self.time.nanoseconds() - start);
1689 transaction_manager.lock().unwrap().maybe_update_transaction_v2_enabled(true);
1690 Ok(())
1691 }
1692 }
1693
1694 /// Should be called before the start of each new transaction. Note that prior
1695 /// to the first invocation of this method, you must invoke
1696 /// [`Self::init_transactions`] exactly one time.
1697 ///
1698 /// # Errors
1699 ///
1700 /// - `Error::is_local_illegal_state_error` if no `transactional.id` has been configured or if
1701 /// `Self::init_transactions` has not yet been invoked, or the producer has been closed
1702 /// - `Error::is_producer_fenced_error` if another producer with the same transactional.id is
1703 /// active
1704 /// - `Error::is_invalid_producer_epoch_error` if the producer has attempted to produce with an
1705 /// old epoch to the partition leader. See the error for more details
1706 /// - `Error::is_unsupported_version_error` fatal error indicating the broker does not support
1707 /// transactions (i.e. if its version is lower than 0.11.0.0)
1708 /// - `Error::is_authorization_error` fatal error indicating that the configured
1709 /// `transactional.id` is not authorized. See the error for more details
1710 /// - `Error::is_kafka_error` if the producer has encountered a previous fatal error or for any
1711 /// other unexpected error
1712 ///
1713 /// # Translation notes
1714 ///
1715 /// Synchronous, as Java's `beginTransaction` never blocks.
1716 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#beginTransaction")]
1717 pub fn begin_transaction(&self) -> Result<(), Error> {
1718 let transaction_manager = self.transaction_manager_or_error()?;
1719 self.ensure_not_closed()?;
1720 // Java: `long now = time.nanoseconds()` (`KafkaProducer`) around the
1721 // synchronous state transition, recorded on success (`KafkaProducer`). Java times this
1722 // even though there is no wait; kept faithful.
1723 let start = self.time.nanoseconds();
1724 transaction_manager.lock().unwrap().begin_transaction()?;
1725 self.producer_metrics.record_begin_txn(self.time.nanoseconds() - start);
1726 Ok(())
1727 }
1728
1729 /// Sends a list of specified offsets to the consumer group coordinator, and
1730 /// also marks those offsets as part of the current transaction. These offsets
1731 /// will be considered committed only if the transaction is committed
1732 /// successfully. The committed offset should be the next message your
1733 /// application will consume, i.e. `next_record_to_be_processed.offset()` (or
1734 /// `ConsumerRecords::next_offsets`). You should also add the leader epoch as
1735 /// commit metadata, which can be obtained from `ConsumerRecord::leader_epoch`
1736 /// or `ConsumerRecords::next_offsets`.
1737 ///
1738 /// This method should be used when you need to batch consumed and produced
1739 /// messages together, typically in a consume-transform-produce pattern. Thus,
1740 /// the specified `group_metadata` should be extracted from the used consumer
1741 /// via `KafkaConsumer::group_metadata` to leverage consumer group metadata.
1742 /// This will provide stronger fencing than just supplying the consumer group
1743 /// id, however note that the full set of consumer group metadata returned by
1744 /// `KafkaConsumer::group_metadata` requires the brokers to be on version 2.5
1745 /// or newer to understand.
1746 ///
1747 /// This method is a blocking call that waits until the request has been
1748 /// received and acknowledged by the consumer group coordinator; but the
1749 /// offsets are not considered as committed until the transaction itself is
1750 /// successfully committed later (via the [`Self::commit_transaction`] call).
1751 ///
1752 /// Note, that the consumer should have `enable.auto.commit=false` and should
1753 /// also not commit offsets manually (via sync or async commits). This method
1754 /// will return a timeout error if the producer cannot resolve the metadata
1755 /// for the topics in `offsets` and send the offsets before expiration of
1756 /// `max.block.ms`. Additionally, it will return an interrupt error if
1757 /// interrupted.
1758 ///
1759 /// # Arguments
1760 ///
1761 /// - `offsets` - the offsets to commit, per partition; an empty map does nothing
1762 /// - `group_metadata` - the consumer group metadata, from `KafkaConsumer::group_metadata`
1763 ///
1764 /// # Errors
1765 ///
1766 /// - `Error::is_local_illegal_argument_error` if `group_metadata` has a generation id greater
1767 /// than zero but an unknown member id
1768 /// - `Error::is_local_illegal_state_error` if no transactional.id has been configured or no
1769 /// transaction has been started, or the producer has been closed.
1770 /// - `Error::is_producer_fenced_error` fatal error indicating another producer with the same
1771 /// transactional.id is active
1772 /// - `Error::is_unsupported_version_error` fatal error indicating the broker does not support
1773 /// transactions (i.e. if its version is lower than 0.11.0.0) or the broker doesn't support
1774 /// the latest version of transactional API with all consumer group metadata (i.e. if its
1775 /// version is lower than 2.5.0).
1776 /// - `Error::is_unsupported_for_message_format_error` fatal error indicating the message format
1777 /// used for the offsets topic on the broker does not support transactions
1778 /// - `Error::is_authorization_error` fatal error indicating that the configured
1779 /// transactional.id is not authorized, or the consumer group id is not authorized.
1780 /// - `Error::is_consumer_commit_failed_error` if the commit failed and cannot be retried (e.g.
1781 /// if the consumer has been kicked out of the group). Users should handle this by aborting
1782 /// the transaction.
1783 /// - `Error::is_fenced_instance_id_error` if this producer instance gets fenced by broker due
1784 /// to a mis-configured consumer instance id within group metadata.
1785 /// - `Error::is_invalid_producer_epoch_error` if the producer has attempted to produce with an
1786 /// old epoch to the partition leader. See the error for more details
1787 /// - `Error::is_kafka_error` if the producer has encountered a previous fatal or abortable
1788 /// error, or for any other unexpected error
1789 /// - `Error::is_timeout_error` if the combined time taken for resolving topic metadata and
1790 /// sending the offsets has surpassed `max.block.ms`.
1791 /// - `Error::is_interrupt_error` if the task is interrupted while blocked
1792 /// - `Error::is_bootstrap_resolution_error` if DNS resolution of the bootstrap servers fails
1793 /// within `bootstrap.resolve.timeout.ms`
1794 ///
1795 /// # Translation notes
1796 ///
1797 /// A Rust task is never interrupted: dropping the future cancels the wait.
1798 /// `offsets` is taken by value, as the request carries it to the
1799 /// coordinator; `group_metadata` is borrowed and copied.
1800 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#sendOffsetsToTransaction")]
1801 pub fn send_offsets_to_transaction(
1802 &self,
1803 offsets: HashMap<TopicPartition, OffsetAndMetadata>,
1804 group_metadata: &dyn ConsumerGroupMetadata,
1805 ) -> impl Future<Output = Result<(), Error>> + Send {
1806 async move {
1807 Self::throw_if_invalid_group_metadata(group_metadata)?;
1808 let transaction_manager = self.transaction_manager_or_error()?;
1809 self.ensure_not_closed()?;
1810
1811 // `KafkaProducer#sendOffsetsToTransaction`: an empty map is a no-op, and in particular does not consult the
1812 // transaction state at all. It is also left unmetered — Java starts the timer
1813 // inside the `if (!offsets.isEmpty())` block, after this check.
1814 if offsets.is_empty() {
1815 return Ok(());
1816 }
1817
1818 // Java: `long start = time.nanoseconds()` inside the non-empty branch
1819 // (`KafkaProducer`), before the KIP-1319 metadata wait, so that wait
1820 // is part of the recorded duration; recorded after a successful await (`KafkaProducer`).
1821 let start = self.time.nanoseconds();
1822
1823 // KIP-1319 (6208dfc014): make sure the metadata cache knows these topics'
1824 // ids before the manager builds the request. No manager guard is held
1825 // across this await (`producer-transactions.md` §4).
1826 let topics: HashSet<&str> = offsets.keys().map(TopicPartition::topic).collect();
1827 let wait_ms = self.await_topic_metadata(&topics).await?;
1828 let remaining_ms = (self.max_block_ms - wait_ms).max(0);
1829 let result = {
1830 // `pending_requests` before the manager, per the field docs.
1831 let mut pending_requests = self.pending_requests.lock().unwrap();
1832 transaction_manager.lock().unwrap().send_offsets_to_transaction(
1833 offsets,
1834 ConsumerGroupMetadataImpl::copy_of(group_metadata),
1835 &mut pending_requests,
1836 )?
1837 };
1838 self.wakeup.notify_one();
1839 result
1840 .await_result_timeout(Duration::from_millis(remaining_ms as u64), Self::SEND_OFFSETS_TIMEOUT_MSG)
1841 .await?;
1842 self.producer_metrics.record_send_offsets(self.time.nanoseconds() - start);
1843 Ok(())
1844 }
1845 }
1846
1847 /// Request a partial metadata refresh for the given topics and await the next
1848 /// metadata update (up to `max.block.ms`). Returns the elapsed wait time so the
1849 /// caller can subtract it from its own `max.block.ms` budget.
1850 ///
1851 /// Translated from `KafkaProducer.awaitTopicMetadata(Set<String>)`
1852 /// (6208dfc014, with KIP-909's bootstrap check). Even when no refresh is
1853 /// needed, a permanent bootstrap failure is surfaced so every API call sees
1854 /// it; on the refresh branch `await_update` surfaces it through
1855 /// `maybe_return_fatal_error`, as Java's `awaitUpdate` does through
1856 /// `maybeThrowFatalException`.
1857 ///
1858 /// # Errors
1859 ///
1860 /// [`Error::Timeout`] (`"Failed to update metadata after <max.block.ms> ms."`)
1861 /// if no update arrives in time, [`Error::BootstrapResolution`] once bootstrap
1862 /// resolution has failed for good, or the fatal metadata error /
1863 /// closed-metadata error `await_update` reports.
1864 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#awaitTopicMetadata")]
1865 async fn await_topic_metadata(&self, topics: &HashSet<&str>) -> Result<i64, Error> {
1866 let start_nanos = self.time.nanoseconds();
1867 let Some(version) = self.metadata.add_with_topics(topics.iter().copied(), self.now_ms()) else {
1868 // Even when no metadata refresh is needed for these topics, a permanent bootstrap
1869 // failure must still be surfaced so every API call sees the error.
1870 self.metadata.maybe_return_bootstrap_fatal_error()?;
1871 return Ok(0);
1872 };
1873 self.wakeup.notify_one();
1874 self.metadata.await_update(version, self.max_block_ms).await?;
1875 let elapsed_nanos = self.time.nanoseconds() - start_nanos;
1876 self.producer_metrics.record_metadata_wait(elapsed_nanos);
1877 Ok(elapsed_nanos / 1_000_000)
1878 }
1879
1880 /// Commits the ongoing transaction. This method will flush any unsent records
1881 /// before actually committing the transaction.
1882 ///
1883 /// Further, if any of the [`send`](Producer::send) calls which were part of
1884 /// the transaction hit irrecoverable errors, this method will return the last
1885 /// received error immediately and the transaction will not be committed. So
1886 /// all `send` calls in a transaction must succeed in order for this method to
1887 /// succeed.
1888 ///
1889 /// If the transaction is committed successfully and this method returns
1890 /// without an error, it is guaranteed that all callbacks for records in the
1891 /// transaction will have been invoked and completed. Note that panics raised
1892 /// by callbacks are ignored; the producer proceeds to commit the transaction
1893 /// in any case.
1894 ///
1895 /// Note that this method will return a timeout error if the transaction
1896 /// cannot be committed before expiration of `max.block.ms`, but this does not
1897 /// mean the request did not actually reach the broker. In fact, it only
1898 /// indicates that we cannot get the acknowledgement response in time, so
1899 /// it's up to the application's logic to decide how to handle timeouts.
1900 /// Additionally, it will return an interrupt error if interrupted. It is safe
1901 /// to retry in either case, but it is not possible to attempt a different
1902 /// operation (such as [`Self::abort_transaction`]) since the commit may
1903 /// already be in the progress of completing. If not retrying, the only
1904 /// option is to close the producer.
1905 ///
1906 /// # Errors
1907 ///
1908 /// - `Error::is_local_illegal_state_error` if no transactional.id has been configured or no
1909 /// transaction has been started, or the producer has been closed
1910 /// - `Error::is_producer_fenced_error` fatal error indicating another producer with the same
1911 /// transactional.id is active
1912 /// - `Error::is_unsupported_version_error` fatal error indicating the broker does not support
1913 /// transactions (i.e. if its version is lower than 0.11.0.0)
1914 /// - `Error::is_authorization_error` fatal error indicating that the configured
1915 /// transactional.id is not authorized. See the error for more details
1916 /// - `Error::is_invalid_producer_epoch_error` if the producer has attempted to produce with an
1917 /// old epoch to the partition leader. See the error for more details
1918 /// - `Error::is_kafka_error` if the producer has encountered a previous fatal or abortable
1919 /// error, or for any other unexpected error
1920 /// - `Error::is_timeout_error` if the time taken for committing the transaction has surpassed
1921 /// `max.block.ms`.
1922 /// - `Error::is_interrupt_error` if the task is interrupted while blocked
1923 ///
1924 /// # Translation notes
1925 ///
1926 /// A Rust task is never interrupted: dropping the future cancels the wait.
1927 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#commitTransaction")]
1928 pub fn commit_transaction(&self) -> impl Future<Output = Result<(), Error>> + Send {
1929 async move {
1930 let transaction_manager = self.transaction_manager_or_error()?;
1931 self.ensure_not_closed()?;
1932 // Java: `long commitStart = time.nanoseconds()` (`KafkaProducer`),
1933 // recorded after a successful await (`KafkaProducer`).
1934 let start = self.time.nanoseconds();
1935 let result = {
1936 // `pending_requests` before the manager, per the field docs.
1937 let mut pending_requests = self.pending_requests.lock().unwrap();
1938 transaction_manager.lock().unwrap().begin_commit(&mut pending_requests)?
1939 };
1940 self.wakeup.notify_one();
1941 result
1942 .await_result_timeout(self.max_block_timeout(), Self::COMMIT_TXN_TIMEOUT_MSG)
1943 .await?;
1944 self.producer_metrics.record_commit_txn(self.time.nanoseconds() - start);
1945 Ok(())
1946 }
1947 }
1948
1949 /// Aborts the ongoing transaction. Any unflushed produce messages will be
1950 /// aborted when this call is made. This call will return an error
1951 /// immediately if any prior [`send`](Producer::send) calls failed with a
1952 /// producer-fenced error or an authorization error.
1953 ///
1954 /// Note that this method will return a timeout error if the transaction
1955 /// cannot be aborted before expiration of `max.block.ms`, but this does not
1956 /// mean the request did not actually reach the broker. In fact, it only
1957 /// indicates that we cannot get the acknowledgement response in time, so
1958 /// it's up to the application's logic to decide how to handle timeouts.
1959 /// Additionally, it will return an interrupt error if interrupted. It is safe
1960 /// to retry in either case, but it is not possible to attempt a different
1961 /// operation (such as [`Self::commit_transaction`]) since the abort may
1962 /// already be in the progress of completing. If not retrying, the only
1963 /// option is to close the producer.
1964 ///
1965 /// # Errors
1966 ///
1967 /// - `Error::is_local_illegal_state_error` if no transactional.id has been configured or no
1968 /// transaction has been started, or the producer has been closed
1969 /// - `Error::is_producer_fenced_error` fatal error indicating another producer with the same
1970 /// transactional.id is active
1971 /// - `Error::is_invalid_producer_epoch_error` if the producer has attempted to produce with an
1972 /// old epoch to the partition leader. See the error for more details
1973 /// - `Error::is_unsupported_version_error` fatal error indicating the broker does not support
1974 /// transactions (i.e. if its version is lower than 0.11.0.0)
1975 /// - `Error::is_authorization_error` fatal error indicating that the configured
1976 /// transactional.id is not authorized. See the error for more details
1977 /// - `Error::is_kafka_error` if the producer has encountered a previous fatal error or for any
1978 /// other unexpected error
1979 /// - `Error::is_timeout_error` if the time taken for aborting the transaction has surpassed
1980 /// `max.block.ms`.
1981 /// - `Error::is_interrupt_error` if the task is interrupted while blocked
1982 ///
1983 /// # Translation notes
1984 ///
1985 /// A Rust task is never interrupted: dropping the future cancels the wait.
1986 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#abortTransaction")]
1987 pub fn abort_transaction(&self) -> impl Future<Output = Result<(), Error>> + Send {
1988 async move {
1989 let transaction_manager = self.transaction_manager_or_error()?;
1990 self.ensure_not_closed()?;
1991 kafka_info!(self.log_context, "Aborting incomplete transaction");
1992 // Java: `long abortStart = time.nanoseconds()` (`KafkaProducer`),
1993 // taken after the log line, recorded after a successful await (`KafkaProducer`).
1994 let start = self.time.nanoseconds();
1995 let result = {
1996 // `pending_requests` before the manager, per the field docs.
1997 let mut pending_requests = self.pending_requests.lock().unwrap();
1998 // `Caller::App`: this runs on the application task (rules §1).
1999 transaction_manager
2000 .lock()
2001 .unwrap()
2002 .begin_abort(&mut pending_requests, Caller::App)?
2003 };
2004 self.wakeup.notify_one();
2005 result
2006 .await_result_timeout(self.max_block_timeout(), Self::ABORT_TXN_TIMEOUT_MSG)
2007 .await?;
2008 self.producer_metrics.record_abort_txn(self.time.nanoseconds() - start);
2009 Ok(())
2010 }
2011 }
2012
2013 /// The shared [`TransactionManager`], or the error Java's
2014 /// `throwIfNoTransactionManager()` throws.
2015 ///
2016 /// Java checks `transactionManager == null` only, so an *idempotent* producer
2017 /// passes this check and is rejected one level down by the manager's own
2018 /// `ensureTransactional()` with a different message. That split is preserved:
2019 /// this must not also test `is_transactional()`.
2020 fn transaction_manager_or_error(&self) -> Result<Arc<Mutex<TransactionManager>>, Error> {
2021 match &self.transaction_manager {
2022 Some(transaction_manager) => Ok(Arc::clone(transaction_manager)),
2023 None => Err(Error::local_illegal_state(format!(
2024 "Cannot use transactional methods without enabling transactions by setting the {} configuration property",
2025 ProducerConfig::TRANSACTIONAL_ID_CONFIG
2026 ))),
2027 }
2028 }
2029
2030 /// Validates the consumer group metadata handed to
2031 /// [`Self::send_offsets_to_transaction`].
2032 ///
2033 /// Translated from `KafkaProducer.throwIfInvalidGroupMetadata`
2034 /// (`KafkaProducer#throwIfInvalidGroupMetadata`). Java's first arm rejects a `null`
2035 /// argument; a `ConsumerGroupMetadata` value cannot be null in Rust, so the
2036 /// type system enforces that arm and only the second is translated.
2037 ///
2038 /// # Errors
2039 ///
2040 /// [`Error::LocalIllegalArgument`] when the generation id is greater than zero
2041 /// but the member id is unknown.
2042 fn throw_if_invalid_group_metadata(group_metadata: &dyn ConsumerGroupMetadata) -> Result<(), Error> {
2043 if group_metadata.generation_id() > 0 && group_metadata.member_id() == TxnOffsetCommitRequest::UNKNOWN_MEMBER_ID
2044 {
2045 return Err(Error::local_illegal_argument(format!(
2046 "Passed in group metadata {} has generationId > 0 but the member.id is unknown",
2047 group_metadata
2048 )));
2049 }
2050 Ok(())
2051 }
2052
2053 /// Verify that this producer instance has not been closed.
2054 ///
2055 /// Corresponds to Java's `throwIfProducerClosed()`.
2056 fn ensure_not_closed(&self) -> Result<(), Error> {
2057 if !self.running.load(Ordering::Acquire) {
2058 return Err(Error::local_illegal_state(
2059 "Cannot perform operation after producer has been closed",
2060 ));
2061 }
2062 Ok(())
2063 }
2064
2065 /// The time a send may spend blocked before its record reaches the
2066 /// accumulator: `max.block.ms` on the blocking path, nothing at all on the
2067 /// non-blocking one.
2068 fn max_block_ms_for(&self, wait: SendWait) -> i64 {
2069 match wait {
2070 SendWait::MaxBlock => self.max_block_ms,
2071 SendWait::Never => 0,
2072 }
2073 }
2074
2075 /// Implementation of asynchronously send a record to a topic.
2076 ///
2077 /// Translated from `KafkaProducer.doSend()`.
2078 ///
2079 /// For `ApiException`-type errors (record-too-large, invalid topic, an
2080 /// `ApiException` raised by a serializer, etc.), the callback is invoked with
2081 /// the error and a completed-with-error future is returned
2082 /// (`Ok(failed_future)`). This matches Java's contract where `send()` always
2083 /// returns a `Future` for API errors and always invokes the callback.
2084 ///
2085 /// Everything else is propagated as `Err(...)`: the generic runtime errors
2086 /// (`LocalIllegalState` when the producer is closed, or `LocalIllegalArgument`
2087 /// when a custom partitioner returns a negative partition —
2088 /// `KafkaProducer#partition`, which escapes `doSend` through
2089 /// `catch (Exception e)` and is rethrown out of `send()`) and the
2090 /// `KafkaException`s that are not `ApiException`s. `SerializationException` is
2091 /// one of the latter — it extends `KafkaException` directly — so a
2092 /// serialization failure is returned as `Err`, not as a failed future.
2093 ///
2094 /// `wait` decides what happens at the two points where Java blocks for up to
2095 /// `max.block.ms` before the record reaches the accumulator — `waitOnMetadata`
2096 /// and `BufferPool.allocate`: with [`SendWait::MaxBlock`] this is Java's
2097 /// `doSend`; with [`SendWait::Never`] a wait at either point is reported as
2098 /// [`TrySendOutcome::WouldBlock`] instead, with the callback handed back
2099 /// un-invoked (see [`try_send`](Self::try_send)). The blocking callers never
2100 /// see `WouldBlock`.
2101 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#doSend")]
2102 async fn do_send(
2103 &self,
2104 record: ProducerRecord<K, V>,
2105 callback: Option<Box<dyn Callback>>,
2106 wait: SendWait,
2107 ) -> Result<TrySendOutcome, Error> {
2108 self.ensure_not_closed()?;
2109
2110 // First make sure the metadata for the topic is available
2111 let now_ms = self.now_ms();
2112 let max_block_ms = self.max_block_ms_for(wait);
2113 let cluster_and_wait_time = match self
2114 .wait_on_metadata(record.topic(), record.partition(), now_ms, max_block_ms)
2115 .await
2116 {
2117 Ok(cwt) => cwt,
2118 Err(e) => {
2119 // `KafkaProducer#doSend` relabels a closed-producer race first, then the
2120 // outer catches dispatch on the resulting class. The relabel comes
2121 // before the would-block check: a producer closed while the send
2122 // was in flight is an `Err` on both paths, not something to retry.
2123 let e = self.relabel_if_closed_while_sending(e);
2124 // The non-blocking path's first would-block point. With a zero
2125 // budget, `wait_on_metadata` fails with a `TimeoutException` exactly
2126 // when it would have had to wait — the metadata request it issued
2127 // stays in flight, so a later attempt can succeed. Every other error
2128 // (`InvalidTopicException`, the topic's own error, a fatal metadata
2129 // error) is a real failure and dispatches as it would on `send`.
2130 if wait == SendWait::Never && matches!(e, Error::Timeout(_)) {
2131 return Ok(TrySendOutcome::WouldBlock(callback));
2132 }
2133 if e.is_api_error() {
2134 // Java's `AppendCallbacks.topicPartition()` (`KafkaProducer`):
2135 // no partition is computed yet, so the record's own, if it names one.
2136 let partition = record.partition().unwrap_or(RecordMetadata::UNKNOWN_PARTITION);
2137 return self
2138 .handle_api_error(e, record.topic(), partition, callback)
2139 .map(TrySendOutcome::Accepted);
2140 }
2141 return Err(e);
2142 },
2143 };
2144 let now_ms = now_ms + cluster_and_wait_time.waited_on_metadata_ms;
2145 let remaining_wait_ms = 0i64.max(max_block_ms - cluster_and_wait_time.waited_on_metadata_ms);
2146 let cluster = cluster_and_wait_time.cluster;
2147
2148 // Destructure the record to take ownership of key/value for serialization.
2149 // The headers stay owned and mutable until both serializers have run:
2150 // Java hands each serializer `record.headers()`, which it may add to
2151 // (`KafkaProducer`).
2152 let (record_topic, partition_opt, timestamp_opt, mut record_headers, key, value) = record.into_parts();
2153
2154 // Java's only catch around either serializer call is
2155 // `catch (ClassCastException cce)` (`KafkaProducer#doSend`),
2156 // which relabels a key/value whose runtime class does not match the
2157 // configured serializer. That cannot happen here — the serializer is
2158 // statically typed on `K` / `V` — so there is nothing to convert, and the
2159 // serializer's own error propagates with its class intact. `doSend`'s outer
2160 // catches then dispatch on that class exactly as Java does: an
2161 // `ApiException` (a schema-registry `TimeoutException`, say) fires the
2162 // callback and yields a failed future, anything else is returned
2163 // as `Err`. Rewriting every serializer error as
2164 // `SerializationException` flipped `is_retriable_error()` for the first
2165 // case and skipped its callback entirely.
2166 //
2167 // The serializers borrow `key` / `value`, so a serializer whose input
2168 // already is the bytes (`ByteArraySerializer`, `StringSerializer`) hands
2169 // back a borrow of the record's own buffer, and the accumulator's append
2170 // is the only copy (CLAUDE.md §14). Each serializer future lives inside
2171 // this method's own future, so awaiting one through `dyn` allocates
2172 // nothing (`InlineFuture`). The serialized bytes borrow only the record
2173 // and the serializer, not the headers, so the key's bytes are held while
2174 // the value serializer gets the same headers.
2175 let serialized_key = match DynSerializer::serialize_with_headers(
2176 &*self.key_serializer,
2177 &record_topic,
2178 &mut record_headers,
2179 key.as_ref(),
2180 )
2181 .await
2182 {
2183 Ok(bytes) => bytes,
2184 Err(e) if e.is_api_error() => {
2185 // `AppendCallbacks.topicPartition()`, as for the metadata wait.
2186 let partition = partition_opt.unwrap_or(RecordMetadata::UNKNOWN_PARTITION);
2187 return self
2188 .handle_api_error(e, &record_topic, partition, callback)
2189 .map(TrySendOutcome::Accepted);
2190 },
2191 Err(e) => return Err(e),
2192 };
2193
2194 let serialized_value = match DynSerializer::serialize_with_headers(
2195 &*self.value_serializer,
2196 &record_topic,
2197 &mut record_headers,
2198 value.as_ref(),
2199 )
2200 .await
2201 {
2202 Ok(bytes) => bytes,
2203 Err(e) if e.is_api_error() => {
2204 // `AppendCallbacks.topicPartition()`, as for the metadata wait.
2205 let partition = partition_opt.unwrap_or(RecordMetadata::UNKNOWN_PARTITION);
2206 return self
2207 .handle_api_error(e, &record_topic, partition, callback)
2208 .map(TrySendOutcome::Accepted);
2209 },
2210 Err(e) => return Err(e),
2211 };
2212
2213 // A custom partitioner is handed the *typed* key/value
2214 // (`KafkaProducer#partition` passes `record.key()` / `record.value()`),
2215 // so with one the partition is computed here, where the typed references
2216 // are still at hand. It is passed to `do_send_bytes` as an explicit
2217 // `Some(..)`, which short-circuits that method's own partition step so the
2218 // partitioner runs EXACTLY once, even a stateful one such as
2219 // `RoundRobinPartitioner`. Without one, `do_send_bytes` runs the built-in
2220 // key-hash partitioning on the serialized key.
2221 let partition = if self.partitioner.is_some() {
2222 Some(self.compute_partition(
2223 &record_topic,
2224 partition_opt,
2225 key.as_ref(),
2226 serialized_key.as_deref(),
2227 value.as_ref(),
2228 serialized_value.as_deref(),
2229 &cluster,
2230 )?)
2231 } else {
2232 partition_opt
2233 };
2234
2235 // `KafkaProducer`: once both serializers have run, the
2236 // headers are read-only and are sent as they are.
2237 Self::set_read_only(&mut record_headers);
2238 let headers = record_headers.to_array();
2239
2240 self.do_send_bytes(
2241 &record_topic,
2242 partition,
2243 timestamp_opt,
2244 serialized_key.as_deref(),
2245 serialized_value.as_deref(),
2246 headers,
2247 callback,
2248 now_ms,
2249 remaining_wait_ms,
2250 &cluster,
2251 wait,
2252 )
2253 .await
2254 }
2255
2256 /// The non-blocking form of `send`: the same path, except that it never waits
2257 /// on metadata or on buffer memory. It still awaits the key and value
2258 /// serializers, which complete at once when they are built-in but may wait
2259 /// for an asynchronous implementation; the C FFI therefore runs it on the
2260 /// calling thread only for a producer whose serializers are built-in.
2261 ///
2262 /// Java's `send` blocks for up to `max.block.ms` at two points before the
2263 /// record is appended — `waitOnMetadata` (`KafkaProducer`) and
2264 /// `BufferPool.allocate` inside `RecordAccumulator.append`. Where `send`
2265 /// would wait at either, this returns [`TrySendOutcome::WouldBlock`] at once,
2266 /// having done nothing to the record: no batch was touched, no future was
2267 /// created and `callback` was **not** invoked (it is handed back in the
2268 /// outcome). Everything else is exactly `send`: an appended record is
2269 /// [`TrySendOutcome::Accepted`] with its future, and a record `send` would
2270 /// have rejected with an `ApiException` — too large, say — is `Accepted` too,
2271 /// with the callback already fired and the future already failed, because
2272 /// the producer *did* resolve it. The `Err` cases are `send`'s `Err` cases: a
2273 /// closed producer, a bad partition, and the failures outside the
2274 /// `ApiException` hierarchy, a `SerializationException` among them.
2275 ///
2276 /// Asking for metadata is a side effect, not a wait, so an unknown topic is
2277 /// `WouldBlock` **and** a metadata request has been issued; the next attempt,
2278 /// once the response is in, is `Accepted`.
2279 ///
2280 /// There is no Java counterpart (DoD #7), which is why it is crate-internal:
2281 /// the public surface stays Java's `send`. The C FFI's non-blocking send uses
2282 /// it as its fast path — append on the caller's own thread when the producer
2283 /// is ready and its serializers are built-in, and fall back to the
2284 /// submission queue, rather than stall the caller, when it is not.
2285 pub(crate) async fn try_send(
2286 &self,
2287 record: ProducerRecord<K, V>,
2288 callback: Option<Box<dyn Callback>>,
2289 ) -> Result<TrySendOutcome, Error> {
2290 self.do_send(record, callback, SendWait::Never).await
2291 }
2292
2293 /// Common send path for already-serialized key/value bytes.
2294 ///
2295 /// Both [`do_send`](Self::do_send) (after serialization) and
2296 /// [`send`](KafkaProducer::<Vec<u8>, Vec<u8>>::send) (zero-copy borrowed path)
2297 /// delegate here for partition calculation, size validation, and accumulator
2298 /// append.
2299 ///
2300 /// `wait` decides what happens when the append has to wait for buffer memory:
2301 /// with [`SendWait::MaxBlock`] the `BufferExhaustedException` of an expired
2302 /// `remaining_wait_ms` is dispatched like any other `ApiException`; with
2303 /// [`SendWait::Never`] (`remaining_wait_ms` is then zero) it is reported as
2304 /// [`TrySendOutcome::WouldBlock`] instead, with the callback handed back
2305 /// un-invoked. The blocking callers therefore never see `WouldBlock`.
2306 #[expect(clippy::too_many_arguments)]
2307 async fn do_send_bytes(
2308 &self,
2309 topic: &str,
2310 partition: Option<i32>,
2311 timestamp: Option<i64>,
2312 key: Option<&[u8]>,
2313 value: Option<&[u8]>,
2314 headers: &[RecordHeader],
2315 callback: Option<Box<dyn Callback>>,
2316 now_ms: i64,
2317 remaining_wait_ms: i64,
2318 cluster: &Cluster,
2319 wait: SendWait,
2320 ) -> Result<TrySendOutcome, Error> {
2321 // `compute_partition` with `None` typed key/value: this method has only the
2322 // serialized bytes. When called from `do_send`'s custom-partitioner branch the
2323 // partition arrives pre-computed as `Some(..)`, so this short-circuits and the
2324 // partitioner is not consulted twice. On the borrowed `send(&[u8], &[u8])` path
2325 // the typed key/value ARE the bytes, so passing them as `serialized_*` (and
2326 // `None` typed) is the faithful call there.
2327 let partition = self.compute_partition(topic, partition, None, key, None, value, cluster)?;
2328
2329 // An `i64`: unlike a Java `byte[]`, a key or value slice can be longer than
2330 // `i32::MAX`, and such a record must still be rejected below.
2331 let serialized_size = AbstractRecords::estimate_size_in_bytes_upper_bound(
2332 RecordBatch::CURRENT_MAGIC_VALUE,
2333 self.compression_type,
2334 key,
2335 value,
2336 headers,
2337 );
2338 if let Err(err) = self.ensure_valid_record_size(serialized_size) {
2339 return self
2340 .handle_api_error(err, topic, partition, callback)
2341 .map(TrySendOutcome::Accepted);
2342 }
2343
2344 let timestamp = timestamp.unwrap_or(now_ms);
2345
2346 // Java's `accumulator.append(..)` dispatches virtually to `ChunkedRecordAccumulator.append`
2347 // for the incremental strategy (`KafkaProducer`); here the strategy picks the
2348 // accumulator explicitly (see `chunked_accumulator`). Each arm awaits its own concrete
2349 // future, so the dispatch costs a branch, not a boxed future (CLAUDE.md §13).
2350 let appended = match &self.chunked_accumulator {
2351 Some(chunked_accumulator) => {
2352 chunked_accumulator
2353 .append(
2354 topic,
2355 partition,
2356 timestamp,
2357 key,
2358 value,
2359 headers,
2360 callback,
2361 remaining_wait_ms,
2362 now_ms,
2363 cluster,
2364 )
2365 .await
2366 },
2367 None => {
2368 self.accumulator
2369 .append(
2370 topic,
2371 partition,
2372 timestamp,
2373 key,
2374 value,
2375 headers,
2376 callback,
2377 remaining_wait_ms,
2378 now_ms,
2379 cluster,
2380 )
2381 .await
2382 },
2383 };
2384 match appended {
2385 Ok(result) => {
2386 // `append` only ever returns an appended result, whose future and partition are
2387 // set (`RecordAppendResult::appended`); the other outcomes are signals between
2388 // the accumulator's own steps.
2389 let (Some(future), Some(topic_partition)) = (result.future, result.topic_partition) else {
2390 return Err(Error::local_illegal_state("append returned a result that was not appended"));
2391 };
2392 // Add the partition to the transaction (if in progress) after it has
2393 // been successfully appended to the accumulator. We cannot do it
2394 // before because the partition may be unknown. Note that the `Sender`
2395 // will refuse to dequeue batches from the accumulator until they have
2396 // been added to the transaction (`KafkaProducer`).
2397 //
2398 // `topic_partition` is what Java reads back as
2399 // `appendCallbacks.topicPartition()`. It is borrowed, not rebuilt:
2400 // constructing it here from `topic: &str` would allocate a `String` and
2401 // an `Arc<str>` and copy the topic name twice on **every** record, which
2402 // CLAUDE.md §13 forbids on the send path. The accumulator interns one
2403 // `Arc<str>` per topic and hands the `TopicPartition` back, so this costs
2404 // nothing. (Critic 44 issue 1.)
2405 if let Some(transaction_manager) = &self.transaction_manager {
2406 // `Caller::App`: this runs on the application task.
2407 //
2408 // The guard is bound to its own statement so it is released at the
2409 // `;`. It MUST NOT stay alive into the error handling below:
2410 // `handle_api_error` re-locks this same non-reentrant
2411 // `std::sync::Mutex` through `maybe_transition_to_error_state`, and an
2412 // `if let` scrutinee's temporaries live for the whole success arm —
2413 // edition 2024 only shortens them across the `else`. Written inline,
2414 // this self-deadlocks the application task.
2415 let add_partition = transaction_manager.lock().unwrap().maybe_add_partition(&topic_partition);
2416 if let Err(error) = add_partition {
2417 // `maybeAddPartition` throws across two of `doSend`'s catch
2418 // blocks, so the error class decides how the failure surfaces:
2419 //
2420 // - `ProducerFencedException` / `InvalidProducerEpochException`
2421 // (`TransactionManager#maybeFailWithError`)
2422 // are `ApiException`s: `catch (ApiException e)` records the
2423 // error state and returns a failed future.
2424 // - `IllegalStateException` — a send that is out of order with
2425 // the transactional API (`TransactionManager#maybeAddPartition`),
2426 // or a previous operation that timed out
2427 // (`TransactionManager#throwIfPendingState`), or a previous
2428 // invalid transition (`TransactionManager#maybeFailWithError`) —
2429 // is not even a `KafkaException`, so it reaches
2430 // `catch (Exception e)` and is rethrown out of `send()`.
2431 // - The bare `KafkaException` of `maybeFailWithError`'s last line is not an
2432 // `ApiException` either, so `catch (KafkaException e)`
2433 // rethrows it as well. Neither rethrowing block calls
2434 // `maybeTransitionToErrorState`.
2435 //
2436 // `is_api_error()` alone is the whole test. It used to be
2437 // `&& error.error() != Errors::UnknownServerError`, because a bare
2438 // `KafkaException` was then spelled
2439 // `Error::with_message(Errors::UnknownServerError, ..)`, which
2440 // resolves the code to `UnknownServerException` — an
2441 // `ApiException`. Every producer site now builds a bare
2442 // `KafkaException` as `Error::kafka_message(..)` / `Error::kafka_message_source(..)`
2443 // (the `Error::KafkaError` variant), for which `is_api_error()`
2444 // answers `false` directly, so the code-based workaround is gone.
2445 if error.is_api_error() {
2446 let partition = topic_partition.partition();
2447 // `None`, not a callback: unlike the pre-append failure paths
2448 // (`ensure_valid_record_size`, `wait_on_metadata`), the record
2449 // has ALREADY been appended here, and `append` above took
2450 // ownership of `callback` and registered it with the record's
2451 // future. The callback is therefore both (a) moved — no longer
2452 // available at this point — and (b) already carried by that
2453 // future, which fires it exactly once when the batch is later
2454 // completed/aborted. Firing it here too would double-invoke it,
2455 // breaking the exactly-once-per-record contract (CLAUDE.md §11.5).
2456 // (Java's `doSend` catch fires the raw `callback` at
2457 // `KafkaProducer#doSend`, but keeps it as a reference separate
2458 // from the `appendCallbacks` it registered, so Java can fire it
2459 // twice on this path; Rust's single-owner model fires it once.)
2460 return self
2461 .handle_api_error(error, topic, partition, None)
2462 .map(TrySendOutcome::Accepted);
2463 }
2464 return Err(error);
2465 }
2466 }
2467
2468 if result.batch_is_full || result.new_batch_created {
2469 kafka_trace!(
2470 self.log_context,
2471 "Waking up the sender since topic {} is either full or getting a new batch",
2472 topic
2473 );
2474 self.wakeup.notify_one();
2475 }
2476 Ok(TrySendOutcome::Accepted(KafkaFuture::new(future)))
2477 },
2478 // The non-blocking path's second would-block point. `remaining_wait_ms`
2479 // is zero here, so the pool gave up at once with `BufferExhaustedException`
2480 // (`BufferPool.allocate`, or `try_allocate_chunks` on the incremental
2481 // strategy) — on `send` with `max.block.ms = 0` that is an `ApiException`
2482 // and takes the arm below; `try_send` instead hands the record (and its
2483 // still un-invoked callback) back so the caller can retry it with `send`
2484 // when it can afford to block. The pool is untouched: a failed allocation
2485 // releases whatever it accumulated before returning.
2486 Err(failure) if wait == SendWait::Never && matches!(failure.error, Error::ProducerBufferExhausted(_)) => {
2487 Ok(TrySendOutcome::WouldBlock(failure.callback))
2488 },
2489 // Java's `catch (ApiException e)` (`KafkaProducer`) fires the
2490 // user `Callback` with a null-metadata `RecordMetadata(tp, -1, -1,
2491 // NO_TIMESTAMP, -1, -1)` *and* returns a failed future. `append` gives the
2492 // callback back (`AppendFailure::callback`) precisely so this arm can honour
2493 // that obligation exactly once (CLAUDE.md §11.5) — the four sibling arms
2494 // covering the same Java block all route through `handle_api_error` too.
2495 //
2496 // The callback is given the partition `append` chose, as Java reports
2497 // `appendCallbacks.topicPartition()`, not the
2498 // `partition` passed in, which is `UNKNOWN_PARTITION` when the built-in
2499 // partitioner chooses.
2500 Err(failure) if failure.error.is_api_error() => self
2501 .handle_api_error(failure.error, topic, failure.partition, failure.callback)
2502 .map(TrySendOutcome::Accepted),
2503 // Java's `catch (KafkaException e)` / `catch (Exception e)`
2504 // rethrow, and neither invokes the callback. `failure.callback` is dropped
2505 // here, exactly as Java drops its `appendCallbacks` reference when `send()`
2506 // throws.
2507 Err(failure) => Err(failure.error),
2508 }
2509 }
2510
2511 /// `transactionManager.maybeTransitionToErrorState(e)`, the tail of
2512 /// `KafkaProducer.doSend`'s `catch (ApiException e)` block
2513 /// (`KafkaProducer#doSend`).
2514 ///
2515 /// [`Caller::App`](crate::producer::internals::Caller::App): `doSend` runs on the
2516 /// application task.
2517 ///
2518 /// Locks the [`TransactionManager`]. Java's monitor is reentrant and
2519 /// `std::sync::Mutex` is not, so no caller — directly or through
2520 /// [`handle_api_error`](Self::handle_api_error) — may already hold that
2521 /// lock, including in a still-live `if let` / `match` scrutinee temporary.
2522 fn maybe_transition_to_error_state(&self, error: &Error) {
2523 if let Some(transaction_manager) = &self.transaction_manager {
2524 // Java lets an invalid transition propagate out of `doSend`. That cannot
2525 // happen on the idempotent path — the only transition
2526 // `maybeTransitionToErrorState` performs is to `FATAL_ERROR`, which is
2527 // always valid — and swallowing it here would hide a Phase-5 regression,
2528 // so it is logged rather than dropped.
2529 if let Err(transition_error) = transaction_manager
2530 .lock()
2531 .unwrap()
2532 .maybe_transition_to_error_state(error, Caller::App)
2533 {
2534 kafka_warn!(
2535 self.log_context,
2536 "Failed to record a send error in the transaction manager: {}",
2537 transition_error
2538 );
2539 }
2540 }
2541 }
2542
2543 /// `doSend`'s **inner** `catch (KafkaException e)` around `waitOnMetadata`
2544 /// (`KafkaProducer#doSend`):
2545 ///
2546 /// ```java
2547 /// } catch (KafkaException e) {
2548 /// if (metadata.isClosed())
2549 /// throw new KafkaException("Producer closed while send in progress", e);
2550 /// throw e;
2551 /// }
2552 /// ```
2553 ///
2554 /// Its whole job is to relabel a `close()` racing an in-flight send, so the
2555 /// caller can tell it apart from a metadata timeout. Anything else — and
2556 /// anything outside the `KafkaException` hierarchy — passes through untouched,
2557 /// to be dispatched by the outer `catch (ApiException e)` /
2558 /// `catch (KafkaException e)` split at the call site.
2559 fn relabel_if_closed_while_sending(&self, error: Error) -> Error {
2560 if error.is_kafka_error() && self.metadata.is_closed() {
2561 // A bare `KafkaException`, matching Java: not an `ApiException`, so the
2562 // caller returns it as `Err` rather than as a failed future.
2563 return Error::kafka_message_source("Producer closed while send in progress", error);
2564 }
2565 error
2566 }
2567
2568 /// Handle an `ApiException`-type error by invoking the callback (if any)
2569 /// and returning a completed-with-error future.
2570 ///
2571 /// This matches Java's `catch (ApiException e)` block in `doSend()`.
2572 fn handle_api_error(
2573 &self,
2574 error: Error,
2575 topic: &str,
2576 partition: i32,
2577 callback: Option<Box<dyn Callback>>,
2578 ) -> Result<KafkaFuture<RecordMetadata>, Error> {
2579 kafka_debug!(self.log_context, "Error occurred during message send: {}", error);
2580 self.maybe_transition_to_error_state(&error);
2581 if let Some(cb) = callback {
2582 let tp = TopicPartition::new(topic.to_string(), partition);
2583 let null_metadata = RecordMetadata::new(tp, -1, -1, RecordBatch::NO_TIMESTAMP, -1, -1);
2584 cb.on_completion(Some(&null_metadata), Some(&error));
2585 }
2586 let tp = TopicPartition::new(topic.to_string(), partition);
2587 Ok(KafkaFuture::new(Arc::new(FutureRecordMetadata::failed(tp, error))))
2588 }
2589
2590 /// Wait for cluster metadata including partitions for the given topic to be available.
2591 ///
2592 /// Translated from `KafkaProducer.waitOnMetadata()`.
2593 ///
2594 /// # Arguments
2595 /// * `topic` - The topic we want metadata for
2596 /// * `partition` - A specific partition expected to exist in metadata, or `None`
2597 /// * `now_ms` - The current time in ms
2598 /// * `max_wait_ms` - The maximum time in ms for waiting on the metadata
2599 ///
2600 /// # Returns
2601 /// The cluster containing topic metadata and the amount of time we waited in ms.
2602 ///
2603 /// # Errors
2604 /// Returns `Err` if:
2605 /// - The topic is invalid ([`InvalidTopic`](Error::InvalidTopic))
2606 /// - Metadata could not be refreshed within `max_wait_ms` ([`Timeout`](Error::Timeout))
2607 /// - The producer is closed
2608 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#waitOnMetadata")]
2609 async fn wait_on_metadata(
2610 &self,
2611 topic: &str,
2612 partition: Option<i32>,
2613 now_ms: i64,
2614 max_wait_ms: i64,
2615 ) -> Result<ClusterAndWaitTime, Error> {
2616 let cluster = self.metadata.fetch();
2617
2618 if cluster.invalid_topics().contains(topic) {
2619 // Java: `throw new InvalidTopicException(topic)` — `topic` is a
2620 // `String`, so this binds to the `(String message)` constructor:
2621 // the message is the topic name and `invalidTopics()` is empty.
2622 return Err(Error::invalid_topics_message(
2623 std::collections::HashSet::new(),
2624 topic.to_string(),
2625 ));
2626 }
2627
2628 // Add topic to metadata topic list if it is not there already and reset expiry
2629 self.metadata.add(topic, now_ms);
2630
2631 let partitions_count = cluster.partition_count_for_topic(topic);
2632 // Return cached metadata if we have it, and if the record's partition is either
2633 // undefined or within the known partition range
2634 if let Some(count) = partitions_count
2635 && (partition.is_none() || partition.unwrap() < count as i32)
2636 {
2637 return Ok(ClusterAndWaitTime { cluster, waited_on_metadata_ms: 0 });
2638 }
2639
2640 let mut remaining_wait_ms = max_wait_ms;
2641 let mut elapsed: i64 = 0;
2642 let mut partitions_count = partitions_count;
2643
2644 // Java `waitOnMetadata`: `long nowNanos = time.nanoseconds()` right
2645 // before the refresh loop (after the early cached-metadata return), and
2646 // `producerMetrics.recordMetadataWait(time.nanoseconds() - nowNanos)`
2647 // once the loop succeeds. A timeout inside the loop throws before the
2648 // record, so only successful waits are recorded — preserved here.
2649 let now_nanos = self.time.nanoseconds();
2650
2651 // Issue metadata requests until we have metadata for the topic and the
2652 // requested partition, or until max_wait_ms is exceeded.
2653 loop {
2654 if let Some(p) = partition {
2655 kafka_trace!(
2656 self.log_context,
2657 "Requesting metadata update for partition {} of topic {}.",
2658 p,
2659 topic
2660 );
2661 } else {
2662 kafka_trace!(self.log_context, "Requesting metadata update for topic {}.", topic);
2663 }
2664 self.metadata.add(topic, now_ms + elapsed);
2665 let version = self.metadata.request_update_for_topic(topic);
2666 self.wakeup.notify_one();
2667
2668 match self.metadata.await_update(version, remaining_wait_ms).await {
2669 Ok(()) => {},
2670 // A fatal metadata error (an authentication failure, say) must
2671 // reach the caller as itself. Java's `awaitUpdate` throws it from
2672 // the wait predicate via `maybeThrowFatalException()`, so it never
2673 // becomes a `TimeoutException`; only an actual deadline expiry does.
2674 // Re-wrapping everything as a timeout made a dead producer look
2675 // retriable, so the application retried bad credentials forever.
2676 Err(e) if !e.is_timeout_error() => return Err(e),
2677 Err(_) => {
2678 // Rethrow with the original `max_wait_ms` to keep the message
2679 // free of the shrinking `remaining_wait_ms` (`KafkaProducer#waitOnMetadata`).
2680 let error_message = self.get_error_message(partitions_count, topic, partition, max_wait_ms);
2681 if let Some(code) = self.metadata.get_error(topic) {
2682 // `new TimeoutException(errorMessage, metadata.getError(topic).exception())`:
2683 // the broker error is the timeout's *cause*, so
2684 // `Error::source()` can be walked back to it. Flattening it
2685 // into the message left `source()` empty.
2686 return Err(Error::Timeout(TimeoutError::with_source(error_message, Error::new(code))));
2687 }
2688 return Err(Error::timeout(error_message));
2689 },
2690 }
2691
2692 let cluster = self.metadata.fetch();
2693 elapsed = self.now_ms() - now_ms;
2694 if elapsed >= max_wait_ms {
2695 let error_message = self.get_error_message(partitions_count, topic, partition, max_wait_ms);
2696 // `KafkaProducer#waitOnMetadata` attaches the topic's error as the cause here too,
2697 // but only when it is retriable — a non-retriable one is about to be
2698 // raised as itself by `maybe_return_error_for_topic` on the next
2699 // iteration, so pinning it under a timeout would mislabel it.
2700 if let Some(code) = self.metadata.get_error(topic) {
2701 let underlying = Error::new(code);
2702 if underlying.is_retriable_error() {
2703 return Err(Error::Timeout(TimeoutError::with_source(error_message, underlying)));
2704 }
2705 }
2706 return Err(Error::timeout(error_message));
2707 }
2708 self.metadata.maybe_return_error_for_topic(topic)?;
2709 remaining_wait_ms = max_wait_ms - elapsed;
2710 partitions_count = cluster.partition_count_for_topic(topic);
2711
2712 let done = match partitions_count {
2713 None => false,
2714 Some(count) => partition.is_none() || partition.unwrap() < count as i32,
2715 };
2716 if done {
2717 self.producer_metrics.record_metadata_wait(self.time.nanoseconds() - now_nanos);
2718 return Ok(ClusterAndWaitTime { cluster, waited_on_metadata_ms: elapsed });
2719 }
2720 }
2721 }
2722
2723 /// Format the error message for a metadata wait timeout.
2724 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#getErrorMessage")]
2725 fn get_error_message(
2726 &self,
2727 partitions_count: Option<usize>,
2728 topic: &str,
2729 partition: Option<i32>,
2730 max_wait_ms: i64,
2731 ) -> String {
2732 match partitions_count {
2733 None => format!("Topic {} not present in metadata after {} ms.", topic, max_wait_ms),
2734 Some(count) => format!(
2735 "Partition {} of topic {} with partition count {} is not present in metadata after {} ms.",
2736 partition.unwrap_or(-1),
2737 topic,
2738 count,
2739 max_wait_ms
2740 ),
2741 }
2742 }
2743
2744 /// Validate that the record size isn't too large.
2745 ///
2746 /// Translated from `KafkaProducer.ensureValidRecordSize()`.
2747 ///
2748 /// Takes an `i64` so that a size above `i32::MAX`, which
2749 /// `AbstractRecords::estimate_size_in_bytes_upper_bound` can return, is rejected
2750 /// and reported exactly.
2751 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#ensureValidRecordSize")]
2752 fn ensure_valid_record_size(&self, size: i64) -> Result<(), Error> {
2753 if size > self.max_request_size as i64 {
2754 return Err(Error::record_too_large(format!(
2755 "The message is {} bytes when serialized which is larger than {}, which is the value of the {} configuration.",
2756 size,
2757 self.max_request_size,
2758 ProducerConfig::MAX_REQUEST_SIZE_CONFIG
2759 )));
2760 }
2761 if size > self.total_memory_size {
2762 return Err(Error::record_too_large(format!(
2763 "The message is {} bytes when serialized which is larger than the total memory buffer you have configured with the {} configuration.",
2764 size,
2765 ProducerConfig::BUFFER_MEMORY_CONFIG
2766 )));
2767 }
2768 Ok(())
2769 }
2770
2771 /// Resolve the key-based partition when no explicit partition was supplied,
2772 /// or [`UNKNOWN_PARTITION`](RecordMetadata::UNKNOWN_PARTITION) to defer to
2773 /// the keyless (sticky, KIP-794) path.
2774 ///
2775 /// Mirrors the key branch of Java's `KafkaProducer.partition()`: the key is
2776 /// hashed only when it is present, keys are not being ignored
2777 /// (`partitioner.ignore.keys=false`), the configured [`KeyHasher`] actually
2778 /// hashes this key ([`KeyHasher::hashes_key`] — CRC-32 leaves an *empty* key
2779 /// to the keyless path, matching librdkafka `consistent_random`; murmur2
2780 /// hashes it, matching the Java client), and the topic has known partitions.
2781 ///
2782 /// Both send call sites ([`do_send_bytes`](Self::do_send_bytes) and
2783 /// [`partition`](Self::partition)) route their key branch through this one
2784 /// helper so the hashing gate cannot diverge between them.
2785 fn partition_for_key_or_unknown(&self, key: Option<&[u8]>, topic: &str, cluster: &Cluster) -> i32 {
2786 if let Some(k) = key
2787 && !self.partitioner_ignore_keys
2788 && self.key_hasher.hashes_key(k)
2789 {
2790 let num_partitions = cluster.partitions_for_topic(topic).len() as i32;
2791 if num_partitions > 0 {
2792 return BuiltInPartitioner::partition_for_key(k, num_partitions, self.key_hasher);
2793 }
2794 }
2795 RecordMetadata::UNKNOWN_PARTITION
2796 }
2797
2798 /// Compute the partition for a record, mirroring Java's `KafkaProducer.partition()`
2799 /// (`KafkaProducer#partition`).
2800 ///
2801 /// Precedence is exactly Java's:
2802 /// 1. an explicit `partition` on the record wins outright;
2803 /// 2. otherwise, if a custom [`Partitioner`] is configured, it decides — and a
2804 /// negative result is rejected with the same `IllegalArgumentException` message
2805 /// Java throws. In Java that exception escapes `doSend`'s `catch (Exception e)`
2806 /// and is rethrown out of `send()`; here it is the `Err` variant, propagated by
2807 /// the caller with `?`.
2808 /// 3. otherwise fall back to the built-in key-hash / `UNKNOWN_PARTITION` path
2809 /// ([`partition_for_key_or_unknown`](Self::partition_for_key_or_unknown)).
2810 ///
2811 /// The typed `key` / `value` are handed straight to the custom partitioner (Java
2812 /// passes it `record.key()` / `record.value()` alongside the serialized bytes); the
2813 /// built-in fallback in step 3 uses only `serialized_key`.
2814 #[expect(clippy::too_many_arguments)]
2815 fn compute_partition(
2816 &self,
2817 topic: &str,
2818 partition: Option<i32>,
2819 key: Option<&K>,
2820 serialized_key: Option<&[u8]>,
2821 value: Option<&V>,
2822 serialized_value: Option<&[u8]>,
2823 cluster: &Cluster,
2824 ) -> Result<i32, Error> {
2825 if let Some(p) = partition {
2826 return Ok(p);
2827 }
2828
2829 if let Some(partitioner) = &self.partitioner {
2830 let custom_partition = partitioner.partition(topic, key, serialized_key, value, serialized_value, cluster);
2831 if custom_partition < 0 {
2832 return Err(Error::local_illegal_argument(format!(
2833 "The partitioner generated an invalid partition number: {}. Partition number should always be non-negative.",
2834 custom_partition
2835 )));
2836 }
2837 return Ok(custom_partition);
2838 }
2839
2840 Ok(self.partition_for_key_or_unknown(serialized_key, topic, cluster))
2841 }
2842
2843 /// Compute partition for the given record.
2844 ///
2845 /// A thin test wrapper over [`compute_partition`](Self::compute_partition), which is
2846 /// the faithful translation of `KafkaProducer.partition()`. Every caller is a test
2847 /// that uses a producer WITHOUT a custom partitioner, so the negative-partition `Err`
2848 /// path is unreachable and unwrapping it here is safe.
2849 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#partition")]
2850 fn partition(
2851 &self,
2852 record: &ProducerRecord<K, V>,
2853 serialized_key: Option<&[u8]>,
2854 serialized_value: Option<&[u8]>,
2855 cluster: &Cluster,
2856 ) -> i32 {
2857 self.compute_partition(
2858 record.topic(),
2859 record.partition(),
2860 record.key(),
2861 serialized_key,
2862 record.value(),
2863 serialized_value,
2864 cluster,
2865 )
2866 .expect("partition() test helper is only used without a custom partitioner")
2867 }
2868
2869 /// Initiate a graceful close of the sender.
2870 ///
2871 /// Closes the accumulator first to guarantee that no more appends are
2872 /// accepted after breaking from the sender loop. Otherwise, we may miss
2873 /// some callbacks when shutting down.
2874 ///
2875 /// Translated from `Sender.initiateClose()`.
2876 fn initiate_close(&self) {
2877 // Ensure accumulator is closed first to guarantee that no more appends
2878 // are accepted after breaking from the sender loop.
2879 self.accumulator.close();
2880 self.running.store(false, Ordering::Release);
2881 self.wakeup.notify_one();
2882 }
2883
2884 /// Whether no close has started on this producer.
2885 ///
2886 /// Every close clears `running` in [`Self::initiate_close`].
2887 pub(crate) fn is_running(&self) -> bool {
2888 self.running.load(Ordering::Acquire)
2889 }
2890
2891 /// Force-close the sender, aborting all pending batches.
2892 fn force_close(&self) {
2893 self.force_close.store(true, Ordering::Release);
2894 self.initiate_close();
2895 }
2896
2897 /// Await the sender task handle with a timeout.
2898 ///
2899 /// Takes the `JoinHandle` from the mutex and awaits it with the given
2900 /// timeout. Returns `true` if the sender task completed within the
2901 /// timeout, `false` if it is still running.
2902 ///
2903 /// If there is no sender handle (e.g. in tests that don't spawn a sender),
2904 /// returns `true` immediately.
2905 ///
2906 /// Corresponds to Java's `ioThread.join(closeTimer.remainingMs())`.
2907 ///
2908 /// On expiry the handle is **put back**, because Java's `close` force-closes and
2909 /// then joins unconditionally (`KafkaProducer#close`) — dropping it here
2910 /// would leave [`Self::await_sender_handle_indefinitely`] with nothing to join and
2911 /// `close` would return while the Sender task was still running, which CLAUDE.md
2912 /// §11.4 forbids. `JoinHandle` is `Unpin`, so `&mut` is enough to await it without
2913 /// giving it away.
2914 async fn await_sender_handle(&self, timeout: Duration) -> bool {
2915 let handle = self.sender_handle.lock().unwrap().take();
2916 match handle {
2917 None => true,
2918 Some(mut join_handle) => {
2919 let completed = tokio::time::timeout(timeout, &mut join_handle).await.is_ok();
2920 if !completed {
2921 *self.sender_handle.lock().unwrap() = Some(join_handle);
2922 }
2923 completed
2924 },
2925 }
2926 }
2927
2928 /// Await the sender task handle indefinitely.
2929 ///
2930 /// Called after force-close to ensure the sender task has exited.
2931 /// Corresponds to Java's `ioThread.join()` (no timeout).
2932 async fn await_sender_handle_indefinitely(&self) {
2933 let handle = self.sender_handle.lock().unwrap().take();
2934 if let Some(join_handle) = handle {
2935 let _ = join_handle.await;
2936 }
2937 }
2938}
2939
2940impl KafkaProducer<Vec<u8>, Vec<u8>> {
2941 /// Asynchronously send a record to a topic and invoke the provided callback
2942 /// when the send has been acknowledged. See
2943 /// [`Producer::send_with_callback`] for details.
2944 ///
2945 /// # Arguments
2946 ///
2947 /// - `record` - The record to send. If the topic or the partition specified in it cannot be
2948 /// found in metadata within `max.block.ms`, the returned future will time out when retrieved.
2949 /// - `callback` - A user-supplied callback to execute when the record has been acknowledged by
2950 /// the server (`None` indicates no callback)
2951 ///
2952 /// # Returns
2953 ///
2954 /// A future which will eventually contain the response information.
2955 ///
2956 /// # Errors
2957 ///
2958 /// - `Error::is_local_illegal_state_error` if a transactional.id has been configured and no
2959 /// transaction has been started, or when send is invoked after producer has been closed.
2960 /// - `Error::is_interrupt_error` If the task is interrupted while blocked
2961 /// - `Error::is_serialization_error` If the key or value are not valid objects given the
2962 /// configured serializers
2963 /// - `Error::is_bootstrap_resolution_error` if DNS resolution of the bootstrap servers fails
2964 /// within `bootstrap.resolve.timeout.ms`
2965 /// - `Error::is_kafka_error` If a Kafka related error occurs that does not belong to the public
2966 /// API errors.
2967 ///
2968 /// # Translation notes
2969 ///
2970 /// The byte-slice key and value bypass serialization, so no serialization
2971 /// error is returned: they are written straight into the batch buffer, the
2972 /// zero-copy path for callers that already hold `&[u8]` data (the C FFI).
2973 /// A Rust task is never interrupted: dropping the future cancels the wait.
2974 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#send")]
2975 pub fn send(
2976 &self,
2977 record: ProducerRecord<&[u8], &[u8]>,
2978 callback: Option<Box<dyn Callback>>,
2979 ) -> impl Future<Output = Result<KafkaFuture<RecordMetadata>, Error>> + Send {
2980 async move {
2981 self.ensure_not_closed()?;
2982
2983 let now_ms = self.now_ms();
2984 let cluster_and_wait_time = match self
2985 .wait_on_metadata(record.topic(), record.partition(), now_ms, self.max_block_ms)
2986 .await
2987 {
2988 Ok(cwt) => cwt,
2989 Err(e) => {
2990 // `KafkaProducer#doSend`'s relabel, as in `do_send`.
2991 let e = self.relabel_if_closed_while_sending(e);
2992 if e.is_api_error() {
2993 // Java's `AppendCallbacks.topicPartition()` (`KafkaProducer`):
2994 // no partition is computed yet, so the record's own, if it names one.
2995 let partition = record.partition().unwrap_or(RecordMetadata::UNKNOWN_PARTITION);
2996 return self.handle_api_error(e, record.topic(), partition, callback);
2997 }
2998 return Err(e);
2999 },
3000 };
3001 let now_ms = now_ms + cluster_and_wait_time.waited_on_metadata_ms;
3002 let remaining_wait_ms = 0i64.max(self.max_block_ms - cluster_and_wait_time.waited_on_metadata_ms);
3003 let cluster = cluster_and_wait_time.cluster;
3004
3005 let (record_topic, partition, timestamp, mut record_headers, key, value) = record.into_parts();
3006
3007 // Java's `doSend` makes the headers read-only and appends
3008 // `record.headers().toArray()` (`KafkaProducer#doSend`,
3009 // `KafkaProducer#doSend`); the header bytes are written straight into the
3010 // batch buffer, as on the typed path.
3011 Self::set_read_only(&mut record_headers);
3012 self.do_send_bytes(
3013 &record_topic,
3014 partition,
3015 timestamp,
3016 key,
3017 value,
3018 record_headers.to_array(),
3019 callback,
3020 now_ms,
3021 remaining_wait_ms,
3022 &cluster,
3023 SendWait::MaxBlock,
3024 )
3025 .await?
3026 .into_accepted_future()
3027 }
3028 }
3029}
3030
3031impl<K, V> Producer<K, V> for KafkaProducer<K, V>
3032where
3033 K: Send + Sync,
3034 V: Send + Sync,
3035{
3036 // Needs to be called before any other method when the `transactional.id` is
3037 // set in the configuration.
3038 async fn init_transactions(&self) -> Result<(), Error> {
3039 KafkaProducer::init_transactions(self).await
3040 }
3041
3042 // Should be called before the start of each new transaction.
3043 fn begin_transaction(&self) -> Result<(), Error> {
3044 KafkaProducer::begin_transaction(self)
3045 }
3046
3047 // Sends a list of specified offsets to the consumer group coordinator, and
3048 // also marks those offsets as part of the current transaction.
3049 async fn send_offsets_to_transaction(
3050 &self,
3051 offsets: HashMap<TopicPartition, OffsetAndMetadata>,
3052 group_metadata: &dyn ConsumerGroupMetadata,
3053 ) -> Result<(), Error> {
3054 KafkaProducer::send_offsets_to_transaction(self, offsets, group_metadata).await
3055 }
3056
3057 // Commits the ongoing transaction.
3058 async fn commit_transaction(&self) -> Result<(), Error> {
3059 KafkaProducer::commit_transaction(self).await
3060 }
3061
3062 // Aborts the ongoing transaction.
3063 async fn abort_transaction(&self) -> Result<(), Error> {
3064 KafkaProducer::abort_transaction(self).await
3065 }
3066
3067 // Asynchronously send a record to a topic.
3068 //
3069 // See [`send_with_callback`](Producer::send_with_callback) for details.
3070 async fn send(&self, record: ProducerRecord<K, V>) -> Result<KafkaFuture<RecordMetadata>, Error> {
3071 self.do_send(record, None, SendWait::MaxBlock).await?.into_accepted_future()
3072 }
3073
3074 // Asynchronously send a record to a topic and invoke the provided callback
3075 // when the send has been acknowledged.
3076 async fn send_with_callback(
3077 &self,
3078 record: ProducerRecord<K, V>,
3079 callback: Option<Box<dyn Callback>>,
3080 ) -> Result<KafkaFuture<RecordMetadata>, Error> {
3081 self.do_send(record, callback, SendWait::MaxBlock).await?.into_accepted_future()
3082 }
3083
3084 // Invoking this method makes all buffered records immediately available to
3085 // send and awaits the completion of the requests associated with these
3086 // records.
3087 //
3088 // Translated from `KafkaProducer.flush()`.
3089 async fn flush(&self) -> Result<(), Error> {
3090 kafka_trace!(self.log_context, "Flushing accumulated records in producer.");
3091 // Java: `long start = time.nanoseconds()` then a try/finally recording
3092 // `producerMetrics.recordFlush(time.nanoseconds() - start)`
3093 // (`KafkaProducer`). `await_flush_completion` returns
3094 // `()` (no error channel), so the finally reduces to recording after
3095 // the await.
3096 let start = self.time.nanoseconds();
3097 self.accumulator.begin_flush();
3098 self.wakeup.notify_one();
3099 self.accumulator.await_flush_completion().await;
3100 self.producer_metrics.record_flush(self.time.nanoseconds() - start);
3101 Ok(())
3102 }
3103
3104 // Get the full set of producer metrics maintained by this producer.
3105 //
3106 // Translated from `KafkaProducer.metrics()` — a live view of the registry.
3107 fn metrics(&self) -> MetricsView {
3108 self.metrics.metrics()
3109 }
3110
3111 // Get the partition metadata for the given topic.
3112 async fn partitions_for(&self, topic: &str) -> Result<Vec<PartitionInfo>, Error> {
3113 let now_ms = self.now_ms();
3114 let cluster_and_wait_time = self.wait_on_metadata(topic, None, now_ms, self.max_block_ms).await?;
3115 Ok(cluster_and_wait_time.cluster.partitions_for_topic(topic).to_vec())
3116 }
3117
3118 // Close this producer. This method awaits until all previously sent requests
3119 // complete.
3120 async fn close(&self) -> Result<(), Error> {
3121 self.close_with_timeout(Duration::from_millis(i64::MAX as u64)).await
3122 }
3123
3124 // Close this producer, waiting up to the given timeout for pending requests
3125 // to complete.
3126 //
3127 // Translated from `KafkaProducer.close(Duration timeout)`.
3128 //
3129 // If `timeout > 0`: initiates a graceful close and awaits the sender task up
3130 // to the remaining time. If the sender task is still alive after the timeout,
3131 // it is force-closed and awaited indefinitely.
3132 //
3133 // If `timeout == 0`: force-closes immediately without draining.
3134 //
3135 // A timeout whose length in milliseconds exceeds `i64::MAX`, such as
3136 // `Duration::MAX`, is clamped to `i64::MAX` ms, the value passed by `close()`.
3137 // The producer then closes gracefully and waits for the sender task to finish.
3138 //
3139 // Note: Rust's `Duration` is unsigned, so the negative-timeout check from
3140 // Java is omitted (impossible to construct a negative `Duration`).
3141 async fn close_with_timeout(&self, timeout: Duration) -> Result<(), Error> {
3142 // Java: `long timeoutMs = timeout.toMillis()`. The millisecond count is
3143 // clamped rather than cast with `as i64`, which truncates it to the low 64
3144 // bits and can produce a negative or zero value that neither branch below
3145 // handles as intended. Java's `toMillis()` throws `ArithmeticException` on
3146 // overflow; clamping is preferred here because `Duration::MAX` is the
3147 // conventional way to express an unbounded timeout in Rust.
3148 // `AsyncKafkaConsumer` applies the same clamp on close. The `Duration` is
3149 // normalized to the same whole-millisecond value, so the sender wait below
3150 // uses the timeout that is logged and checked, as Java joins the I/O thread
3151 // for `timeoutMs`.
3152 let timeout = Duration::from_millis(timeout.as_millis().min(i64::MAX as u128) as u64);
3153 let timeout_ms = timeout.as_millis() as i64;
3154 kafka_info!(
3155 self.log_context,
3156 "Closing the Kafka producer with timeoutMillis = {} ms.",
3157 timeout_ms
3158 );
3159
3160 // Track whether the sender is still alive after the graceful close attempt.
3161 let mut sender_still_alive = false;
3162
3163 if timeout_ms > 0 {
3164 // Try to close gracefully: close accumulator, set running=false, wake sender.
3165 self.initiate_close();
3166
3167 // Await the sender task with the remaining timeout.
3168 sender_still_alive = !self.await_sender_handle(timeout).await;
3169 }
3170
3171 if timeout_ms == 0 || sender_still_alive {
3172 // Force close if timeout is 0 or sender is still alive after timeout
3173 kafka_info!(
3174 self.log_context,
3175 "Proceeding to force close the producer since pending requests could not be \
3176 completed within timeout {} ms.",
3177 timeout_ms
3178 );
3179 self.force_close();
3180
3181 // Await the sender task indefinitely after force close.
3182 self.await_sender_handle_indefinitely().await;
3183 }
3184
3185 // Java `close`'s `Utils.closeQuietly(...)` chain, in order:
3186 // `producerMetrics` → `metrics` → `keySerializer` → `valueSerializer` →
3187 // `partitioner` (`KafkaProducer#close`).
3188 self.producer_metrics.close();
3189 self.metrics.close();
3190 let first_error = self.close_serializers().await;
3191 if let Some(partitioner) = &self.partitioner {
3192 partitioner.close();
3193 }
3194
3195 // `KafkaProducer`: the first failure `closeQuietly`
3196 // recorded is raised once everything is closed.
3197 if let Some(error) = first_error {
3198 return Err(Error::kafka_message_source("Failed to close kafka producer", error));
3199 }
3200 kafka_debug!(self.log_context, "Kafka producer has been closed");
3201 Ok(())
3202 }
3203}
3204
3205impl<K, V> KafkaProducer<K, V> {
3206 /// Makes the record's headers read-only once both serializers have run, so
3207 /// nothing changes them after the producer reads them
3208 /// (`KafkaProducer#setReadOnly`; the headers are always `RecordHeaders`
3209 /// here, so Java's `instanceof` test always holds).
3210 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducer#setReadOnly")]
3211 fn set_read_only(headers: &mut RecordHeaders) {
3212 headers.set_read_only();
3213 }
3214
3215 /// `Utils.closeQuietly(keySerializerPlugin, "producer keySerializer", firstException)`
3216 /// then the same for the value serializer (`KafkaProducer`),
3217 /// through [`close_serializers_quietly`](Self::close_serializers_quietly).
3218 /// Only the first call closes them.
3219 async fn close_serializers(&self) -> Option<Error> {
3220 if self.serializers_closed.swap(true, Ordering::AcqRel) {
3221 return None;
3222 }
3223 Self::close_serializers_quietly(&*self.key_serializer, &*self.value_serializer, &self.log_context).await
3224 }
3225
3226 /// `Utils.closeQuietly(keySerializerPlugin, "producer keySerializer", firstException)`
3227 /// then `Utils.closeQuietly(valueSerializerPlugin, "producer valueSerializer", firstException)`
3228 /// (`KafkaProducer#close`): both are closed, each failure (an `Err` from
3229 /// `close`, or a caught panic) is logged as `closeQuietly` logs one
3230 /// (`Utils`), and the first one is returned. Shared by `close` and a failed
3231 /// constructor (`CloseSerializersOnError`).
3232 async fn close_serializers_quietly(
3233 key_serializer: &dyn DynSerializer<K>,
3234 value_serializer: &dyn DynSerializer<V>,
3235 log_context: &LogContext,
3236 ) -> Option<Error> {
3237 let mut first_error = None;
3238 let key = AssertUnwindSafe(key_serializer.close()).catch_unwind().await;
3239 Self::record_serializer_close_failure(&mut first_error, log_context, "producer keySerializer", key);
3240 let value = AssertUnwindSafe(value_serializer.close()).catch_unwind().await;
3241 Self::record_serializer_close_failure(&mut first_error, log_context, "producer valueSerializer", value);
3242 first_error
3243 }
3244
3245 /// Records a serializer's `close` failure, an `Err` or a caught panic, as
3246 /// `closeQuietly` records a throw: logged, and kept in `first_error` when it
3247 /// is the first (`Utils`).
3248 fn record_serializer_close_failure(
3249 first_error: &mut Option<Error>,
3250 log_context: &LogContext,
3251 name: &str,
3252 outcome: Result<Result<(), Error>, Box<dyn std::any::Any + Send>>,
3253 ) {
3254 let error = match outcome {
3255 Ok(Ok(())) => return,
3256 Ok(Err(error)) => error,
3257 Err(payload) => Error::kafka_message(panic_payload_message(&*payload).to_string()),
3258 };
3259 kafka_error!(log_context, "Failed to close {}: {}", name, error);
3260 first_error.get_or_insert(error);
3261 }
3262}
3263
3264/// Closes the producer's serializers when construction fails, as the `catch
3265/// (Throwable t)` of Java's constructor does (`KafkaProducer#KafkaProducer`):
3266/// it calls `close(Duration.ofMillis(0), true)`, whose `closeQuietly` of the
3267/// key and value serializers (`KafkaProducer#close`) closes them and only logs a
3268/// failure, since `swallowException` is `true`.
3269///
3270/// Rust-only (DoD #7): an RAII stand-in for that `catch`, the producer's twin
3271/// of the consumer's `CloseDeserializersOnError`. `new_inner` returns through
3272/// many `?`s; holding the serializers in this guard covers every one of them,
3273/// and any step added later. It is disarmed when the serializers are handed to
3274/// the producer. `new` is synchronous and a serializer's `close` is async, so
3275/// the close runs through [`Utils::close_quietly_from_sync`].
3276///
3277/// Java wraps the serializers after resolving the partitioner
3278/// (`KafkaProducer#KafkaProducer`), so a partitioner that cannot be
3279/// instantiated leaves a caller-supplied serializer unclosed there. Here the
3280/// producer owns the serializers from the start of `new_inner`, so the caller
3281/// can no longer close them: every failure closes them.
3282struct CloseSerializersOnError<K: 'static, V: 'static> {
3283 serializers: Option<KeyAndValueSerializers<K, V>>,
3284 log_context: LogContext,
3285}
3286
3287/// The key and the value serializer held by [`CloseSerializersOnError`].
3288type KeyAndValueSerializers<K, V> = (Box<dyn DynSerializer<K>>, Box<dyn DynSerializer<V>>);
3289
3290impl<K: 'static, V: 'static> CloseSerializersOnError<K, V> {
3291 fn new(
3292 key_serializer: Box<dyn DynSerializer<K>>,
3293 value_serializer: Box<dyn DynSerializer<V>>,
3294 log_context: LogContext,
3295 ) -> Self {
3296 Self { serializers: Some((key_serializer, value_serializer)), log_context }
3297 }
3298
3299 /// Construction succeeded: hands the serializers on, so nothing closes
3300 /// them here.
3301 fn disarm(mut self) -> KeyAndValueSerializers<K, V> {
3302 self.serializers.take().expect("the guard is disarmed once")
3303 }
3304}
3305
3306impl<K: 'static, V: 'static> Drop for CloseSerializersOnError<K, V> {
3307 fn drop(&mut self) {
3308 let Some((key_serializer, value_serializer)) = self.serializers.take() else {
3309 return;
3310 };
3311 let log_context = self.log_context.clone();
3312 Utils::close_quietly_from_sync(
3313 Box::pin(async move {
3314 // `swallowException` is `true`: the failures are only logged.
3315 let _ = KafkaProducer::<K, V>::close_serializers_quietly(
3316 &*key_serializer,
3317 &*value_serializer,
3318 &log_context,
3319 )
3320 .await;
3321 }),
3322 "producer serializers",
3323 );
3324 }
3325}
3326
3327/// Starts a graceful close of a producer that was not closed, so its sender task
3328/// still sends the records it holds and then stops.
3329///
3330/// Java has no counterpart: its I/O thread is a daemon (built in the `KafkaProducer` constructor),
3331/// which keeps sending the records of a producer nothing references any more. The
3332/// sender task here does the same for as long as its runtime runs.
3333impl<K, V> Drop for KafkaProducer<K, V> {
3334 fn drop(&mut self) {
3335 if self.is_running() {
3336 kafka_warn!(
3337 self.log_context,
3338 "KafkaProducer was not closed before being dropped. Call close() to avoid resource leaks."
3339 );
3340 self.initiate_close();
3341 }
3342 }
3343}
3344
3345#[cfg(test)]
3346mod tests {
3347 use crate::common::utils::MockTime;
3348 use std::collections::HashMap;
3349
3350 use super::*;
3351 use crate::MockClient;
3352 use crate::common::Node;
3353 use crate::common::compress::Compression;
3354 use crate::common::internals::ClusterResourceListeners;
3355 use crate::common::protocol::Errors;
3356 use crate::common::requests::ConcreteResponse;
3357 use crate::common::serialization::Serializer;
3358 use crate::common::serialization::StringSerializer;
3359 use crate::common::utils::Utils;
3360 use crate::producer::MockPartitioner;
3361 use crate::producer::ProducerConfig;
3362 use crate::producer::ProducerRecordOptionsBuilder;
3363 use crate::producer::RoundRobinPartitioner;
3364 use crate::producer::internals::BufferPool;
3365 use crate::producer::internals::{PartitionerConfig, RecordAccumulator};
3366 use std::borrow::Cow;
3367
3368 const TOPIC: &str = "test-topic";
3369
3370 /// Java's `"some.id"`, the `transactional.id` most transactional
3371 /// `KafkaProducerTest` methods configure.
3372 const TRANSACTIONAL_ID: &str = "some.id";
3373
3374 /// Java's `initProducerIdResponse(1L, (short) 5, ..)` pair
3375 /// (`KafkaProducerTest#initProducerIdResponse`).
3376 const PRODUCER_ID: i64 = 1;
3377 const EPOCH: i16 = 5;
3378
3379 fn default_time() -> Arc<dyn Time> {
3380 Arc::new(SystemTime)
3381 }
3382
3383 fn create_metadata_with_topic(topic: &str, num_partitions: i32) -> Arc<ProducerMetadata> {
3384 use crate::MetadataResponseData;
3385 use crate::common::protocol::ApiKeys;
3386 use crate::common::protocol::Errors;
3387 use crate::common::requests::MetadataResponse;
3388 use crate::metadata_response_data::{MetadataResponseBroker, MetadataResponsePartition, MetadataResponseTopic};
3389
3390 let metadata = Arc::new(ProducerMetadata::new(
3391 100,
3392 1000,
3393 300_000,
3394 300_000,
3395 ClusterResourceListeners::new(),
3396 ));
3397
3398 let mut data = MetadataResponseData::new();
3399 data.set_controller_id(0);
3400 data.set_cluster_id(Some("test-cluster".to_string()));
3401
3402 let mut broker = MetadataResponseBroker::new();
3403 broker.set_node_id(0);
3404 broker.set_host("localhost".to_string());
3405 broker.set_port(9092);
3406 data.set_brokers(vec![broker]);
3407
3408 let mut topic_resp = MetadataResponseTopic::new();
3409 topic_resp.set_name(Some(topic.to_string()));
3410 topic_resp.set_error_code(Errors::None.code());
3411 topic_resp.set_is_internal(false);
3412
3413 let mut partitions = Vec::new();
3414 for i in 0..num_partitions {
3415 let mut p = MetadataResponsePartition::new();
3416 p.set_partition_index(i);
3417 p.set_leader_id(0);
3418 p.set_leader_epoch(0);
3419 p.set_replica_nodes(vec![0]);
3420 p.set_isr_nodes(vec![0]);
3421 p.set_error_code(Errors::None.code());
3422 partitions.push(p);
3423 }
3424 topic_resp.set_partitions(partitions);
3425 data.set_topics(vec![topic_resp]);
3426
3427 let response = MetadataResponse::with_version(data, ApiKeys::METADATA.latest_version());
3428
3429 metadata.add(topic, 0);
3430 metadata.update_with_current_request_version(&response, false, 0);
3431
3432 metadata
3433 }
3434
3435 fn create_accumulator() -> Arc<RecordAccumulator> {
3436 Arc::new(RecordAccumulator::new_for_test(
3437 16384,
3438 Compression::none().build(),
3439 5,
3440 100,
3441 1000,
3442 120_000,
3443 PartitionerConfig::new(true, 0, false, "").unwrap(),
3444 default_time(),
3445 Arc::new(BufferPool::new_for_test(32 * 1024 * 1024, 16384)),
3446 None,
3447 ))
3448 }
3449
3450 fn create_producer(
3451 metadata: Arc<ProducerMetadata>,
3452 accumulator: Arc<RecordAccumulator>,
3453 ) -> KafkaProducer<String, String> {
3454 create_producer_with_config(ProducerConfig::defaults(), metadata, accumulator)
3455 }
3456
3457 fn create_producer_with_config(
3458 config: ProducerConfig,
3459 metadata: Arc<ProducerMetadata>,
3460 accumulator: Arc<RecordAccumulator>,
3461 ) -> KafkaProducer<String, String> {
3462 let running = Arc::new(AtomicBool::new(true));
3463 let force_close = Arc::new(AtomicBool::new(false));
3464 let wakeup = Arc::new(Notify::new());
3465
3466 KafkaProducer::with_options(
3467 KafkaProducerOptionsBuilder::new()
3468 .set_config(&config)
3469 .set_key_serializer(Box::new(StringSerializer))
3470 .set_value_serializer(Box::new(StringSerializer))
3471 .set_metadata(metadata)
3472 .set_accumulator(accumulator)
3473 .set_running(running)
3474 .set_force_close(force_close)
3475 .set_wakeup(wakeup)
3476 .set_time(default_time())
3477 .set_pending_requests(Arc::new(Mutex::new(PendingRequests::new())))
3478 .build()
3479 .expect("KafkaProducerOptionsBuilder::build: every mandatory parameter is set above"),
3480 )
3481 }
3482
3483 /// A serializer that always fails with a caller-chosen error, so `do_send`'s
3484 /// dispatch on the serializer's error *class* is observable.
3485 struct FailingSerializer(Error);
3486
3487 impl Serializer<String> for FailingSerializer {
3488 async fn serialize<'a>(
3489 &'a self,
3490 _topic: &'a str,
3491 _data: Option<&'a String>,
3492 ) -> Result<Option<Cow<'a, [u8]>>, Error> {
3493 Err(self.0.clone())
3494 }
3495 }
3496
3497 fn create_producer_with_key_serializer(
3498 metadata: Arc<ProducerMetadata>,
3499 accumulator: Arc<RecordAccumulator>,
3500 key_serializer: Box<dyn DynSerializer<String>>,
3501 ) -> KafkaProducer<String, String> {
3502 KafkaProducer::with_options(
3503 KafkaProducerOptionsBuilder::new()
3504 .set_config(&ProducerConfig::defaults())
3505 .set_key_serializer(key_serializer)
3506 .set_value_serializer(Box::new(StringSerializer))
3507 .set_metadata(metadata)
3508 .set_accumulator(accumulator)
3509 .set_time(default_time())
3510 .set_pending_requests(Arc::new(Mutex::new(PendingRequests::new())))
3511 .build()
3512 .expect("KafkaProducerOptionsBuilder::build: every mandatory parameter is set above"),
3513 )
3514 }
3515
3516 /// Java's only catch around the serializer call is
3517 /// `catch (ClassCastException cce)` (`KafkaProducer#doSend`), which
3518 /// cannot happen in Rust — the serializer is statically typed on `K`. Everything
3519 /// else the serializer throws keeps its class and is dispatched by `doSend`'s
3520 /// outer catches: an `ApiException` at `KafkaProducer#doSend` fires the callback and returns a
3521 /// failed future.
3522 ///
3523 /// Rewriting every serializer error as `SerializationException` flipped
3524 /// `is_retriable_error()` from `true` to `false` for exactly this case (a
3525 /// schema-registry `TimeoutException`), skipped the callback entirely because
3526 /// `Serialization` is not an `ApiException`, and flattened the cause into a
3527 /// display string.
3528 #[tokio::test]
3529 async fn a_retriable_serializer_error_keeps_its_class_and_fires_the_callback() {
3530 let metadata = create_metadata_with_topic(TOPIC, 1);
3531 let accumulator = create_accumulator();
3532 let producer = create_producer_with_key_serializer(
3533 metadata,
3534 accumulator,
3535 Box::new(FailingSerializer(Error::timeout("schema registry lookup timed out"))),
3536 );
3537
3538 let callback_error: Arc<Mutex<Option<Error>>> = Arc::new(Mutex::new(None));
3539 let sink = Arc::clone(&callback_error);
3540 let callback: Box<dyn Callback> = Box::new(move |_metadata: Option<&RecordMetadata>, error: Option<&Error>| {
3541 *sink.lock().unwrap() = error.cloned();
3542 });
3543
3544 let record = ProducerRecord::with_key(TOPIC.to_string(), Some("k".to_string()), Some("v".to_string()));
3545 // An `ApiException` yields `Ok(failed_future)`, not `Err` — Java's
3546 // `return new FutureFailure(e)`.
3547 let future = producer
3548 .send_with_callback(record, Some(callback))
3549 .await
3550 .expect("an API-error serializer failure must come back as a failed future, not Err");
3551
3552 let error = future.get().await.expect_err("the future must be failed");
3553 assert!(error.is_timeout_error(), "the serializer's class must survive: {error:?}");
3554 assert!(error.is_retriable_error(), "a schema-registry timeout is retriable: {error:?}");
3555 assert_eq!(error.message(), "schema registry lookup timed out");
3556
3557 let invoked = callback_error.lock().unwrap().take().expect("the callback must be invoked");
3558 assert!(invoked.is_timeout_error(), "got {invoked:?}");
3559 }
3560
3561 /// The callback partition of a pre-append API error: Java's `catch (ApiException e)`
3562 /// (`KafkaProducer`) builds its null metadata from
3563 /// `AppendCallbacks.topicPartition()` (`KafkaProducer`), which, before a partition is
3564 /// computed, names the record's own partition when the record names one, and
3565 /// `UNKNOWN_PARTITION` only otherwise. Covers a serializer failure on both
3566 /// serialization paths (built-in and custom partitioner) and the metadata wait on both
3567 /// `send` paths (typed and borrowed bytes), each with and without a record partition.
3568 #[tokio::test]
3569 async fn a_pre_append_api_error_reports_the_record_partition() {
3570 use crate::common::serialization::ByteArraySerializer;
3571
3572 fn recording_callback(slot: &Arc<Mutex<Option<(String, i32)>>>) -> Box<dyn Callback> {
3573 let slot = Arc::clone(slot);
3574 Box::new(move |metadata: Option<&RecordMetadata>, error: Option<&Error>| {
3575 assert!(error.is_some(), "the callback is handed the error");
3576 let metadata = metadata.expect("Java passes a non-null RecordMetadata here");
3577 *slot.lock().unwrap() = Some((metadata.topic().to_string(), metadata.partition()));
3578 })
3579 }
3580
3581 // A serializer API error, on the built-in partitioner's zero-copy path.
3582 for (partition, expected) in [(Some(2), 2), (None, RecordMetadata::UNKNOWN_PARTITION)] {
3583 let producer = create_producer_with_key_serializer(
3584 create_metadata_with_topic(TOPIC, 3),
3585 create_accumulator(),
3586 Box::new(FailingSerializer(Error::timeout("schema registry lookup timed out"))),
3587 );
3588 let seen = Arc::new(Mutex::new(None));
3589 let record =
3590 ProducerRecord::with_partition_key(TOPIC.to_string(), partition, Some("k".to_string()), None).unwrap();
3591 let future = producer
3592 .send_with_callback(record, Some(recording_callback(&seen)))
3593 .await
3594 .expect("an API error comes back as a failed future");
3595 assert!(future.get().await.unwrap_err().is_timeout_error());
3596 assert_eq!(seen.lock().unwrap().take(), Some((TOPIC.to_string(), expected)));
3597 }
3598
3599 // The same on the custom partitioner's borrowing path.
3600 for (partition, expected) in [(Some(2), 2), (None, RecordMetadata::UNKNOWN_PARTITION)] {
3601 let producer: KafkaProducer<String, String> = KafkaProducer::with_options(
3602 KafkaProducerOptionsBuilder::new()
3603 .set_config(&ProducerConfig::defaults())
3604 .set_key_serializer(Box::new(FailingSerializer(Error::timeout("schema registry lookup timed out"))))
3605 .set_value_serializer(Box::new(StringSerializer))
3606 .set_metadata(create_metadata_with_topic(TOPIC, 3))
3607 .set_accumulator(create_accumulator())
3608 .set_time(default_time())
3609 .set_pending_requests(Arc::new(Mutex::new(PendingRequests::new())))
3610 .set_partitioner(Some(Box::new(RoundRobinPartitioner::new())))
3611 .build()
3612 .expect("KafkaProducerOptionsBuilder::build: every mandatory parameter is set above"),
3613 );
3614 let seen = Arc::new(Mutex::new(None));
3615 let record =
3616 ProducerRecord::with_partition_key(TOPIC.to_string(), partition, Some("k".to_string()), None).unwrap();
3617 let future = producer
3618 .send_with_callback(record, Some(recording_callback(&seen)))
3619 .await
3620 .expect("an API error comes back as a failed future");
3621 assert!(future.get().await.unwrap_err().is_timeout_error());
3622 assert_eq!(seen.lock().unwrap().take(), Some((TOPIC.to_string(), expected)));
3623 }
3624
3625 // A metadata-wait timeout: the record names a partition the topic does not have,
3626 // and `max.block.ms = 0` fails the wait at once.
3627 let config = || ProducerConfig { max_block_ms: 0, ..ProducerConfig::defaults() };
3628 let producer =
3629 create_producer_with_config(config(), create_metadata_with_topic(TOPIC, 1), create_accumulator());
3630 let seen = Arc::new(Mutex::new(None));
3631 let record =
3632 ProducerRecord::with_partition_key(TOPIC.to_string(), Some(5), None, Some("v".to_string())).unwrap();
3633 let future = producer
3634 .send_with_callback(record, Some(recording_callback(&seen)))
3635 .await
3636 .expect("a metadata timeout comes back as a failed future");
3637 assert!(future.get().await.unwrap_err().is_timeout_error());
3638 assert_eq!(seen.lock().unwrap().take(), Some((TOPIC.to_string(), 5)));
3639
3640 // The same on the borrowed byte-slice `send`.
3641 let producer: KafkaProducer<Vec<u8>, Vec<u8>> = KafkaProducer::with_options(
3642 KafkaProducerOptionsBuilder::new()
3643 .set_config(&config())
3644 .set_key_serializer(Box::new(ByteArraySerializer))
3645 .set_value_serializer(Box::new(ByteArraySerializer))
3646 .set_metadata(create_metadata_with_topic(TOPIC, 1))
3647 .set_accumulator(create_accumulator())
3648 .set_time(default_time())
3649 .set_pending_requests(Arc::new(Mutex::new(PendingRequests::new())))
3650 .build()
3651 .expect("KafkaProducerOptionsBuilder::build: every mandatory parameter is set above"),
3652 );
3653 let seen = Arc::new(Mutex::new(None));
3654 let record =
3655 ProducerRecord::with_partition_key(TOPIC.to_string(), Some(5), None, Some(b"v".as_slice())).unwrap();
3656 let future = producer
3657 .send(record, Some(recording_callback(&seen)))
3658 .await
3659 .expect("a metadata timeout comes back as a failed future");
3660 assert!(future.get().await.unwrap_err().is_timeout_error());
3661 assert_eq!(seen.lock().unwrap().take(), Some((TOPIC.to_string(), 5)));
3662 }
3663
3664 /// The other half of the same dispatch: a `SerializationException` — which
3665 /// extends `KafkaException` directly and is NOT an `ApiException` — reaches
3666 /// `catch (KafkaException e)` at `KafkaProducer` and is rethrown, so
3667 /// `send()` returns `Err` rather than a failed future.
3668 #[tokio::test]
3669 async fn a_serialization_error_from_the_serializer_is_returned_as_err() {
3670 let metadata = create_metadata_with_topic(TOPIC, 1);
3671 let accumulator = create_accumulator();
3672 let producer = create_producer_with_key_serializer(
3673 metadata,
3674 accumulator,
3675 Box::new(FailingSerializer(Error::serialization("not a valid string"))),
3676 );
3677
3678 let record = ProducerRecord::with_key(TOPIC.to_string(), Some("k".to_string()), Some("v".to_string()));
3679 let error = producer
3680 .send(record)
3681 .await
3682 .expect_err("a serialization error is not an API error, so send() returns Err");
3683 assert!(matches!(error, Error::Serialization(_)), "got {error:?}");
3684 assert!(error.is_kafka_error(), "a serialization error is a Kafka error: {error:?}");
3685 assert!(!error.is_api_error(), "but it is not an API error: {error:?}");
3686 assert_eq!(error.message(), "not a valid string");
3687 }
3688
3689 /// `waitOnMetadata`'s first throw attaches the topic's metadata error as the
3690 /// `TimeoutException`'s **cause** (`KafkaProducer#waitOnMetadata`,
3691 /// `new TimeoutException(errorMessage, metadata.getError(topic).exception())`).
3692 ///
3693 /// The message used to be built by interpolating the underlying error into it,
3694 /// which left `Error::source()` empty — a caller (or a log line) could no longer
3695 /// walk from the timeout to the broker error that produced it.
3696 #[tokio::test]
3697 async fn wait_on_metadata_carries_the_topic_error_as_the_timeout_cause() {
3698 use crate::MetadataResponseData;
3699 use crate::common::protocol::ApiKeys;
3700 use crate::common::protocol::Errors;
3701 use crate::common::requests::MetadataResponse;
3702 use crate::metadata_response_data::{MetadataResponseBroker, MetadataResponseTopic};
3703
3704 let metadata = create_metadata_with_topic(TOPIC, 1);
3705 let accumulator = create_accumulator();
3706 let producer = create_producer(Arc::clone(&metadata), accumulator);
3707
3708 // A topic the broker reports a retriable error for and no partitions, so the
3709 // wait times out with `get_error(topic)` populated.
3710 const MISSING: &str = "topic-with-an-error";
3711 let mut data = MetadataResponseData::new();
3712 data.set_controller_id(0);
3713 let mut broker = MetadataResponseBroker::new();
3714 broker.set_node_id(0);
3715 broker.set_host("localhost".to_string());
3716 broker.set_port(9092);
3717 data.set_brokers(vec![broker]);
3718 let mut topic_resp = MetadataResponseTopic::new();
3719 topic_resp.set_name(Some(MISSING.to_string()));
3720 topic_resp.set_error_code(Errors::LeaderNotAvailable.code());
3721 topic_resp.set_is_internal(false);
3722 topic_resp.set_partitions(Vec::new());
3723 data.set_topics(vec![topic_resp]);
3724 let response = MetadataResponse::with_version(data, ApiKeys::METADATA.latest_version());
3725 metadata.add(MISSING, 0);
3726 metadata.update_with_current_request_version(&response, false, 0);
3727 assert_eq!(
3728 metadata.get_error(MISSING),
3729 Some(Errors::LeaderNotAvailable),
3730 "the fixture must actually record a topic error"
3731 );
3732
3733 let now_ms = producer.now_ms();
3734 let error = producer
3735 .wait_on_metadata(MISSING, None, now_ms, 50)
3736 .await
3737 .expect_err("the topic has no metadata, so the wait must time out");
3738
3739 assert!(error.is_timeout_error(), "got {error:?}");
3740 assert!(
3741 error.message().contains("not present in metadata"),
3742 "the message keeps Java's text, without the cause interpolated: {}",
3743 error.message()
3744 );
3745 assert_eq!(
3746 error.source().expect("the metadata error must be the timeout's cause").error(),
3747 Errors::LeaderNotAvailable
3748 );
3749 }
3750
3751 /// Translated from `KafkaProducerTest.testMetricConfigRecordingLevel`.
3752 ///
3753 /// The Java test constructs a producer with the default config and asserts
3754 /// `producer.metrics.config().recordLevel() == INFO`, then with
3755 /// `metrics.recording.level=DEBUG` and asserts `DEBUG`.
3756 #[test]
3757 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testMetricConfigRecordingLevel")]
3758 fn test_metric_config_recording_level() {
3759 let default_producer = create_producer_with_config(
3760 ProducerConfig::defaults(),
3761 create_metadata_with_topic(TOPIC, 1),
3762 create_accumulator(),
3763 );
3764 assert_eq!(default_producer.metrics.config().record_level(), RecordingLevel::Info);
3765
3766 let debug_config =
3767 ProducerConfig { metrics_recording_level: "DEBUG".to_string(), ..ProducerConfig::defaults() };
3768 let debug_producer =
3769 create_producer_with_config(debug_config, create_metadata_with_topic(TOPIC, 1), create_accumulator());
3770 assert_eq!(debug_producer.metrics.config().record_level(), RecordingLevel::Debug);
3771 }
3772
3773 /// The producer registers the `producer-metrics` latency sensors and
3774 /// exposes them through `metrics()`; `flush()` records `flush-time-ns-total`.
3775 #[tokio::test]
3776 async fn test_flush_records_producer_metrics() {
3777 let producer = create_producer_with_config(
3778 ProducerConfig::defaults(),
3779 create_metadata_with_topic(TOPIC, 1),
3780 create_accumulator(),
3781 );
3782
3783 // The metrics snapshot exposes the producer-metrics latency sensors.
3784 let flush_name = producer.metrics.metric_name("flush-time-ns-total", "producer-metrics");
3785 let snapshot = producer.metrics();
3786 assert!(
3787 snapshot.contains_key(&flush_name),
3788 "metrics() should expose flush-time-ns-total"
3789 );
3790
3791 // flush() drives a CumulativeSum record; the metric stays present.
3792 producer.flush().await.expect("flush");
3793 assert!(producer.metrics().contains_key(&flush_name));
3794 }
3795
3796 /// Translated from `KafkaProducerTest.testSendToInvalidTopic`.
3797 ///
3798 /// Tests that sending to an invalid topic name returns a failed future with
3799 /// an InvalidTopic error. Matches Java behavior where InvalidTopicException
3800 /// (an ApiException) is caught and returned via a FutureFailure.
3801 #[tokio::test]
3802 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testSendToInvalidTopic")]
3803 async fn test_send_to_invalid_topic() {
3804 use crate::MetadataResponseData;
3805 use crate::common::protocol::ApiKeys;
3806 use crate::common::protocol::Errors;
3807 use crate::common::requests::MetadataResponse;
3808 use crate::metadata_response_data::{MetadataResponseBroker, MetadataResponseTopic};
3809
3810 let metadata = Arc::new(ProducerMetadata::new(
3811 100,
3812 1000,
3813 300_000,
3814 300_000,
3815 ClusterResourceListeners::new(),
3816 ));
3817
3818 // Create a metadata response with an invalid topic
3819 let mut data = MetadataResponseData::new();
3820 data.set_controller_id(0);
3821
3822 let mut broker = MetadataResponseBroker::new();
3823 broker.set_node_id(0);
3824 broker.set_host("localhost".to_string());
3825 broker.set_port(9092);
3826 data.set_brokers(vec![broker]);
3827
3828 let mut topic_resp = MetadataResponseTopic::new();
3829 topic_resp.set_name(Some("".to_string()));
3830 topic_resp.set_error_code(Errors::InvalidTopicError.code());
3831 topic_resp.set_is_internal(false);
3832 data.set_topics(vec![topic_resp]);
3833
3834 let response = MetadataResponse::with_version(data, ApiKeys::METADATA.latest_version());
3835 metadata.add("", 0);
3836 metadata.update_with_current_request_version(&response, false, 0);
3837
3838 let accumulator = create_accumulator();
3839 let producer = create_producer(metadata, accumulator);
3840
3841 let record = ProducerRecord::new("".to_string(), Some("test".to_string()));
3842 let result = producer.send(record).await;
3843 // Java returns a FutureFailure for ApiExceptions like InvalidTopicException
3844 assert!(result.is_ok(), "send() should return Ok with a failed future for InvalidTopic");
3845 let future = result.unwrap();
3846 assert!(future.is_done(), "Failed future should be immediately done");
3847 let err = future.get().await.unwrap_err();
3848 assert!(
3849 matches!(err, Error::InvalidTopic(_)),
3850 "Expected InvalidTopic error, got: {:?}",
3851 err
3852 );
3853 // `waitOnMetadata` throws `new InvalidTopicException(topic)` — the
3854 // `(String message)` constructor, so `getMessage()` is the topic name
3855 // (empty here) and `invalidTopics()` is empty, NOT `{topic}`.
3856 assert_eq!(err.message(), "");
3857 match &err {
3858 Error::InvalidTopic(e) => assert!(
3859 e.invalid_topics().is_empty(),
3860 "the (String) constructor leaves invalidTopics empty"
3861 ),
3862 _ => unreachable!(),
3863 }
3864 }
3865
3866 /// Translated from `KafkaProducerTest.closeShouldBeIdempotent`.
3867 ///
3868 /// Tests that calling close multiple times is safe.
3869 #[tokio::test]
3870 async fn test_close_should_be_idempotent() {
3871 let metadata = create_metadata_with_topic(TOPIC, 1);
3872 let accumulator = create_accumulator();
3873 let producer = create_producer(metadata, accumulator);
3874
3875 producer.close().await.unwrap();
3876 producer.close().await.unwrap();
3877 }
3878
3879 /// Translated from `KafkaProducerTest.closeWithNegativeTimestampShouldThrow`.
3880 ///
3881 /// Tests that close with zero timeout works correctly.
3882 /// In Java this tests negative Duration; in Rust `Duration` is unsigned
3883 /// so we test `Duration::ZERO` instead.
3884 #[tokio::test]
3885 async fn test_close_with_zero_timeout() {
3886 let metadata = create_metadata_with_topic(TOPIC, 1);
3887 let accumulator = create_accumulator();
3888 let producer = create_producer(metadata, accumulator);
3889
3890 let result = producer.close_with_timeout(Duration::ZERO).await;
3891 assert!(result.is_ok());
3892 }
3893
3894 /// Translated from `KafkaProducerTest.testPartitionsForWithNullTopic`.
3895 ///
3896 /// Tests that partitions_for returns the correct number of partitions.
3897 #[tokio::test]
3898 async fn test_partitions_for_returns_partitions() {
3899 let metadata = create_metadata_with_topic(TOPIC, 3);
3900 let accumulator = create_accumulator();
3901 let producer = create_producer(metadata, accumulator);
3902
3903 let partitions = producer.partitions_for(TOPIC).await.unwrap();
3904 assert_eq!(3, partitions.len());
3905 }
3906
3907 /// Tests that sending after close returns an error.
3908 ///
3909 /// Translated from `KafkaProducerTest.testTransactionalMethodThrowsWhenSenderClosed`
3910 /// (non-transactional part).
3911 #[tokio::test]
3912 async fn test_send_after_close_returns_error() {
3913 let metadata = create_metadata_with_topic(TOPIC, 1);
3914 let accumulator = create_accumulator();
3915 let producer = create_producer(metadata, accumulator);
3916
3917 producer.close().await.unwrap();
3918
3919 let record = ProducerRecord::new(TOPIC.to_string(), Some("test".to_string()));
3920 let result = producer.send(record).await;
3921 assert!(result.is_err());
3922 match result.unwrap_err() {
3923 Error::LocalIllegalState(msg) => {
3924 assert!(msg.message().contains("after producer has been closed"));
3925 },
3926 other => panic!("Expected IllegalState error, got: {:?}", other),
3927 }
3928 }
3929
3930 /// Tests that a record too large for max_request_size returns a failed future.
3931 ///
3932 /// Translated from `KafkaProducerTest.testInterceptorPartitionSetOnTooLargeRecord`
3933 /// (the record-too-large validation part).
3934 ///
3935 /// Matches Java behavior: ApiExceptions like RecordTooLargeException are returned
3936 /// via a completed-with-error future, not propagated as Err from send().
3937 #[tokio::test]
3938 async fn test_ensure_valid_record_size_rejects_too_large() {
3939 let config = ProducerConfig { max_request_size: 10, ..ProducerConfig::defaults() };
3940 let metadata = create_metadata_with_topic(TOPIC, 1);
3941 let accumulator = create_accumulator();
3942
3943 let producer = create_producer_with_config(config, metadata, accumulator);
3944
3945 // Create a record that's larger than 10 bytes
3946 let large_value = "a".repeat(100);
3947 let record = ProducerRecord::new(TOPIC.to_string(), Some(large_value));
3948 let result = producer.send(record).await;
3949 // Java returns a FutureFailure, not an exception from send()
3950 assert!(
3951 result.is_ok(),
3952 "send() should return Ok with a failed future for RecordTooLarge"
3953 );
3954 let future = result.unwrap();
3955 assert!(future.is_done(), "Failed future should be immediately done");
3956 let err = future.get().await.unwrap_err();
3957 match err {
3958 Error::RecordTooLarge(msg) => {
3959 assert!(
3960 msg.message().contains(ProducerConfig::MAX_REQUEST_SIZE_CONFIG),
3961 "Error message should mention the config key: {}",
3962 msg
3963 );
3964 },
3965 other => panic!("Expected RecordTooLarge error, got: {:?}", other),
3966 }
3967 }
3968
3969 /// Tests that records larger than total buffer memory return a failed future.
3970 ///
3971 /// Matches Java behavior: ApiExceptions like RecordTooLargeException are returned
3972 /// via a completed-with-error future, not propagated as Err from send().
3973 #[tokio::test]
3974 async fn test_ensure_valid_record_size_rejects_larger_than_buffer_memory() {
3975 let config = ProducerConfig { buffer_memory: 10, ..ProducerConfig::defaults() };
3976 let metadata = create_metadata_with_topic(TOPIC, 1);
3977 let accumulator = create_accumulator();
3978
3979 let producer = create_producer_with_config(config, metadata, accumulator);
3980
3981 let large_value = "a".repeat(100);
3982 let record = ProducerRecord::new(TOPIC.to_string(), Some(large_value));
3983 let result = producer.send(record).await;
3984 assert!(
3985 result.is_ok(),
3986 "send() should return Ok with a failed future for RecordTooLarge"
3987 );
3988 let future = result.unwrap();
3989 assert!(future.is_done(), "Failed future should be immediately done");
3990 let err = future.get().await.unwrap_err();
3991 match err {
3992 Error::RecordTooLarge(msg) => {
3993 assert!(
3994 msg.message().contains(ProducerConfig::BUFFER_MEMORY_CONFIG),
3995 "Error message should mention the config key: {}",
3996 msg
3997 );
3998 },
3999 other => panic!("Expected RecordTooLarge error, got: {:?}", other),
4000 }
4001 }
4002
4003 /// A size above `i32::MAX` is rejected with the `max.request.size` message.
4004 #[tokio::test]
4005 async fn test_ensure_valid_record_size_rejects_size_beyond_i32_max() {
4006 let metadata = create_metadata_with_topic(TOPIC, 1);
4007 let producer = create_producer_with_config(ProducerConfig::defaults(), metadata, create_accumulator());
4008
4009 let size = (1i64 << 32) + 98;
4010 let err = producer.ensure_valid_record_size(size).expect_err("too large");
4011 assert!(matches!(err, Error::RecordTooLarge(_)), "Expected RecordTooLarge, got: {err:?}");
4012 assert_eq!(
4013 err.message(),
4014 "The message is 4294967394 bytes when serialized which is larger than 1048576, which is the value of \
4015 the max.request.size configuration."
4016 );
4017 }
4018
4019 /// With `max.request.size` at `i32::MAX`, a size of exactly `i32::MAX` is valid and
4020 /// `i32::MAX + 1` is not: the size does not wrap to a negative `i32` and pass.
4021 #[tokio::test]
4022 async fn test_ensure_valid_record_size_at_i32_max() {
4023 let config = ProducerConfig {
4024 max_request_size: i32::MAX,
4025 buffer_memory: i64::MAX,
4026 ..ProducerConfig::defaults()
4027 };
4028 let producer = create_producer_with_config(config, create_metadata_with_topic(TOPIC, 1), create_accumulator());
4029
4030 producer
4031 .ensure_valid_record_size(i32::MAX as i64)
4032 .expect("exactly max.request.size is valid");
4033 let err = producer
4034 .ensure_valid_record_size(i32::MAX as i64 + 1)
4035 .expect_err("one byte too large");
4036 assert!(matches!(err, Error::RecordTooLarge(_)), "Expected RecordTooLarge, got: {err:?}");
4037 assert_eq!(
4038 err.message(),
4039 "The message is 2147483648 bytes when serialized which is larger than 2147483647, which is the value of \
4040 the max.request.size configuration."
4041 );
4042 }
4043
4044 /// A size the estimate saturated at `i64::MAX` is rejected and reported as is.
4045 #[tokio::test]
4046 async fn test_ensure_valid_record_size_rejects_saturated_size() {
4047 let producer = create_producer_with_config(
4048 ProducerConfig::defaults(),
4049 create_metadata_with_topic(TOPIC, 1),
4050 create_accumulator(),
4051 );
4052
4053 let err = producer.ensure_valid_record_size(i64::MAX).expect_err("too large");
4054 assert!(matches!(err, Error::RecordTooLarge(_)), "Expected RecordTooLarge, got: {err:?}");
4055 assert_eq!(
4056 err.message(),
4057 "The message is 9223372036854775807 bytes when serialized which is larger than 1048576, which is the \
4058 value of the max.request.size configuration."
4059 );
4060 }
4061
4062 /// `doSend`'s `catch (ApiException e)` block invokes the user `Callback` exactly
4063 /// once with a null-metadata `RecordMetadata(tp, -1, -1, NO_TIMESTAMP, -1, -1)`
4064 /// and the error, then returns a failed future
4065 /// (`KafkaProducer`).
4066 ///
4067 /// The arm covering a failure of `accumulator.append` itself used to drop the
4068 /// callback: it is *moved* into `append`, and `append` returned it only on the
4069 /// success path. An application that registers callbacks and never awaits the
4070 /// future was therefore never told the record had been dropped.
4071 ///
4072 /// `buffer.memory` sized for exactly one batch plus `max.block.ms = 0` makes the
4073 /// second `append` fail deterministically inside `BufferPool::allocate` with
4074 /// `BufferExhaustedException`, which is an `ApiException`
4075 /// (`BufferExhaustedException extends TimeoutException`) — so this is the
4076 /// `catch (ApiException e)` arm.
4077 #[tokio::test]
4078 async fn test_api_error_from_append_fires_the_callback_exactly_once() {
4079 let producer = create_producer_with_room_for_one_batch(0);
4080
4081 // The first record consumes the pool's only batch.
4082 producer
4083 .send_with_callback(
4084 ProducerRecord::with_partition_key(TOPIC.to_string(), Some(0), None, Some("first".to_string()))
4085 .unwrap(),
4086 None,
4087 )
4088 .await
4089 .expect("the first send has memory");
4090
4091 // The second record targets partition 1, so its append has to allocate; it
4092 // finds the pool empty and `max.block.ms = 0`, and fails with
4093 // `BufferExhaustedException`.
4094 let invocations: Arc<Mutex<Vec<(i64, i32, String)>>> = Arc::new(Mutex::new(Vec::new()));
4095 let recorder = Arc::clone(&invocations);
4096 let callback: Box<dyn Callback> = Box::new(move |metadata: Option<&RecordMetadata>, error: Option<&Error>| {
4097 let metadata = metadata.expect("Java passes a non-null RecordMetadata here");
4098 let error = error.expect("Java passes the exception here");
4099 recorder
4100 .lock()
4101 .unwrap()
4102 .push((metadata.offset(), metadata.partition(), error.message().to_string()));
4103 });
4104
4105 let future = producer
4106 .send_with_callback(
4107 ProducerRecord::with_partition_key(TOPIC.to_string(), Some(1), None, Some("second".to_string()))
4108 .unwrap(),
4109 Some(callback),
4110 )
4111 .await
4112 .expect("an ApiException becomes a failed future, not an Err");
4113
4114 // The callback fired exactly once. `on_completion` consumes its `Box<dyn Callback>`, so
4115 // "at most once" is a type-level guarantee; this pins "at least once".
4116 // The guard is cloned out and dropped before the `await` below (CLAUDE.md §11.6).
4117 let invocations = invocations.lock().unwrap().clone();
4118 assert_eq!(invocations.len(), 1, "the callback must fire exactly once, got {invocations:?}");
4119 let (offset, partition, message) = &invocations[0];
4120 // Java's `nullMetadata`: `new RecordMetadata(tp, -1, -1, NO_TIMESTAMP, -1, -1)`.
4121 assert_eq!(*offset, -1, "the null metadata carries offset -1");
4122 assert_eq!(*partition, 1, "the null metadata names the record's topic-partition");
4123 assert!(
4124 message.contains("Failed to allocate"),
4125 "the callback receives the BufferExhausted error: {message}"
4126 );
4127
4128 // And the future is failed with the same error, as Java's `FutureFailure` is.
4129 let error = future.get().await.expect_err("the future must be failed");
4130 assert!(
4131 matches!(error, Error::ProducerBufferExhausted(_)),
4132 "expected ProducerBufferExhausted, got {error:?}"
4133 );
4134 assert!(
4135 error.is_api_error(),
4136 "BufferExhaustedException extends TimeoutException, an ApiException"
4137 );
4138 }
4139
4140 /// The sibling of the test above on the *rethrowing* path: Java's
4141 /// `catch (KafkaException e)` (`KafkaProducer`) does NOT invoke the
4142 /// callback — it rethrows out of `send()`. `RecordAccumulator.tryAppend`'s
4143 /// `KafkaException("Producer closed while send in progress")` is a bare
4144 /// `KafkaException`, so it lands there.
4145 #[tokio::test]
4146 async fn test_non_api_error_from_append_does_not_fire_the_callback() {
4147 let metadata = create_metadata_with_topic(TOPIC, 1);
4148 let accumulator = create_accumulator();
4149 accumulator.close();
4150 let producer = create_producer_with_config(ProducerConfig::defaults(), metadata, accumulator);
4151
4152 let invoked = Arc::new(AtomicBool::new(false));
4153 let flag = Arc::clone(&invoked);
4154 let callback: Box<dyn Callback> =
4155 Box::new(move |_: Option<&RecordMetadata>, _: Option<&Error>| flag.store(true, Ordering::SeqCst));
4156
4157 let error = producer
4158 .send_with_callback(
4159 ProducerRecord::with_partition_key(TOPIC.to_string(), Some(0), None, Some("v".to_string())).unwrap(),
4160 Some(callback),
4161 )
4162 .await
4163 .expect_err("a bare KafkaException is rethrown out of send()");
4164 assert_eq!(error.message(), "Producer closed while send in progress");
4165 assert!(!error.is_api_error(), "a bare KafkaException is not an ApiException");
4166 assert!(
4167 !invoked.load(Ordering::SeqCst),
4168 "Java's catch (KafkaException e) block does not invoke the callback"
4169 );
4170 }
4171
4172 /// The fixture of `test_api_error_from_append_fires_the_callback_exactly_once`:
4173 /// `buffer.memory` sized for exactly one batch, and two partitions, so the
4174 /// second record needs a *new* batch rather than appending to the first
4175 /// record's still-roomy one — its append has to allocate, and finds the pool
4176 /// empty.
4177 fn create_producer_with_room_for_one_batch(max_block_ms: i64) -> KafkaProducer<String, String> {
4178 const BATCH_SIZE: usize = 16384;
4179 let config = ProducerConfig {
4180 batch_size: BATCH_SIZE as i32,
4181 buffer_memory: BATCH_SIZE as i64,
4182 max_block_ms,
4183 linger_ms: 0,
4184 ..ProducerConfig::defaults()
4185 };
4186 let metadata = create_metadata_with_topic(TOPIC, 2);
4187 let accumulator = Arc::new(RecordAccumulator::new(
4188 BATCH_SIZE as i32,
4189 Compression::none().build(),
4190 0,
4191 100,
4192 1000,
4193 120_000,
4194 PartitionerConfig::new(true, 0, false, "").unwrap(),
4195 Arc::new(Metrics::new()),
4196 KafkaProducer::<String, String>::PRODUCER_METRIC_GROUP_NAME,
4197 Arc::new(crate::common::utils::SystemTime),
4198 // Room for exactly one batch.
4199 Arc::new(BufferPool::new_for_test(BATCH_SIZE as i64, BATCH_SIZE)),
4200 None,
4201 ));
4202 create_producer_with_config(config, metadata, accumulator)
4203 }
4204
4205 fn string_record(partition: Option<i32>, value: &str) -> ProducerRecord<String, String> {
4206 ProducerRecord::with_partition_key(TOPIC.to_string(), partition, None, Some(value.to_string())).unwrap()
4207 }
4208
4209 /// A callback that records every invocation, plus the recorder to read back.
4210 #[expect(clippy::type_complexity)]
4211 fn recording_callback() -> (Box<dyn Callback>, Arc<Mutex<Vec<(i32, String)>>>) {
4212 let invocations: Arc<Mutex<Vec<(i32, String)>>> = Arc::new(Mutex::new(Vec::new()));
4213 let recorder = Arc::clone(&invocations);
4214 let callback: Box<dyn Callback> = Box::new(crate::producer::callback::new(move |metadata, error| {
4215 let metadata = metadata.expect("Java passes a non-null RecordMetadata here");
4216 let error = error.expect("Java passes the exception here");
4217 recorder
4218 .lock()
4219 .unwrap()
4220 .push((metadata.partition(), error.message().to_string()));
4221 }));
4222 (callback, invocations)
4223 }
4224
4225 /// `try_send` on a producer that has metadata and memory is `send`: the record
4226 /// is appended and its future returned, and the callback waits for the
4227 /// broker's answer rather than firing now.
4228 #[tokio::test]
4229 async fn try_send_accepts_a_record_the_producer_can_take_at_once() {
4230 let producer = create_producer_with_room_for_one_batch(0);
4231 let (callback, invocations) = recording_callback();
4232
4233 let outcome = producer
4234 .try_send(string_record(Some(0), "first"), Some(callback))
4235 .await
4236 .expect("a ready producer accepts the record");
4237
4238 let future = match outcome {
4239 TrySendOutcome::Accepted(future) => future,
4240 other => panic!("expected Accepted, got {other:?}"),
4241 };
4242 assert!(!future.is_done(), "the record is in a batch awaiting the sender, not resolved");
4243 assert!(invocations.lock().unwrap().is_empty(), "nothing has completed, so no callback");
4244 }
4245
4246 /// The buffer-memory would-block point. `send` with `max.block.ms = 0` fails the
4247 /// same record with `BufferExhaustedException` — callback fired, failed future
4248 /// (`test_api_error_from_append_fires_the_callback_exactly_once`). `try_send`
4249 /// instead does nothing to it: no callback, no future, the pool's memory
4250 /// untouched, and the callback handed back intact for the retry.
4251 #[tokio::test]
4252 async fn try_send_reports_would_block_on_buffer_exhaustion_without_firing_the_callback() {
4253 let producer = create_producer_with_room_for_one_batch(0);
4254
4255 // The first record consumes the pool's only batch.
4256 producer
4257 .try_send(string_record(Some(0), "first"), None)
4258 .await
4259 .expect("the first send has memory");
4260 let memory_before = producer.accumulator.buffer_pool_available_memory();
4261
4262 // The second record targets partition 1, so its append has to allocate.
4263 let (callback, invocations) = recording_callback();
4264 let outcome = producer
4265 .try_send(string_record(Some(1), "second"), Some(callback))
4266 .await
4267 .expect("would-block is an outcome, not an error");
4268
4269 let returned = match outcome {
4270 TrySendOutcome::WouldBlock(callback) => callback,
4271 other => panic!("expected WouldBlock, got {other:?}"),
4272 };
4273 assert!(
4274 invocations.lock().unwrap().is_empty(),
4275 "a would-block must not invoke the callback"
4276 );
4277 assert_eq!(
4278 producer.accumulator.buffer_pool_available_memory(),
4279 memory_before,
4280 "a would-block must leave the pool as it found it"
4281 );
4282
4283 // The very callback that was passed in comes back, still live: firing it
4284 // ourselves proves the producer neither invoked nor dropped it.
4285 let returned = returned.expect("the caller's callback is handed back");
4286 let tp = TopicPartition::new(TOPIC.to_string(), 1);
4287 let null_metadata = RecordMetadata::new(tp, -1, -1, RecordBatch::NO_TIMESTAMP, -1, -1);
4288 returned.on_completion(Some(&null_metadata), Some(&Error::timeout("retry")));
4289 assert_eq!(invocations.lock().unwrap().as_slice(), &[(1, "retry".to_string())]);
4290 }
4291
4292 /// The same fixture through the blocking `send`: `SendWait::MaxBlock` dispatches
4293 /// `BufferExhaustedException` as the `ApiException` it is — callback fired once
4294 /// with the null metadata, failed future — so `send` keeps Java's contract
4295 /// while sharing its body with `try_send`.
4296 #[tokio::test]
4297 async fn send_still_fails_the_record_where_try_send_would_block() {
4298 let producer = create_producer_with_room_for_one_batch(0);
4299 producer
4300 .send_with_callback(string_record(Some(0), "first"), None)
4301 .await
4302 .expect("the first send has memory");
4303
4304 let (callback, invocations) = recording_callback();
4305 let future = producer
4306 .send_with_callback(string_record(Some(1), "second"), Some(callback))
4307 .await
4308 .expect("an ApiException becomes a failed future, not an Err");
4309
4310 let invocations = invocations.lock().unwrap().clone();
4311 assert_eq!(invocations.len(), 1, "the callback must fire exactly once, got {invocations:?}");
4312 assert_eq!(invocations[0].0, 1, "the null metadata names the record's partition");
4313 assert!(invocations[0].1.contains("Failed to allocate"), "got: {}", invocations[0].1);
4314 let error = future.get().await.expect_err("the future must be failed");
4315 assert!(matches!(error, Error::ProducerBufferExhausted(_)), "got {error:?}");
4316 }
4317
4318 /// The metadata would-block point. An unknown topic makes `send` wait for a
4319 /// metadata response; `try_send` reports `WouldBlock` with the callback
4320 /// un-invoked — and has still asked for the metadata, so the attempt is not
4321 /// wasted: the topic is now tracked and an update is requested.
4322 #[tokio::test]
4323 async fn try_send_reports_would_block_for_an_unknown_topic_and_requests_its_metadata() {
4324 const UNKNOWN_TOPIC: &str = "not-in-metadata-yet";
4325 let metadata = create_metadata_with_topic(TOPIC, 1);
4326 let producer =
4327 create_producer_with_config(ProducerConfig::defaults(), Arc::clone(&metadata), create_accumulator());
4328 assert!(!metadata.contains_topic(UNKNOWN_TOPIC));
4329
4330 let (callback, invocations) = recording_callback();
4331 let record =
4332 ProducerRecord::with_partition_key(UNKNOWN_TOPIC.to_string(), None, None, Some("v".to_string())).unwrap();
4333 let outcome = producer
4334 .try_send(record, Some(callback))
4335 .await
4336 .expect("would-block is an outcome");
4337
4338 assert!(matches!(outcome, TrySendOutcome::WouldBlock(Some(_))), "got {outcome:?}");
4339 assert!(
4340 invocations.lock().unwrap().is_empty(),
4341 "a would-block must not invoke the callback"
4342 );
4343 assert!(
4344 metadata.contains_topic(UNKNOWN_TOPIC),
4345 "the metadata request is the side effect worth keeping"
4346 );
4347 assert!(
4348 metadata.metadata_arc().update_requested(),
4349 "an update was requested for the topic"
4350 );
4351 }
4352
4353 /// Only a *wait* is a would-block. A record the producer rejects outright — too
4354 /// large for `max.request.size` — is resolved exactly as `send` resolves it:
4355 /// `Accepted`, with the callback already fired and the future already failed.
4356 #[tokio::test]
4357 async fn try_send_resolves_a_rejected_record_as_accepted_with_a_failed_future() {
4358 let config = ProducerConfig { max_request_size: 64, ..ProducerConfig::defaults() };
4359 let producer = create_producer_with_config(config, create_metadata_with_topic(TOPIC, 1), create_accumulator());
4360
4361 let (callback, invocations) = recording_callback();
4362 let outcome = producer
4363 .try_send(string_record(Some(0), &"x".repeat(1024)), Some(callback))
4364 .await
4365 .expect("an ApiException becomes a failed future, not an Err");
4366
4367 let future = match outcome {
4368 TrySendOutcome::Accepted(future) => future,
4369 other => panic!("expected Accepted, got {other:?}"),
4370 };
4371 let invocations = invocations.lock().unwrap().clone();
4372 assert_eq!(invocations.len(), 1, "the callback fires exactly once, got {invocations:?}");
4373 assert!(invocations[0].1.contains("larger than 64"), "got: {}", invocations[0].1);
4374 let error = future.get().await.expect_err("the future must be failed");
4375 assert!(matches!(error, Error::RecordTooLarge(_)), "got {error:?}");
4376 }
4377
4378 /// `throwIfProducerClosed()` comes before either wait point, so a closed
4379 /// producer is an `Err` from `try_send` just as from `send` — never a
4380 /// `WouldBlock` that would have the caller queue the record for a producer
4381 /// that will not take it.
4382 #[tokio::test]
4383 async fn try_send_on_a_closed_producer_is_an_error_not_a_would_block() {
4384 let producer = create_producer_with_config(
4385 ProducerConfig::defaults(),
4386 create_metadata_with_topic(TOPIC, 1),
4387 create_accumulator(),
4388 );
4389 producer.close().await.unwrap();
4390
4391 let (callback, invocations) = recording_callback();
4392 let error = producer
4393 .try_send(string_record(Some(0), "v"), Some(callback))
4394 .await
4395 .expect_err("a closed producer rejects the send");
4396
4397 assert!(matches!(error, Error::LocalIllegalState(_)), "got {error:?}");
4398 assert_eq!(error.message(), "Cannot perform operation after producer has been closed");
4399 assert!(
4400 invocations.lock().unwrap().is_empty(),
4401 "Java's rethrow path never invokes the callback"
4402 );
4403 }
4404
4405 /// Tests that the partition() method returns the explicit partition when set.
4406 #[test]
4407 fn test_partition_returns_explicit_partition() {
4408 let metadata = create_metadata_with_topic(TOPIC, 3);
4409 let accumulator = create_accumulator();
4410 let producer = create_producer(metadata.clone(), accumulator);
4411
4412 let record = ProducerRecord::with_partition_key(
4413 TOPIC.to_string(),
4414 Some(2),
4415 Some("key".to_string()),
4416 Some("value".to_string()),
4417 )
4418 .unwrap();
4419 let cluster = metadata.fetch();
4420 let partition = producer.partition(&record, Some(b"key"), Some(b"value"), &cluster);
4421 assert_eq!(2, partition);
4422 }
4423
4424 /// Tests that the partition() method uses key hashing when no partition is set.
4425 #[test]
4426 fn test_partition_uses_key_hash() {
4427 let metadata = create_metadata_with_topic(TOPIC, 3);
4428 let accumulator = create_accumulator();
4429 let producer = create_producer(metadata.clone(), accumulator);
4430
4431 let record = ProducerRecord::with_key(TOPIC.to_string(), Some("key".to_string()), Some("value".to_string()));
4432 let cluster = metadata.fetch();
4433 let partition = producer.partition(&record, Some(b"key"), Some(b"value"), &cluster);
4434 // Should be deterministic based on key hash
4435 assert!((0..3).contains(&partition));
4436
4437 // Same key should give the same partition
4438 let partition2 = producer.partition(&record, Some(b"key"), Some(b"value"), &cluster);
4439 assert_eq!(partition, partition2);
4440 }
4441
4442 /// Tests that the partition() method returns UNKNOWN_PARTITION when no key and no partition.
4443 #[test]
4444 fn test_partition_returns_unknown_when_no_key() {
4445 let metadata = create_metadata_with_topic(TOPIC, 3);
4446 let accumulator = create_accumulator();
4447 let producer = create_producer(metadata.clone(), accumulator);
4448
4449 let record: ProducerRecord<String, String> = ProducerRecord::new(TOPIC.to_string(), Some("value".to_string()));
4450 let cluster = metadata.fetch();
4451 let partition = producer.partition(&record, None, Some(b"value"), &cluster);
4452 assert_eq!(RecordMetadata::UNKNOWN_PARTITION, partition);
4453 }
4454
4455 /// Tests that the partition() method ignores keys when partitioner_ignore_keys is true.
4456 #[test]
4457 fn test_partition_ignores_keys_when_configured() {
4458 let config = ProducerConfig { partitioner_ignore_keys: true, ..ProducerConfig::defaults() };
4459 let metadata = create_metadata_with_topic(TOPIC, 3);
4460 let accumulator = create_accumulator();
4461
4462 let producer = create_producer_with_config(config, metadata.clone(), accumulator);
4463
4464 let record = ProducerRecord::with_key(TOPIC.to_string(), Some("key".to_string()), Some("value".to_string()));
4465 let cluster = metadata.fetch();
4466 let partition = producer.partition(&record, Some(b"key"), Some(b"value"), &cluster);
4467 assert_eq!(RecordMetadata::UNKNOWN_PARTITION, partition);
4468 }
4469
4470 /// Builds a `KafkaProducer` whose `partitioner.type` selects
4471 /// `Murmur2RandomPartitioner` (the `KeyHasher::Murmur2` hash).
4472 fn create_murmur2_producer(
4473 metadata: Arc<ProducerMetadata>,
4474 accumulator: Arc<RecordAccumulator>,
4475 ) -> KafkaProducer<String, String> {
4476 let mut props = HashMap::from([("bootstrap.servers".to_string(), "localhost:9092".to_string())]);
4477 props.insert("partitioner.type".to_string(), "Murmur2RandomPartitioner".to_string());
4478 let config = ProducerConfig::new(&props).expect("murmur2 partitioner config is valid");
4479 create_producer_with_config(config, metadata, accumulator)
4480 }
4481
4482 /// The default (unset `partitioner.type`) keyed partition path uses the
4483 /// IEEE CRC-32 (librdkafka `consistent_random`) hash taken UNSIGNED modulo
4484 /// the partition count — NOT murmur2 / `Utils.toPositive`. This is the
4485 /// deliberate, user-approved deviation from Java parity.
4486 #[test]
4487 fn test_partition_default_hasher_is_crc32() {
4488 let metadata = create_metadata_with_topic(TOPIC, 7);
4489 let accumulator = create_accumulator();
4490 let producer = create_producer(metadata.clone(), accumulator);
4491 assert_eq!(KeyHasher::Crc32, producer.key_hasher);
4492
4493 let record = ProducerRecord::with_key(TOPIC.to_string(), Some("key".to_string()), Some("value".to_string()));
4494 let cluster = metadata.fetch();
4495 let partition = producer.partition(&record, Some(b"key"), Some(b"value"), &cluster);
4496 assert_eq!((crc32fast::hash(b"key") % 7) as i32, partition);
4497 // Both send call sites route through the same helper, so it agrees.
4498 assert_eq!(partition, producer.partition_for_key_or_unknown(Some(b"key"), TOPIC, &cluster));
4499 }
4500
4501 /// Under the CRC-32 default, an EMPTY (but non-null) key defers to the
4502 /// keyless sticky (KIP-794) path — `UNKNOWN_PARTITION` — matching
4503 /// librdkafka `consistent_random` (which treats `keylen == 0` as no key).
4504 /// Asserted through both send call sites' shared helper.
4505 #[test]
4506 fn test_partition_empty_key_crc32_defers_to_sticky() {
4507 let metadata = create_metadata_with_topic(TOPIC, 3);
4508 let accumulator = create_accumulator();
4509 let producer = create_producer(metadata.clone(), accumulator);
4510
4511 let record = ProducerRecord::with_key(TOPIC.to_string(), Some(String::new()), Some("value".to_string()));
4512 let cluster = metadata.fetch();
4513 // via KafkaProducer.partition()
4514 assert_eq!(
4515 RecordMetadata::UNKNOWN_PARTITION,
4516 producer.partition(&record, Some(b""), Some(b"value"), &cluster)
4517 );
4518 // via the helper do_send_bytes uses
4519 assert_eq!(
4520 RecordMetadata::UNKNOWN_PARTITION,
4521 producer.partition_for_key_or_unknown(Some(b""), TOPIC, &cluster)
4522 );
4523 }
4524
4525 /// `Murmur2RandomPartitioner` reproduces the exact Java-client formula
4526 /// `Utils.toPositive(Utils.murmur2(key)) % numPartitions` for a non-empty
4527 /// key.
4528 #[test]
4529 fn test_partition_murmur2_matches_java_formula() {
4530 let metadata = create_metadata_with_topic(TOPIC, 7);
4531 let accumulator = create_accumulator();
4532 let producer = create_murmur2_producer(metadata.clone(), accumulator);
4533 assert_eq!(KeyHasher::Murmur2, producer.key_hasher);
4534
4535 let record = ProducerRecord::with_key(TOPIC.to_string(), Some("key".to_string()), Some("value".to_string()));
4536 let cluster = metadata.fetch();
4537 let partition = producer.partition(&record, Some(b"key"), Some(b"value"), &cluster);
4538 assert_eq!(Utils::to_positive(Utils::murmur2(b"key")) % 7, partition);
4539 assert_eq!(partition, producer.partition_for_key_or_unknown(Some(b"key"), TOPIC, &cluster));
4540 }
4541
4542 /// murmur2 hashes an EMPTY (non-null) key rather than deferring to the
4543 /// sticky path — exact Java-client parity, and the point of contrast with
4544 /// the CRC-32 default (`test_partition_empty_key_crc32_defers_to_sticky`).
4545 #[test]
4546 fn test_partition_empty_key_murmur2_is_hashed() {
4547 let metadata = create_metadata_with_topic(TOPIC, 3);
4548 let accumulator = create_accumulator();
4549 let producer = create_murmur2_producer(metadata.clone(), accumulator);
4550
4551 let cluster = metadata.fetch();
4552 let partition = producer.partition_for_key_or_unknown(Some(b""), TOPIC, &cluster);
4553 assert_eq!(Utils::to_positive(Utils::murmur2(b"")) % 3, partition);
4554 assert_ne!(RecordMetadata::UNKNOWN_PARTITION, partition);
4555 }
4556
4557 /// An explicit record partition wins over key hashing regardless of the
4558 /// selected hasher (here: murmur2). Complements
4559 /// `test_partition_returns_explicit_partition` (which covers the default).
4560 #[test]
4561 fn test_partition_explicit_partition_wins_under_murmur2() {
4562 let metadata = create_metadata_with_topic(TOPIC, 3);
4563 let accumulator = create_accumulator();
4564 let producer = create_murmur2_producer(metadata.clone(), accumulator);
4565
4566 let record = ProducerRecord::with_partition_key(
4567 TOPIC.to_string(),
4568 Some(2),
4569 Some("key".to_string()),
4570 Some("value".to_string()),
4571 )
4572 .unwrap();
4573 let cluster = metadata.fetch();
4574 assert_eq!(2, producer.partition(&record, Some(b"key"), Some(b"value"), &cluster));
4575 }
4576
4577 /// `partitioner.ignore.keys=true` forces `UNKNOWN_PARTITION` even when a
4578 /// non-empty key would otherwise hash, under murmur2 as well as the default.
4579 #[test]
4580 fn test_partition_ignore_keys_overrides_murmur2() {
4581 let mut props = HashMap::from([("bootstrap.servers".to_string(), "localhost:9092".to_string())]);
4582 props.insert("partitioner.type".to_string(), "Murmur2RandomPartitioner".to_string());
4583 props.insert("partitioner.ignore.keys".to_string(), "true".to_string());
4584 let config = ProducerConfig::new(&props).unwrap();
4585 let metadata = create_metadata_with_topic(TOPIC, 3);
4586 let accumulator = create_accumulator();
4587 let producer = create_producer_with_config(config, metadata.clone(), accumulator);
4588
4589 let cluster = metadata.fetch();
4590 assert_eq!(
4591 RecordMetadata::UNKNOWN_PARTITION,
4592 producer.partition_for_key_or_unknown(Some(b"key"), TOPIC, &cluster)
4593 );
4594 }
4595
4596 // =====================================================================
4597 // PHASE-2 TEST ACCOUNTING — the pluggable-`Partitioner` `KafkaProducerTest` methods
4598 //
4599 // SCOPE CRITERION. A `KafkaProducerTest.java` method is in scope for Phase 2 iff its
4600 // body sets `PARTITIONER_CLASS_CONFIG` or names one of the partitioner nested classes
4601 // (`MockPartitioner` / `PartitionerForClientId` / `BuggyPartitioner` /
4602 // `MonitorablePartitioner`) as its subject. That marker set is exactly five methods:
4603 //
4604 // - testPartitionerClose (:668) .............. TRANSLATED -> test_partitioner_close
4605 // - negativePartitionShouldThrow (:2410) ..... TRANSLATED -> negative_partition_should_throw
4606 // - configurableObjectsShouldSeeGeneratedClientId (:2296)
4607 // ...... TRANSLATED (partitioner portion) -> configurable_objects_should_see_generated_client_id
4608 // - shouldCloseProperlyAndThrowIfInterrupted (:689) ....... N/A (see below)
4609 // - testMonitorablePlugins (:2828) ........................ N/A (see below)
4610 //
4611 // N/A justifications:
4612 //
4613 // - shouldCloseProperlyAndThrowIfInterrupted asserts that `close()` blocks until an
4614 // in-flight `send` completes and that INTERRUPTING the closing thread surfaces an
4615 // `InterruptException`. Thread interruption (`Thread.interrupt()` /
4616 // `InterruptException`) has no Rust/Tokio analogue — the same gap the producer-
4617 // transaction rules record for `TransactionalRequestResult`. `MockPartitioner` is
4618 // incidental here (merely the configured `partitioner.type`), not the subject, so
4619 // this belongs to the close/interrupt surface, not Phase 2.
4620 // - testMonitorablePlugins exercises the `Plugin` / `Monitorable` metrics SPI, which
4621 // the Phase 2 spec places explicitly out of scope. `MonitorablePartitioner` is only
4622 // the vehicle; there is nothing partitioner-specific to translate.
4623 //
4624 // RUST-ONLY BEHAVIORAL TESTS. Java gets its partitioner round-trip coverage "for free"
4625 // from a real/mock broker plus reflective `partitioner.class` loading. Rust has no
4626 // reflection (a user partitioner is passed as an instance) and these unit tests do not
4627 // stand up a broker, so five behavioral contracts that Java never asserts directly are
4628 // pinned explicitly here:
4629 //
4630 // - test_round_robin_partitioner_resolution_and_gating — `new` resolves a
4631 // `partitioner.type=RoundRobinPartitioner` and a resolved partitioner
4632 // disables adaptive partitioning (`KafkaProducer#KafkaProducer`).
4633 // - test_round_robin_partitioner_used_once_per_record — a stateful partitioner is
4634 // consulted EXACTLY once per record on the send path (do_send computes it,
4635 // do_send_bytes short-circuits on the passed `Some(..)`).
4636 // - test_partitioner_receives_key_value_and_serialized_bytes — the typed key/value
4637 // and the serialized bytes are forwarded unchanged.
4638 // - test_explicit_partition_bypasses_partitioner — an explicit record partition
4639 // short-circuits `compute_partition` before the partitioner is consulted.
4640 // - test_set_partitioner_replaces_partitioner_type_property — the partitioner
4641 // set with `ProducerConfig::set_partitioner` replaces a built-in
4642 // `partitioner.type` from the properties.
4643 // =====================================================================
4644
4645 // --- test-local partitioners (translated from KafkaProducerTest's nested classes) ---
4646
4647 /// Translated from `KafkaProducerTest.BuggyPartitioner` (`KafkaProducerTest.BuggyPartitioner#close`): always
4648 /// returns an invalid negative partition, exercising `compute_partition`'s
4649 /// non-negative guard. Java's empty `close`/`configure` overrides are the trait's
4650 /// default no-ops, so they are intentionally not overridden.
4651 struct BuggyPartitioner;
4652
4653 impl<K, V> Partitioner<K, V> for BuggyPartitioner {
4654 fn partition(
4655 &self,
4656 _topic: &str,
4657 _key: Option<&K>,
4658 _key_bytes: Option<&[u8]>,
4659 _value: Option<&V>,
4660 _value_bytes: Option<&[u8]>,
4661 _cluster: &Cluster,
4662 ) -> i32 {
4663 -1
4664 }
4665 }
4666
4667 /// Translated from `KafkaProducerTest.PartitionerForClientId`:
4668 /// `partition` returns 0, and `configure` records the resolved `client.id` so a
4669 /// test can assert the partitioner saw the generated id. Java accumulates into a
4670 /// static `CLIENT_IDS` set; here the sink is an injected `Arc<Mutex<Vec<String>>>`
4671 /// so tests running in parallel do not share it.
4672 struct PartitionerForClientId {
4673 client_ids: Arc<Mutex<Vec<String>>>,
4674 }
4675
4676 impl<K, V> Partitioner<K, V> for PartitionerForClientId {
4677 fn configure(&mut self, configs: &HashMap<String, String>) {
4678 if let Some(id) = configs.get(ProducerConfig::CLIENT_ID_CONFIG) {
4679 self.client_ids.lock().unwrap().push(id.clone());
4680 }
4681 }
4682
4683 fn partition(
4684 &self,
4685 _topic: &str,
4686 _key: Option<&K>,
4687 _key_bytes: Option<&[u8]>,
4688 _value: Option<&V>,
4689 _value_bytes: Option<&[u8]>,
4690 _cluster: &Cluster,
4691 ) -> i32 {
4692 0
4693 }
4694 }
4695
4696 /// One recorded `Partitioner::partition` invocation: the borrowed arguments
4697 /// (captured as owned copies) and the partition the delegate returned.
4698 #[derive(Debug, Clone)]
4699 struct RecordedCall {
4700 topic: String,
4701 key: Option<String>,
4702 key_bytes: Option<Vec<u8>>,
4703 value: Option<String>,
4704 value_bytes: Option<Vec<u8>>,
4705 returned: i32,
4706 }
4707
4708 /// A `Partitioner` decorator with no Java counterpart: it delegates to an inner
4709 /// partitioner and records every `partition` call. It exists to probe two
4710 /// contracts Java gets implicitly from a real broker — that a stateful partitioner
4711 /// is consulted EXACTLY once per record, and that the producer forwards the typed
4712 /// key/value and the serialized bytes unchanged. A justified `definition-of-done`
4713 /// §7 addition (a test-only decorator, present only under `#[cfg(test)]`).
4714 struct RecordingPartitioner {
4715 inner: Box<dyn Partitioner<String, String>>,
4716 calls: Arc<Mutex<Vec<RecordedCall>>>,
4717 }
4718
4719 impl Partitioner<String, String> for RecordingPartitioner {
4720 fn partition(
4721 &self,
4722 topic: &str,
4723 key: Option<&String>,
4724 key_bytes: Option<&[u8]>,
4725 value: Option<&String>,
4726 value_bytes: Option<&[u8]>,
4727 cluster: &Cluster,
4728 ) -> i32 {
4729 let returned = self.inner.partition(topic, key, key_bytes, value, value_bytes, cluster);
4730 self.calls.lock().unwrap().push(RecordedCall {
4731 topic: topic.to_string(),
4732 key: key.cloned(),
4733 key_bytes: key_bytes.map(|b| b.to_vec()),
4734 value: value.cloned(),
4735 value_bytes: value_bytes.map(|b| b.to_vec()),
4736 returned,
4737 });
4738 returned
4739 }
4740 }
4741
4742 /// Seam constructor for a producer that owns a custom `partitioner` (the
4743 /// `KafkaProducer::new` last argument). Metadata is pre-seeded (see
4744 /// `create_metadata_with_topic`) and `sender_handle` is `None`, so `send()` and
4745 /// `close()` neither block on metadata nor hang on a sender task — the send-path
4746 /// partitioner tests need a producer that can actually append a record. NOTE the
4747 /// `new()` seam stores the partitioner as-is and does NOT `configure` it or gate
4748 /// adaptive partitioning; the `new`-based tests below cover those.
4749 fn create_producer_with_partitioner(
4750 config: ProducerConfig,
4751 metadata: Arc<ProducerMetadata>,
4752 accumulator: Arc<RecordAccumulator>,
4753 partitioner: Box<dyn Partitioner<String, String>>,
4754 ) -> KafkaProducer<String, String> {
4755 let running = Arc::new(AtomicBool::new(true));
4756 let force_close = Arc::new(AtomicBool::new(false));
4757 let wakeup = Arc::new(Notify::new());
4758
4759 KafkaProducer::with_options(
4760 KafkaProducerOptionsBuilder::new()
4761 .set_config(&config)
4762 .set_key_serializer(Box::new(StringSerializer))
4763 .set_value_serializer(Box::new(StringSerializer))
4764 .set_metadata(metadata)
4765 .set_accumulator(accumulator)
4766 .set_running(running)
4767 .set_force_close(force_close)
4768 .set_wakeup(wakeup)
4769 .set_time(default_time())
4770 .set_pending_requests(Arc::new(Mutex::new(PendingRequests::new())))
4771 .set_partitioner(Some(partitioner))
4772 .build()
4773 .expect("KafkaProducerOptionsBuilder::build: every mandatory parameter is set above"),
4774 )
4775 }
4776
4777 /// Translated from `KafkaProducerTest.testPartitionerClose` (`KafkaProducer`).
4778 ///
4779 /// Java resolves the partitioner reflectively through `partitioner.class`, which
4780 /// invokes the `MockPartitioner` constructor (bumping `INIT_COUNT`) and then
4781 /// `configure`. `MockPartitioner` is a test-only type with no built-in
4782 /// `partitioner.type` name, so here it is supplied as an explicit instance;
4783 /// constructing it via `MockPartitioner::new()` bumps `INIT_COUNT` exactly as the
4784 /// Java ctor does. `close()` then bumps `CLOSE_COUNT` (`KafkaProducer#close`).
4785 ///
4786 /// The counters are process-global statics, so the whole body must hold
4787 /// `lock_counters()` and reset at start AND end (Java's `finally`) to serialize
4788 /// against the other counter-touching test (`mock_partitioner`'s `test_counts_init_and_close`).
4789 /// That serializer is a `std::sync::Mutex` guard (matching the codebase's
4790 /// preference for CPU-bound state, and shared with the D6 test), and it must stay
4791 /// held across the async `close()` — otherwise a parallel counter test could
4792 /// interleave between the INIT and CLOSE assertions. Holding a `std` guard across
4793 /// an `.await` is exactly what `clippy::await_holding_lock` forbids, so this is a
4794 /// plain `#[test]` that drives `close()` on a dedicated current-thread runtime via
4795 /// `block_on`: the guard spans a synchronous `block_on` call rather than an
4796 /// `.await` point (and the close path never locks `COUNTER_GUARD`, so parking the
4797 /// thread inside `block_on` while holding it cannot deadlock).
4798 #[test]
4799 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testPartitionerClose")]
4800 fn test_partitioner_close() {
4801 let _guard = MockPartitioner::lock_counters();
4802 MockPartitioner::reset_counters();
4803
4804 let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap();
4805 runtime.block_on(async {
4806 let producer = create_producer_with_partitioner(
4807 ProducerConfig::defaults(),
4808 create_metadata_with_topic(TOPIC, 1),
4809 create_accumulator(),
4810 Box::new(MockPartitioner::new()),
4811 );
4812 assert_eq!(1, MockPartitioner::init_count().load(Ordering::SeqCst));
4813 assert_eq!(0, MockPartitioner::close_count().load(Ordering::SeqCst));
4814
4815 producer.close().await.expect("seam producer closes cleanly");
4816
4817 assert_eq!(1, MockPartitioner::init_count().load(Ordering::SeqCst));
4818 assert_eq!(1, MockPartitioner::close_count().load(Ordering::SeqCst));
4819 });
4820
4821 // Cleanup since we use mutable process-global statics in MockPartitioner.
4822 MockPartitioner::reset_counters();
4823 }
4824
4825 /// Java's `MockSerializer` (`clients/src/test/java/org/apache/kafka/test/MockSerializer.java`):
4826 /// counts its constructions and closes. Java keeps the counters in statics;
4827 /// here each test shares its own, so parallel tests cannot disturb them.
4828 use std::sync::atomic::AtomicUsize;
4829
4830 struct MockSerializer {
4831 close_count: Arc<AtomicUsize>,
4832 panic_on_close: Option<&'static str>,
4833 fail_on_close: Option<&'static str>,
4834 }
4835
4836 impl MockSerializer {
4837 fn new(init_count: &AtomicUsize, close_count: &Arc<AtomicUsize>) -> Self {
4838 init_count.fetch_add(1, Ordering::SeqCst);
4839 Self { close_count: Arc::clone(close_count), panic_on_close: None, fail_on_close: None }
4840 }
4841 }
4842
4843 impl Serializer<String> for MockSerializer {
4844 async fn serialize<'a>(
4845 &'a self,
4846 _topic: &'a str,
4847 data: Option<&'a String>,
4848 ) -> Result<Option<Cow<'a, [u8]>>, Error> {
4849 Ok(data.map(|data| Cow::Borrowed(data.as_bytes())))
4850 }
4851
4852 async fn close(&self) -> Result<(), Error> {
4853 self.close_count.fetch_add(1, Ordering::SeqCst);
4854 if let Some(message) = self.panic_on_close {
4855 panic!("{message}");
4856 }
4857 match self.fail_on_close {
4858 Some(message) => Err(Error::serialization(message)),
4859 None => Ok(()),
4860 }
4861 }
4862 }
4863
4864 fn create_producer_with_serializers(
4865 key_serializer: Box<dyn DynSerializer<String>>,
4866 value_serializer: Box<dyn DynSerializer<String>>,
4867 ) -> KafkaProducer<String, String> {
4868 KafkaProducer::with_options(
4869 KafkaProducerOptionsBuilder::new()
4870 .set_config(&ProducerConfig::defaults())
4871 .set_key_serializer(key_serializer)
4872 .set_value_serializer(value_serializer)
4873 .set_metadata(create_metadata_with_topic(TOPIC, 1))
4874 .set_accumulator(create_accumulator())
4875 .set_time(default_time())
4876 .set_pending_requests(Arc::new(Mutex::new(PendingRequests::new())))
4877 .build()
4878 .expect("KafkaProducerOptionsBuilder::build: every mandatory parameter is set above"),
4879 )
4880 }
4881
4882 /// Translated from `KafkaProducerTest.testSerializerClose`:
4883 /// `close` closes both serializers (`KafkaProducer#close`). The
4884 /// second `close` (Java's try-with-resources closes once) shows the
4885 /// serializers are closed only once.
4886 #[tokio::test]
4887 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testSerializerClose")]
4888 async fn test_serializer_close() {
4889 let init_count = AtomicUsize::new(0);
4890 let close_count = Arc::new(AtomicUsize::new(0));
4891 let producer = create_producer_with_serializers(
4892 Box::new(MockSerializer::new(&init_count, &close_count)),
4893 Box::new(MockSerializer::new(&init_count, &close_count)),
4894 );
4895 assert_eq!(2, init_count.load(Ordering::SeqCst));
4896 assert_eq!(0, close_count.load(Ordering::SeqCst));
4897
4898 producer.close().await.expect("close");
4899
4900 assert_eq!(2, init_count.load(Ordering::SeqCst));
4901 assert_eq!(2, close_count.load(Ordering::SeqCst));
4902
4903 producer.close().await.expect("a second close");
4904 assert_eq!(2, close_count.load(Ordering::SeqCst));
4905 }
4906
4907 /// `Utils.closeQuietly` keeps closing after a failure and `close` raises
4908 /// the first one as the cause of `KafkaException("Failed to close kafka
4909 /// producer", ..)` (`KafkaProducer#close`): the value serializer
4910 /// is closed although the key serializer's `close` panicked.
4911 #[tokio::test]
4912 async fn test_serializer_close_failure_closes_both_and_reports_the_first() {
4913 let init_count = AtomicUsize::new(0);
4914 let close_count = Arc::new(AtomicUsize::new(0));
4915 let mut key = MockSerializer::new(&init_count, &close_count);
4916 key.panic_on_close = Some("key close failed");
4917 let mut value = MockSerializer::new(&init_count, &close_count);
4918 value.panic_on_close = Some("value close failed");
4919 let producer = create_producer_with_serializers(Box::new(key), Box::new(value));
4920
4921 let error = producer.close().await.expect_err("both closes panicked");
4922 assert!(error.is_kafka_error());
4923 assert_eq!("Failed to close kafka producer", error.message());
4924 let source = std::error::Error::source(&error).expect("the first failure is the source");
4925 assert_eq!("key close failed", source.to_string());
4926 assert_eq!(2, close_count.load(Ordering::SeqCst));
4927
4928 producer.close().await.expect("the serializers are not closed again");
4929 assert_eq!(2, close_count.load(Ordering::SeqCst));
4930 }
4931
4932 /// A serializer `close` returning an error takes the same path as a
4933 /// throwing one in Java: it is recorded by `closeQuietly`, the other
4934 /// serializer is still closed, and `close` returns it as the cause of
4935 /// `Failed to close kafka producer` (`KafkaProducer#close`).
4936 #[tokio::test]
4937 async fn test_serializer_close_error_closes_both_and_reports_the_first() {
4938 let init_count = AtomicUsize::new(0);
4939 let close_count = Arc::new(AtomicUsize::new(0));
4940 let mut key = MockSerializer::new(&init_count, &close_count);
4941 key.fail_on_close = Some("key close failed");
4942 let mut value = MockSerializer::new(&init_count, &close_count);
4943 value.panic_on_close = Some("value close failed");
4944 let producer = create_producer_with_serializers(Box::new(key), Box::new(value));
4945
4946 let error = producer.close().await.expect_err("both closes failed");
4947 assert!(error.is_kafka_error());
4948 assert_eq!("Failed to close kafka producer", error.message());
4949 let source = error.source().expect("the first failure is the source");
4950 assert!(source.is_serialization_error(), "the serializer's own error: {source:?}");
4951 assert_eq!("key close failed", source.message());
4952 assert_eq!(2, close_count.load(Ordering::SeqCst));
4953 }
4954
4955 /// KAFKA-2121: a failed constructor closes the serializers, as Java's
4956 /// `catch (Throwable t)` calls `close(Duration.ofMillis(0), true)`
4957 /// (`KafkaProducer#KafkaProducer`), whose `closeQuietly` closes both
4958 /// serializers (`KafkaProducer#close`). Here the first fallible step fails
4959 /// (`configureDeliveryTimeout`'s `ConfigException`), without a Tokio
4960 /// runtime: a serializer whose `close` does not wait is closed inline.
4961 #[test]
4962 fn test_serializers_closed_when_construction_fails() {
4963 let init_count = AtomicUsize::new(0);
4964 let close_count = Arc::new(AtomicUsize::new(0));
4965 let mut props = HashMap::from([("bootstrap.servers".to_string(), "localhost:9999".to_string())]);
4966 props.insert("delivery.timeout.ms".to_string(), "1".to_string());
4967 let config = ProducerConfig::new(&props).expect("valid config");
4968
4969 let error = KafkaProducer::<String, String>::new(
4970 config,
4971 MockSerializer::new(&init_count, &close_count),
4972 MockSerializer::new(&init_count, &close_count),
4973 )
4974 .expect_err("delivery.timeout.ms below linger.ms + request.timeout.ms fails construction");
4975
4976 assert_eq!("Failed to construct kafka producer", error.message());
4977 assert_eq!(2, init_count.load(Ordering::SeqCst));
4978 assert_eq!(2, close_count.load(Ordering::SeqCst), "both serializers are closed once");
4979 }
4980
4981 /// A serializer `close` that panics on a failed constructor is logged, not
4982 /// propagated (`swallowException` is `true`, `KafkaProducer#close`), and
4983 /// the other serializer is still closed; the constructor's error is
4984 /// unchanged.
4985 #[test]
4986 fn test_serializer_close_failure_on_failed_construction_is_swallowed() {
4987 let init_count = AtomicUsize::new(0);
4988 let close_count = Arc::new(AtomicUsize::new(0));
4989 let mut key = MockSerializer::new(&init_count, &close_count);
4990 key.panic_on_close = Some("key close failed");
4991 let mut props = HashMap::from([("bootstrap.servers".to_string(), "localhost:9999".to_string())]);
4992 props.insert("delivery.timeout.ms".to_string(), "1".to_string());
4993 let config = ProducerConfig::new(&props).expect("valid config");
4994
4995 let error = KafkaProducer::<String, String>::new(config, key, MockSerializer::new(&init_count, &close_count))
4996 .expect_err("the construction fails");
4997
4998 assert_eq!("Failed to construct kafka producer", error.message());
4999 assert_eq!(2, close_count.load(Ordering::SeqCst), "the value serializer is closed too");
5000 }
5001
5002 /// Translated from `KafkaProducerTest.negativePartitionShouldThrow`.
5003 ///
5004 /// A partitioner returning a negative partition makes `send()` fail with
5005 /// `IllegalArgument` (Java: `IllegalArgumentException`). The exact message is
5006 /// asserted (DoD §3): error-message content is part of the behavioral contract.
5007 #[tokio::test]
5008 async fn negative_partition_should_throw() {
5009 let producer = create_producer_with_partitioner(
5010 ProducerConfig::defaults(),
5011 create_metadata_with_topic("topic", 1),
5012 create_accumulator(),
5013 Box::new(BuggyPartitioner),
5014 );
5015
5016 let record = ProducerRecord::with_key("topic".to_string(), Some("key".to_string()), Some("value".to_string()));
5017 let err = producer.send(record).await.expect_err("negative partition must be rejected");
5018 match &err {
5019 Error::LocalIllegalArgument(msg) => {
5020 assert_eq!(
5021 msg.message(),
5022 "The partitioner generated an invalid partition number: -1. Partition number should always be non-negative."
5023 );
5024 },
5025 other => panic!("Expected LocalIllegalArgument error, got: {:?}", other),
5026 }
5027 }
5028
5029 /// Translated from `KafkaProducerTest.configurableObjectsShouldSeeGeneratedClientId`
5030 /// (`KafkaProducerTest`), partitioner portion.
5031 ///
5032 /// A `configure`-able partitioner must see the generated `client.id` when the user
5033 /// sets none. Java asserts the same for the two serializers and the interceptor
5034 /// (`CLIENT_IDS.size() == 4`). Java configures only a serializer it builds from
5035 /// `key.serializer` / `value.serializer`, never one passed as an instance
5036 /// (`KafkaProducer#KafkaProducer`), and a Rust serializer is always passed as an
5037 /// instance; no `ProducerInterceptor` exists yet. So only the partitioner is
5038 /// checked here — hence exactly ONE recorded id rather than four. The partitioner is set
5039 /// with `ProducerConfig::set_partitioner`, as Java sets `partitioner.class`,
5040 /// so `new` builds and `configure`s it (the `with_options` seam does not
5041 /// configure).
5042 #[tokio::test]
5043 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#configurableObjectsShouldSeeGeneratedClientId")]
5044 async fn configurable_objects_should_see_generated_client_id() {
5045 let client_ids = Arc::new(Mutex::new(Vec::new()));
5046 let props = guard_props(&[]); // bootstrap only; NO client.id
5047 let config = ProducerConfig::new(&props)
5048 .expect("valid config")
5049 .set_partitioner::<String, String>(Box::new(PartitionerForClientId {
5050 client_ids: Arc::clone(&client_ids),
5051 }));
5052 let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
5053 .expect("constructs with a custom partitioner");
5054
5055 assert!(!producer.client_id().is_empty(), "a client.id is generated when unset");
5056 let recorded = client_ids.lock().unwrap();
5057 assert_eq!(1, recorded.len(), "the partitioner's configure ran exactly once");
5058 assert_eq!(
5059 recorded[0].as_str(),
5060 producer.client_id(),
5061 "the partitioner saw the generated client.id"
5062 );
5063 }
5064
5065 /// `new` gives the producer's metadata the `retry.backoff.ms` / `retry.backoff.max.ms`
5066 /// refresh backoff, as Java's constructor does, not the `reconnect.backoff.*` values
5067 /// that only the `NetworkClient` takes.
5068 ///
5069 /// Not a translation of a Java test. After one failed refresh the backoff is
5070 /// `retry.backoff.ms`, and after repeated failures it reaches `retry.backoff.max.ms`,
5071 /// each within `ExponentialBackoff`'s jitter and never above the maximum. The maximum
5072 /// differs from its default, and neither range overlaps the `reconnect.backoff.*`
5073 /// values, so each bound shows which config is used. The producer is closed first, so
5074 /// the Sender task cannot record a refresh of its own while the metadata is read.
5075 #[tokio::test]
5076 async fn test_metadata_refresh_backoff_follows_retry_backoff() {
5077 const RETRY_BACKOFF_MS: i64 = 100;
5078 const RETRY_BACKOFF_MAX_MS: i64 = 2000;
5079 let jitter = crate::CommonClientConfigs::RETRY_BACKOFF_JITTER;
5080 let props = guard_props(&[
5081 ("retry.backoff.ms", &RETRY_BACKOFF_MS.to_string()),
5082 ("retry.backoff.max.ms", &RETRY_BACKOFF_MAX_MS.to_string()),
5083 ("reconnect.backoff.ms", "50"),
5084 ("reconnect.backoff.max.ms", "50"),
5085 ]);
5086 let config = ProducerConfig::new(&props).expect("valid config");
5087 let producer =
5088 KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer).expect("constructs");
5089 producer.close_with_timeout(Duration::ZERO).await.expect("close");
5090
5091 let now = SystemTime.milliseconds();
5092 producer.metadata.failed_update(now);
5093 let initial = producer.metadata.time_to_allow_update(now);
5094 let initial_range =
5095 (RETRY_BACKOFF_MS as f64 * (1.0 - jitter)) as i64..=(RETRY_BACKOFF_MS as f64 * (1.0 + jitter)) as i64;
5096 assert!(
5097 initial_range.contains(&initial),
5098 "the first refresh backoff must be retry.backoff.ms with jitter, got {initial}"
5099 );
5100
5101 // `RETRY_BACKOFF_EXP_BASE` doubles the backoff per failure, so ten failures take it
5102 // well past the maximum.
5103 for _ in 0..10 {
5104 producer.metadata.failed_update(now);
5105 }
5106 let capped = producer.metadata.time_to_allow_update(now);
5107 let capped_range = (RETRY_BACKOFF_MAX_MS as f64 * (1.0 - jitter)) as i64..=RETRY_BACKOFF_MAX_MS;
5108 assert!(
5109 capped_range.contains(&capped),
5110 "repeated failures must back off up to retry.backoff.max.ms, got {capped}"
5111 );
5112 }
5113
5114 /// `partitioner.type=RoundRobinPartitioner` is resolved by `new` into a live
5115 /// `partitioner`, and a
5116 /// resolved partitioner turns OFF adaptive partitioning in the accumulator (Java
5117 /// `KafkaProducer#KafkaProducer`: "no need ... if we use a custom partitioner"). The
5118 /// control producer (no `partitioner.type`) keeps `partitioner = None` and adaptive
5119 /// partitioning follows the config default.
5120 ///
5121 /// Rust-only behavioral test (Java never inspects these internals); it pins the
5122 /// `new` resolution + accumulator gating D3/D4 added. `#[tokio::test]`:
5123 /// `new` spawns the Sender task, which fails to reach localhost:9999
5124 /// harmlessly and is dropped with the test.
5125 #[tokio::test]
5126 async fn test_round_robin_partitioner_resolution_and_gating() {
5127 {
5128 let name = ProducerConfig::ROUND_ROBIN_PARTITIONER;
5129 let props = guard_props(&[("partitioner.type", name)]);
5130 let config = ProducerConfig::new(&props).expect("valid config");
5131 let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
5132 .expect("RoundRobinPartitioner resolves");
5133 assert!(
5134 producer.partitioner.is_some(),
5135 "partitioner.type={name} must resolve a partitioner"
5136 );
5137 assert!(
5138 !producer.accumulator.enable_adaptive_partitioning_for_test(),
5139 "a resolved partitioner must disable adaptive partitioning (partitioner.type={name})"
5140 );
5141 }
5142
5143 // Control: no partitioner.type -> no partitioner, adaptive follows config.
5144 let props = guard_props(&[]);
5145 let config = ProducerConfig::new(&props).expect("valid config");
5146 let adaptive_default = config.partitioner_adaptive_partitioning_enable;
5147 let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
5148 .expect("default config constructs");
5149 assert!(producer.partitioner.is_none(), "no partitioner.type -> no partitioner");
5150 assert_eq!(
5151 adaptive_default,
5152 producer.accumulator.enable_adaptive_partitioning_for_test(),
5153 "without a partitioner, adaptive partitioning follows partitioner.adaptive.partitioning.enable"
5154 );
5155 }
5156
5157 /// A stateful `RoundRobinPartitioner` used through the producer's send path is
5158 /// consulted EXACTLY once per record and cycles through the topic's available
5159 /// partitions. A recording decorator wraps the RoundRobin; four sends over a
5160 /// 3-partition topic must produce four calls whose first three returned partitions
5161 /// are a permutation of {0,1,2} and whose fourth repeats the first (the counter
5162 /// wraps mod 3). The available-partition ORDER is not contractually fixed, so the
5163 /// assertion is order-robust rather than hard-coded to `[0,1,2,0]`.
5164 #[tokio::test]
5165 async fn test_round_robin_partitioner_used_once_per_record() {
5166 let calls = Arc::new(Mutex::new(Vec::new()));
5167 let partitioner =
5168 RecordingPartitioner { inner: Box::new(RoundRobinPartitioner::new()), calls: Arc::clone(&calls) };
5169 let producer = create_producer_with_partitioner(
5170 ProducerConfig::defaults(),
5171 create_metadata_with_topic("topic", 3),
5172 create_accumulator(),
5173 Box::new(partitioner),
5174 );
5175
5176 for _ in 0..4 {
5177 let record = ProducerRecord::with_key("topic".to_string(), Some("k".to_string()), Some("v".to_string()));
5178 producer.send(record).await.expect("send succeeds");
5179 }
5180
5181 let recorded = calls.lock().unwrap();
5182 assert_eq!(4, recorded.len(), "the partitioner is consulted exactly once per record");
5183 let returned: Vec<i32> = recorded.iter().map(|c| c.returned).collect();
5184 let mut first_cycle = returned[..3].to_vec();
5185 first_cycle.sort_unstable();
5186 assert_eq!(vec![0, 1, 2], first_cycle, "the first cycle visits every partition once");
5187 assert_eq!(returned[0], returned[3], "the fourth record wraps back to the first partition");
5188 }
5189
5190 /// The producer forwards the record's typed key/value AND the serialized bytes to
5191 /// the partitioner unchanged. A single send of key "mykey"/value "myval" with no
5192 /// explicit partition must record `key=Some("mykey")`, `key_bytes=b"mykey"`,
5193 /// `value=Some("myval")`, `value_bytes=b"myval"` (the `StringSerializer` output).
5194 #[tokio::test]
5195 async fn test_partitioner_receives_key_value_and_serialized_bytes() {
5196 let calls = Arc::new(Mutex::new(Vec::new()));
5197 let partitioner =
5198 RecordingPartitioner { inner: Box::new(RoundRobinPartitioner::new()), calls: Arc::clone(&calls) };
5199 let producer = create_producer_with_partitioner(
5200 ProducerConfig::defaults(),
5201 create_metadata_with_topic("topic", 3),
5202 create_accumulator(),
5203 Box::new(partitioner),
5204 );
5205
5206 let record =
5207 ProducerRecord::with_key("topic".to_string(), Some("mykey".to_string()), Some("myval".to_string()));
5208 producer.send(record).await.expect("send succeeds");
5209
5210 let recorded = calls.lock().unwrap();
5211 assert_eq!(1, recorded.len());
5212 let call = &recorded[0];
5213 assert_eq!("topic", call.topic);
5214 assert_eq!(Some("mykey".to_string()), call.key);
5215 assert_eq!(Some(b"mykey".to_vec()), call.key_bytes);
5216 assert_eq!(Some("myval".to_string()), call.value);
5217 assert_eq!(Some(b"myval".to_vec()), call.value_bytes);
5218 }
5219
5220 /// An explicit record partition short-circuits `compute_partition` before the
5221 /// partitioner is consulted (it returns the given `Some(p)` immediately). A send
5222 /// with an explicit partition therefore records ZERO partitioner calls and still
5223 /// succeeds.
5224 #[tokio::test]
5225 async fn test_explicit_partition_bypasses_partitioner() {
5226 let calls = Arc::new(Mutex::new(Vec::new()));
5227 let partitioner =
5228 RecordingPartitioner { inner: Box::new(RoundRobinPartitioner::new()), calls: Arc::clone(&calls) };
5229 let producer = create_producer_with_partitioner(
5230 ProducerConfig::defaults(),
5231 create_metadata_with_topic("topic", 3),
5232 create_accumulator(),
5233 Box::new(partitioner),
5234 );
5235
5236 let record = ProducerRecord::with_partition_key(
5237 "topic".to_string(),
5238 Some(2),
5239 Some("k".to_string()),
5240 Some("v".to_string()),
5241 )
5242 .unwrap();
5243 producer.send(record).await.expect("send with explicit partition succeeds");
5244
5245 assert!(
5246 calls.lock().unwrap().is_empty(),
5247 "explicit partition must bypass the partitioner"
5248 );
5249 }
5250
5251 /// `ProducerConfig::set_partitioner` replaces a built-in `partitioner.type`
5252 /// from the properties, as a later value for the same key would in Java.
5253 /// Verified via `configure`: only `PartitionerForClientId` records the client.id
5254 /// (`RoundRobinPartitioner`'s `configure` is the default no-op), so a single
5255 /// recorded id proves the user-written partitioner — not the RoundRobin named in
5256 /// the properties — was built and configured.
5257 #[tokio::test]
5258 async fn test_set_partitioner_replaces_partitioner_type_property() {
5259 let client_ids = Arc::new(Mutex::new(Vec::new()));
5260 let props = guard_props(&[("partitioner.type", "RoundRobinPartitioner")]);
5261 let config = ProducerConfig::new(&props)
5262 .expect("valid config")
5263 .set_partitioner::<String, String>(Box::new(PartitionerForClientId {
5264 client_ids: Arc::clone(&client_ids),
5265 }));
5266 let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
5267 .expect("set_partitioner overrides partitioner.type");
5268
5269 let recorded = client_ids.lock().unwrap();
5270 assert_eq!(1, recorded.len(), "PartitionerForClientId was configured, not RoundRobin");
5271 assert_eq!(recorded[0].as_str(), producer.client_id());
5272 assert!(
5273 !producer.accumulator.enable_adaptive_partitioning_for_test(),
5274 "a custom partitioner disables adaptive partitioning"
5275 );
5276 }
5277
5278 /// A partitioner set with `ProducerConfig::set_partitioner` for other
5279 /// record types than the producer's fails construction, as Java's
5280 /// `getConfiguredInstance` throws `KafkaException("<class> is not an instance of
5281 /// <type>")` for a class that does not implement the requested interface. The
5282 /// failure is relabelled like every other construction failure.
5283 #[tokio::test]
5284 async fn test_set_partitioner_with_other_record_types_fails() {
5285 let props = guard_props(&[]);
5286 let config = ProducerConfig::new(&props)
5287 .expect("valid config")
5288 .set_partitioner::<Vec<u8>, Vec<u8>>(Box::new(RoundRobinPartitioner::new()));
5289 let err = match KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer) {
5290 Ok(_) => panic!("a partitioner for other record types must be rejected"),
5291 Err(err) => err,
5292 };
5293 assert!(err.to_string().contains("Failed to construct kafka producer"), "{err}");
5294 let source = std::error::Error::source(&err)
5295 .expect("the cause is kept as the source")
5296 .to_string();
5297 assert!(
5298 source.contains(&format!(
5299 "{} is not an instance of {}",
5300 std::any::type_name::<dyn Partitioner<Vec<u8>, Vec<u8>>>(),
5301 std::any::type_name::<dyn Partitioner<String, String>>()
5302 )),
5303 "{source}"
5304 );
5305 }
5306
5307 /// Tests that a record can be successfully sent and appended to the accumulator.
5308 ///
5309 /// Translated from `KafkaProducerTest.testFlushCompleteSendOfInflightBatches`
5310 /// (the send part).
5311 #[tokio::test]
5312 async fn test_send_appends_to_accumulator() {
5313 let metadata = create_metadata_with_topic(TOPIC, 1);
5314 let accumulator = create_accumulator();
5315 let producer = create_producer(metadata, Arc::clone(&accumulator));
5316
5317 let record = ProducerRecord::with_key(TOPIC.to_string(), Some("key".to_string()), Some("value".to_string()));
5318 let result = producer.send(record).await;
5319 assert!(result.is_ok(), "Send should succeed");
5320
5321 // Verify that the accumulator has undrained batches
5322 assert!(accumulator.has_undrained());
5323 }
5324
5325 /// Tests that flush with no incomplete batches returns immediately.
5326 ///
5327 /// When there are no pending records, flush should complete instantly
5328 /// since there is nothing to wait for.
5329 #[tokio::test]
5330 async fn test_flush_with_no_pending_records() {
5331 let metadata = create_metadata_with_topic(TOPIC, 1);
5332 let accumulator = create_accumulator();
5333 let producer = create_producer(metadata, accumulator);
5334
5335 // Flush with nothing pending should succeed immediately
5336 let result = producer.flush().await;
5337 assert!(result.is_ok());
5338 }
5339
5340 /// Translated from `KafkaProducerTest.testDeliveryTimeoutAndLingerMsConfig`.
5341 ///
5342 /// `delivery.timeout.ms` must be >= `linger.ms + request.timeout.ms`. When the
5343 /// user set it explicitly, Java throws `ConfigException` (`KafkaProducer`)
5344 /// — and the Java test asserts `KafkaException.class`, so the error must answer
5345 /// `true` to `is_kafka_error()`.
5346 #[test]
5347 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testDeliveryTimeoutAndLingerMsConfig")]
5348 fn test_delivery_timeout_and_linger_ms_config() {
5349 let mut props = HashMap::new();
5350 props.insert("client.id".to_string(), "testDeliveryTimeoutAndLingerMsConfig".to_string());
5351 props.insert("bootstrap.servers".to_string(), "localhost:9999".to_string());
5352 props.insert("delivery.timeout.ms".to_string(), "1000".to_string());
5353 props.insert("linger.ms".to_string(), "1000".to_string());
5354 props.insert("request.timeout.ms".to_string(), "1".to_string());
5355 let config = ProducerConfig::new(&props).expect("these properties parse");
5356 let log_context = LogContext::new("[test] ".to_string());
5357
5358 let err = KafkaProducer::<String, String>::configure_delivery_timeout(&config, &log_context)
5359 .expect_err("Should reject an explicit delivery_timeout_ms < linger_ms + request_timeout_ms");
5360 assert_eq!(
5361 err.message(),
5362 "delivery.timeout.ms should be equal to or larger than linger.ms + request.timeout.ms"
5363 );
5364 assert!(matches!(err, Error::Config(_)), "Expected a Config error, got: {err:?}");
5365 // `ConfigException extends KafkaException`, which is what the Java test asserts.
5366 assert!(err.is_kafka_error(), "Java's ConfigException is a KafkaException");
5367 assert!(!err.is_api_error(), "Java's ConfigException is not an ApiException");
5368
5369 // Second half of the Java test: linger.ms = 999 makes the sum exactly 1000,
5370 // so construction succeeds.
5371 props.insert("linger.ms".to_string(), "999".to_string());
5372 let config = ProducerConfig::new(&props).expect("these properties parse");
5373 assert_eq!(
5374 KafkaProducer::<String, String>::configure_delivery_timeout(&config, &log_context).unwrap(),
5375 1000
5376 );
5377 }
5378
5379 /// Java's *other* arm: when `delivery.timeout.ms` was NOT supplied by the user,
5380 /// an inconsistency is clamped up to `linger.ms + request.timeout.ms` and warned
5381 /// about rather than rejected (`KafkaProducer`).
5382 ///
5383 /// `request.timeout.ms = 180000` with everything else defaulted
5384 /// (`delivery.timeout.ms = 120000`, `linger.ms = 5`) is a legal Kafka
5385 /// configuration: Java silently raises the delivery timeout to 180005.
5386 #[test]
5387 fn test_delivery_timeout_default_is_clamped_not_rejected() {
5388 let mut props = HashMap::new();
5389 props.insert("bootstrap.servers".to_string(), "localhost:9092".to_string());
5390 props.insert("request.timeout.ms".to_string(), "180000".to_string());
5391 let config = ProducerConfig::new(&props).expect("these properties parse");
5392 assert!(
5393 !config.user_configured(ProducerConfig::DELIVERY_TIMEOUT_MS_CONFIG),
5394 "the test relies on delivery.timeout.ms coming from the default"
5395 );
5396 assert_eq!(config.delivery_timeout_ms, 120_000);
5397 assert_eq!(config.linger_ms, 5);
5398
5399 let log_context = LogContext::new("[test] ".to_string());
5400 let delivery_timeout_ms = KafkaProducer::<String, String>::configure_delivery_timeout(&config, &log_context)
5401 .expect("an inconsistency the user did not ask for is clamped, not rejected");
5402 assert_eq!(delivery_timeout_ms, 180_005, "clamped to linger.ms + request.timeout.ms");
5403 }
5404
5405 /// Tests that configure_delivery_timeout accepts valid configurations.
5406 #[test]
5407 fn test_delivery_timeout_valid_config() {
5408 let log_context = LogContext::new("[test] ".to_string());
5409
5410 // Default values: delivery=120000, linger=5, request=30000
5411 let config = ProducerConfig::defaults();
5412 let result = KafkaProducer::<String, String>::configure_delivery_timeout(&config, &log_context);
5413 assert!(result.is_ok());
5414 assert_eq!(result.unwrap(), 120_000);
5415
5416 // Exactly equal: delivery = linger + request
5417 let config = ProducerConfig {
5418 delivery_timeout_ms: 30_010,
5419 linger_ms: 10,
5420 request_timeout_ms: 30_000,
5421 ..ProducerConfig::defaults()
5422 };
5423 let result = KafkaProducer::<String, String>::configure_delivery_timeout(&config, &log_context);
5424 assert!(result.is_ok());
5425 assert_eq!(result.unwrap(), 30_010);
5426 }
5427
5428 /// `linger.ms` values above `i32::MAX` are clamped to `i32::MAX`, as Java's
5429 /// `KafkaProducer.lingerMs` does, instead of being truncated to their low 32
5430 /// bits. The clamped value also bounds `linger.ms + request.timeout.ms` in
5431 /// `configure_delivery_timeout`.
5432 #[test]
5433 fn test_linger_ms_beyond_i32_max_is_clamped() {
5434 let log_context = LogContext::new("[test] ".to_string());
5435 let max = i32::MAX.to_string();
5436 let i64_max = i64::MAX.to_string();
5437 for (linger_ms, expected) in [
5438 ("0", 0),
5439 ("5", 5),
5440 (max.as_str(), i32::MAX),
5441 ("2147483648", i32::MAX),
5442 ("3000000000", i32::MAX),
5443 ("4294967296", i32::MAX),
5444 (i64_max.as_str(), i32::MAX),
5445 ] {
5446 let config = ProducerConfig::new(&guard_props(&[("linger.ms", linger_ms)])).expect("valid config");
5447 assert_eq!(
5448 KafkaProducer::<String, String>::linger_ms(&config),
5449 expected,
5450 "linger.ms={linger_ms}"
5451 );
5452 if expected == i32::MAX {
5453 assert_eq!(
5454 KafkaProducer::<String, String>::configure_delivery_timeout(&config, &log_context).unwrap(),
5455 i32::MAX,
5456 "the default delivery.timeout.ms is raised to the clamped sum (linger.ms={linger_ms})"
5457 );
5458 }
5459 }
5460 }
5461
5462 /// The producer passes the clamped `linger.ms` to its accumulator, so a value
5463 /// above `i32::MAX` keeps batches lingering instead of making them ready at once.
5464 /// `#[tokio::test]`: `new` spawns the Sender task, which fails to reach
5465 /// localhost:9999 harmlessly and is dropped with the test.
5466 #[tokio::test]
5467 async fn test_linger_ms_beyond_i32_max_reaches_accumulator_clamped() {
5468 for linger_ms in ["3000000000", "4294967296"] {
5469 let config = ProducerConfig::new(&guard_props(&[("linger.ms", linger_ms)])).expect("valid config");
5470 let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
5471 .expect("a large linger.ms is a valid configuration");
5472 assert_eq!(producer.accumulator.linger_ms_for_test(), i32::MAX, "linger.ms={linger_ms}");
5473 }
5474 }
5475
5476 /// Tests that `negativePartitionShouldThrow` from Java is already handled
5477 /// by `ProducerRecord` validation. Negative partition is rejected at record
5478 /// construction time.
5479 ///
5480 /// Translated from `KafkaProducerTest.negativePartitionShouldThrow`.
5481 #[test]
5482 fn test_negative_partition_should_error() {
5483 let result: Result<ProducerRecord<String, String>, _> = ProducerRecord::with_partition_key(
5484 TOPIC.to_string(),
5485 Some(-1),
5486 Some("key".to_string()),
5487 Some("value".to_string()),
5488 );
5489 assert!(result.is_err(), "Negative partition should be rejected");
5490 }
5491
5492 /// Tests the wait_on_metadata method with an already-known topic.
5493 #[tokio::test]
5494 async fn test_wait_on_metadata_returns_immediately_for_known_topic() {
5495 let metadata = create_metadata_with_topic(TOPIC, 3);
5496 let accumulator = create_accumulator();
5497 let producer = create_producer(metadata, accumulator);
5498
5499 let now_ms = producer.now_ms();
5500 let result = producer.wait_on_metadata(TOPIC, None, now_ms, 1000).await;
5501 assert!(result.is_ok());
5502 let cwt = result.unwrap();
5503 assert_eq!(0, cwt.waited_on_metadata_ms);
5504 assert_eq!(3, cwt.cluster.partitions_for_topic(TOPIC).len());
5505 }
5506
5507 /// Tests that wait_on_metadata returns immediately when partition is within known range.
5508 #[tokio::test]
5509 async fn test_wait_on_metadata_returns_for_valid_partition() {
5510 let metadata = create_metadata_with_topic(TOPIC, 3);
5511 let accumulator = create_accumulator();
5512 let producer = create_producer(metadata, accumulator);
5513
5514 let now_ms = producer.now_ms();
5515 let result = producer.wait_on_metadata(TOPIC, Some(2), now_ms, 1000).await;
5516 assert!(result.is_ok());
5517 }
5518
5519 /// Tests that wait_on_metadata times out for a topic that does not exist.
5520 ///
5521 /// Translated from `KafkaProducerTest.testTopicNotExistingInMetadata`.
5522 #[tokio::test]
5523 async fn test_wait_on_metadata_times_out_for_unknown_topic() {
5524 let metadata = create_metadata_with_topic(TOPIC, 1);
5525 let accumulator = create_accumulator();
5526 let producer = create_producer(metadata, accumulator);
5527
5528 let now_ms = producer.now_ms();
5529 // Try to get metadata for a topic that doesn't exist with a very short timeout
5530 let result = producer.wait_on_metadata("nonexistent-topic", None, now_ms, 100).await;
5531 assert!(result.is_err());
5532 match result.unwrap_err() {
5533 Error::Timeout(msg) => {
5534 assert!(msg.message().contains("not present in metadata"), "Got: {}", msg);
5535 },
5536 other => panic!("Expected Timeout error, got: {:?}", other),
5537 }
5538 }
5539
5540 /// Tests the client_id accessor.
5541 ///
5542 /// Translated from `KafkaProducerTest.getClientId` (via visible for testing).
5543 #[test]
5544 fn test_get_client_id() {
5545 let config = ProducerConfig { client_id: "my-producer".to_string(), ..ProducerConfig::defaults() };
5546 let metadata = create_metadata_with_topic(TOPIC, 1);
5547 let accumulator = create_accumulator();
5548
5549 let producer = create_producer_with_config(config, metadata, accumulator);
5550
5551 assert_eq!("my-producer", producer.client_id());
5552 }
5553
5554 /// Tests that multiple sends to the same topic produce deterministic partition
5555 /// assignment when keys are provided.
5556 #[tokio::test]
5557 async fn test_send_multiple_records_with_same_key() {
5558 let metadata = create_metadata_with_topic(TOPIC, 3);
5559 let accumulator = create_accumulator();
5560 let producer = create_producer(metadata, Arc::clone(&accumulator));
5561
5562 let record1 =
5563 ProducerRecord::with_key(TOPIC.to_string(), Some("same-key".to_string()), Some("value1".to_string()));
5564 let record2 =
5565 ProducerRecord::with_key(TOPIC.to_string(), Some("same-key".to_string()), Some("value2".to_string()));
5566
5567 let future1 = producer.send(record1).await.unwrap();
5568 let future2 = producer.send(record2).await.unwrap();
5569
5570 // Both should succeed — each send returns a distinct future
5571 assert!(!future1.is_done());
5572 assert!(!future2.is_done());
5573 }
5574
5575 /// Tests that sending with a callback invokes the callback on completion.
5576 ///
5577 /// Translated from `KafkaProducerTest.testCallbackAndInterceptorHandleError`
5578 /// (the callback invocation part).
5579 #[tokio::test]
5580 async fn test_send_callback() {
5581 use std::sync::atomic::AtomicBool;
5582
5583 let metadata = create_metadata_with_topic(TOPIC, 1);
5584 let accumulator = create_accumulator();
5585 let producer = create_producer(metadata, Arc::clone(&accumulator));
5586
5587 let callback_called = Arc::new(AtomicBool::new(false));
5588 let callback_called_clone = Arc::clone(&callback_called);
5589 let callback: Box<dyn Callback> =
5590 Box::new(move |_metadata: Option<&RecordMetadata>, _error: Option<&Error>| {
5591 callback_called_clone.store(true, Ordering::SeqCst);
5592 });
5593
5594 let record = ProducerRecord::with_key(TOPIC.to_string(), Some("key".to_string()), Some("value".to_string()));
5595 let result = producer.send_with_callback(record, Some(callback)).await;
5596 assert!(result.is_ok(), "Send with callback should succeed");
5597 }
5598
5599 /// Tests that sending a record with null (None) key and value works.
5600 ///
5601 /// Translated from `KafkaProducerTest.testNullTopicName` (partial — tests
5602 /// that the producer handles empty/null values).
5603 #[tokio::test]
5604 async fn test_send_with_none_key_and_value() {
5605 let metadata = create_metadata_with_topic(TOPIC, 1);
5606 let accumulator = create_accumulator();
5607 let producer = create_producer(metadata, Arc::clone(&accumulator));
5608
5609 let record: ProducerRecord<String, String> = ProducerRecord::new(TOPIC.to_string(), None);
5610 let result = producer.send(record).await;
5611 assert!(result.is_ok(), "Send with None value should succeed");
5612 }
5613
5614 /// Translated from `KafkaProducerTest.testCallbackAndInterceptorHandleError`.
5615 ///
5616 /// Tests that when sending to a topic that will cause an ApiException
5617 /// (RecordTooLargeException), the callback is invoked with the error and
5618 /// a non-null RecordMetadata with appropriate defaults.
5619 #[tokio::test]
5620 async fn test_callback_invoked_on_api_error() {
5621 let config = ProducerConfig { max_request_size: 10, ..ProducerConfig::defaults() };
5622 let metadata = create_metadata_with_topic(TOPIC, 1);
5623 let accumulator = create_accumulator();
5624 let producer = create_producer_with_config(config, metadata, accumulator);
5625
5626 let callback_invoked = Arc::new(AtomicBool::new(false));
5627 let got_error = Arc::new(std::sync::Mutex::new(false));
5628 let got_metadata = Arc::new(std::sync::Mutex::new(false));
5629 let metadata_topic = Arc::new(std::sync::Mutex::new(String::new()));
5630 let metadata_offset = Arc::new(std::sync::Mutex::new(0i64));
5631
5632 let inv = Arc::clone(&callback_invoked);
5633 let err = Arc::clone(&got_error);
5634 let meta = Arc::clone(&got_metadata);
5635 let mt = Arc::clone(&metadata_topic);
5636 let mo = Arc::clone(&metadata_offset);
5637
5638 let callback: Box<dyn Callback> =
5639 Box::new(move |record_metadata: Option<&RecordMetadata>, error: Option<&Error>| {
5640 inv.store(true, Ordering::SeqCst);
5641 *err.lock().unwrap() = error.is_some();
5642 if let Some(rm) = record_metadata {
5643 *meta.lock().unwrap() = true;
5644 *mt.lock().unwrap() = rm.topic().to_string();
5645 *mo.lock().unwrap() = rm.offset();
5646 }
5647 });
5648
5649 // Send a record that exceeds max_request_size
5650 let large_value = "a".repeat(100);
5651 let record = ProducerRecord::new(TOPIC.to_string(), Some(large_value));
5652 let result = producer.send_with_callback(record, Some(callback)).await;
5653
5654 // send() returns Ok with a failed future (Java's FutureFailure pattern)
5655 assert!(result.is_ok(), "send() should return Ok with a failed future");
5656 let future = result.unwrap();
5657 assert!(future.is_done(), "Failed future should be immediately done");
5658
5659 // Verify callback was invoked with the error
5660 assert!(callback_invoked.load(Ordering::SeqCst), "Callback should have been invoked");
5661 assert!(*got_error.lock().unwrap(), "Callback should receive error");
5662 assert!(*got_metadata.lock().unwrap(), "Callback should receive non-null metadata");
5663 assert_eq!(
5664 *metadata_topic.lock().unwrap(),
5665 TOPIC,
5666 "Callback metadata should have the topic"
5667 );
5668 assert_eq!(
5669 *metadata_offset.lock().unwrap(),
5670 RecordMetadata::INVALID_OFFSET,
5671 "Callback metadata should have INVALID_OFFSET"
5672 );
5673
5674 // Future.get() should return the error
5675 let err = future.get().await.unwrap_err();
5676 assert!(matches!(err, Error::RecordTooLarge(_)));
5677 }
5678
5679 /// Mirrors the `KafkaProducerTest.testCallbackAndInterceptorHandleError`
5680 /// contract for the failure mode that happens *inside*
5681 /// `RecordAccumulator.append` rather than before it: buffer exhaustion /
5682 /// `max.block.ms` expiry.
5683 ///
5684 /// Java's `BufferPool.allocate` throws `BufferExhaustedException` (an
5685 /// `ApiException`) out of `append`, and `doSend`'s
5686 /// `catch (ApiException e)` arm still fires the user callback with the
5687 /// placeholder metadata because it holds its own `callback` reference
5688 /// (`KafkaProducer`). Rust *moves* the callback into
5689 /// `append`, so the callback obligation (CLAUDE.md §7, §11.5) is only met
5690 /// because `AppendFailure` hands it back — this test is the regression guard
5691 /// for that hand-back.
5692 #[tokio::test]
5693 async fn test_callback_invoked_on_buffer_exhaustion() {
5694 const BATCH_SIZE: i32 = 16384;
5695
5696 // `max.block.ms = 0` so the second allocation fails immediately
5697 // instead of waiting for memory that will never be freed.
5698 let config = ProducerConfig { max_block_ms: 0, ..ProducerConfig::defaults() };
5699 // Two partitions so the second record needs a *new* batch (and hence a
5700 // new buffer) instead of appending to the first one.
5701 let metadata = create_metadata_with_topic(TOPIC, 2);
5702 // A pool that fits exactly one batch.
5703 let accumulator = Arc::new(RecordAccumulator::new_for_test(
5704 BATCH_SIZE,
5705 Compression::none().build(),
5706 5,
5707 100,
5708 1000,
5709 120_000,
5710 PartitionerConfig::new(true, 0, false, "").unwrap(),
5711 default_time(),
5712 Arc::new(BufferPool::new_for_test(BATCH_SIZE as i64, BATCH_SIZE as usize)),
5713 None,
5714 ));
5715 let producer = create_producer_with_config(config, metadata, Arc::clone(&accumulator));
5716
5717 // First record drains the whole pool.
5718 let first: ProducerRecord<String, String> =
5719 ProducerRecord::with_partition_key(TOPIC.to_string(), Some(0), None, Some("value".to_string())).unwrap();
5720 producer.send(first).await.expect("the first send should succeed");
5721
5722 // Second record targets a different partition, so it must allocate.
5723 let invoked = Arc::new(std::sync::atomic::AtomicI32::new(0));
5724 let saw_error = Arc::new(std::sync::Mutex::new(None::<String>));
5725 let saw_metadata = Arc::new(std::sync::Mutex::new(None::<(String, i32, i64)>));
5726
5727 let inv = Arc::clone(&invoked);
5728 let err_slot = Arc::clone(&saw_error);
5729 let meta_slot = Arc::clone(&saw_metadata);
5730 let callback: Box<dyn Callback> =
5731 Box::new(move |record_metadata: Option<&RecordMetadata>, exception: Option<&Error>| {
5732 inv.fetch_add(1, Ordering::SeqCst);
5733 *err_slot.lock().unwrap() = exception.map(|e| e.to_string());
5734 *meta_slot.lock().unwrap() =
5735 record_metadata.map(|rm| (rm.topic().to_string(), rm.partition(), rm.offset()));
5736 });
5737
5738 let second: ProducerRecord<String, String> =
5739 ProducerRecord::with_partition_key(TOPIC.to_string(), Some(1), None, Some("value".to_string())).unwrap();
5740 let result = producer.send_with_callback(second, Some(callback)).await;
5741
5742 // Java returns a `FutureFailure`, not a thrown exception, for an
5743 // `ApiException`.
5744 let future = result.expect("an ApiException is reported through the future, not the Result");
5745 assert!(future.is_done(), "the failed future should be immediately done");
5746 let future_error = future.get().await.unwrap_err();
5747 assert!(
5748 matches!(future_error, Error::ProducerBufferExhausted(_)),
5749 "the future should report buffer exhaustion, got {:?}",
5750 future_error
5751 );
5752
5753 // The callback fired exactly once, with BOTH the placeholder metadata
5754 // and the error, exactly as Java's `catch (ApiException e)` arm does.
5755 assert_eq!(1, invoked.load(Ordering::SeqCst), "the callback must fire exactly once");
5756 let message = saw_error.lock().unwrap().clone().expect("the callback must receive the error");
5757 assert!(
5758 message.contains("Failed to allocate"),
5759 "the callback should receive the buffer-exhaustion error, got {}",
5760 message
5761 );
5762 let (topic, partition, offset) = saw_metadata
5763 .lock()
5764 .unwrap()
5765 .clone()
5766 .expect("the callback must receive non-null metadata");
5767 assert_eq!(TOPIC, topic);
5768 assert_eq!(1, partition);
5769 assert_eq!(RecordMetadata::INVALID_OFFSET, offset);
5770 }
5771
5772 /// When the producer chooses the partition, a buffer-exhaustion failure reports the
5773 /// partition `append` chose, not `UNKNOWN_PARTITION`.
5774 ///
5775 /// Java's `append` passes the chosen partition to `AppendCallbacks.setPartition`
5776 /// before it waits for memory, and `doSend` builds the callback's metadata from
5777 /// `appendCallbacks.topicPartition()`. Not a translation of a Java test:
5778 /// `test_callback_invoked_on_buffer_exhaustion` sends to an explicit partition, so it
5779 /// cannot tell the two apart.
5780 #[tokio::test]
5781 async fn test_buffer_exhaustion_reports_the_partition_chosen_by_the_partitioner() {
5782 const BATCH_SIZE: i32 = 16384;
5783
5784 // `max.block.ms = 0` so the second allocation fails immediately.
5785 let config = ProducerConfig { max_block_ms: 0, ..ProducerConfig::defaults() };
5786 // One partition, so the built-in partitioner can only choose partition 0.
5787 let metadata = create_metadata_with_topic(TOPIC, 1);
5788 // A pool that fits exactly one batch.
5789 let accumulator = Arc::new(RecordAccumulator::new_for_test(
5790 BATCH_SIZE,
5791 Compression::none().build(),
5792 5,
5793 100,
5794 1000,
5795 120_000,
5796 PartitionerConfig::new(true, 0, false, "").unwrap(),
5797 default_time(),
5798 Arc::new(BufferPool::new_for_test(BATCH_SIZE as i64, BATCH_SIZE as usize)),
5799 None,
5800 ));
5801 let producer = create_producer_with_config(config, metadata, Arc::clone(&accumulator));
5802
5803 // The first record takes the pool's only batch and leaves no room in it for
5804 // the second, so the second needs a new batch and must allocate.
5805 let first: ProducerRecord<String, String> =
5806 ProducerRecord::with_partition_key(TOPIC.to_string(), None, None, Some("a".repeat(15_000))).unwrap();
5807 producer.send(first).await.expect("the first send should succeed");
5808
5809 let saw_partition = Arc::new(std::sync::Mutex::new(None::<i32>));
5810 let partition_slot = Arc::clone(&saw_partition);
5811 let callback: Box<dyn Callback> =
5812 Box::new(move |record_metadata: Option<&RecordMetadata>, _exception: Option<&Error>| {
5813 *partition_slot.lock().unwrap() = record_metadata.map(|rm| rm.partition());
5814 });
5815 let second: ProducerRecord<String, String> =
5816 ProducerRecord::with_partition_key(TOPIC.to_string(), None, None, Some("b".repeat(2_000))).unwrap();
5817 let future = producer
5818 .send_with_callback(second, Some(callback))
5819 .await
5820 .expect("an ApiException is reported through the future, not the Result");
5821 assert!(
5822 matches!(future.get().await.unwrap_err(), Error::ProducerBufferExhausted(_)),
5823 "the second send must fail with buffer exhaustion"
5824 );
5825
5826 assert_eq!(
5827 Some(0),
5828 *saw_partition.lock().unwrap(),
5829 "the callback must report the partition the partitioner chose"
5830 );
5831 }
5832
5833 /// The mirror image of `test_callback_invoked_on_buffer_exhaustion`: the
5834 /// other failure that happens *inside* `RecordAccumulator.append` must
5835 /// **not** fire the callback.
5836 ///
5837 /// "Producer closed while send in progress" is a **bare** `KafkaException`
5838 /// in Java (`RecordAccumulator`), not an `ApiException`, so
5839 /// `doSend` skips `catch (ApiException e)` (the arm that invokes the
5840 /// callback) and lands in `catch (KafkaException e)`, which records the
5841 /// error, notifies the interceptors and **rethrows**
5842 /// (`KafkaProducer`). Firing the callback here would break
5843 /// the exactly-once obligation as surely as dropping one where Java fires.
5844 ///
5845 /// The accumulator is closed while the producer itself is not, which is
5846 /// exactly the race the Java message names: `ensure_not_closed()` and
5847 /// `wait_on_metadata` both pass and the rejection comes from `append`.
5848 #[tokio::test]
5849 async fn test_callback_not_invoked_when_the_accumulator_is_closed() {
5850 let metadata = create_metadata_with_topic(TOPIC, 1);
5851 let accumulator = create_accumulator();
5852 let producer = create_producer(metadata, Arc::clone(&accumulator));
5853 accumulator.close();
5854
5855 let invoked = Arc::new(std::sync::atomic::AtomicI32::new(0));
5856 let inv = Arc::clone(&invoked);
5857 let callback: Box<dyn Callback> = Box::new(move |_: Option<&RecordMetadata>, _: Option<&Error>| {
5858 inv.fetch_add(1, Ordering::SeqCst);
5859 });
5860
5861 let record: ProducerRecord<String, String> =
5862 ProducerRecord::with_partition_key(TOPIC.to_string(), Some(0), None, Some("value".to_string())).unwrap();
5863 let error = match producer.send_with_callback(record, Some(callback)).await {
5864 Ok(_) => panic!("a bare KafkaException must propagate as Err, not as a failed future"),
5865 Err(e) => e,
5866 };
5867
5868 assert!(
5869 matches!(error, Error::KafkaError(_)),
5870 "the closed-mid-send failure must be a bare KafkaException, not an ApiException — \
5871 `is_api_error()` is what routes it away from the callback, got {:?}",
5872 error
5873 );
5874 assert_eq!("Producer closed while send in progress", error.message());
5875 assert!(!error.is_api_error(), "a bare KafkaException is not an ApiException");
5876 assert!(error.is_kafka_error(), "it is still a KafkaException");
5877 assert_eq!(
5878 0,
5879 invoked.load(Ordering::SeqCst),
5880 "Java's `catch (KafkaException e)` arm rethrows without invoking the callback"
5881 );
5882 }
5883
5884 /// Translated from `KafkaProducerTest.testHeadersSuccess`.
5885 ///
5886 /// Tests that headers added to a ProducerRecord before send() are passed
5887 /// through serialization correctly and stored in the accumulator.
5888 #[tokio::test]
5889 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testHeadersSuccess")]
5890 async fn test_headers_success() {
5891 use crate::common::header::Headers;
5892 use crate::common::header::RecordHeader;
5893
5894 let metadata = create_metadata_with_topic(TOPIC, 1);
5895 let accumulator = create_accumulator();
5896 let producer = create_producer(metadata, Arc::clone(&accumulator));
5897
5898 let mut record: ProducerRecord<String, String> =
5899 ProducerRecord::with_key(TOPIC.to_string(), Some("key".to_string()), Some("value".to_string()));
5900
5901 // Add a header pre-send
5902 record
5903 .headers_mut()
5904 .add_with_header(RecordHeader::new("test".to_string(), Some(b"header-value".to_vec())))
5905 .unwrap();
5906
5907 let result = producer.send(record).await;
5908 assert!(result.is_ok(), "Send with headers should succeed");
5909 assert!(accumulator.has_undrained(), "Accumulator should have batches");
5910 assert_eq!(
5911 accumulator.record_headers_for_test(&TopicPartition::new(TOPIC.to_string(), 0)),
5912 vec![vec![("test".to_string(), Some(b"header-value".to_vec()))]]
5913 );
5914 }
5915
5916 /// A `String` serializer that adds a header named `name`, with the data as
5917 /// its value, and records the keys of the headers it was handed and
5918 /// whether they were writable.
5919 struct HeaderAddingSerializer {
5920 name: &'static str,
5921 seen: Arc<Mutex<Vec<SeenHeaders>>>,
5922 }
5923
5924 /// The header keys a serializer was handed, and whether they were writable.
5925 type SeenHeaders = (Vec<String>, bool);
5926
5927 impl Serializer<String> for HeaderAddingSerializer {
5928 async fn serialize<'a>(
5929 &'a self,
5930 _topic: &'a str,
5931 data: Option<&'a String>,
5932 ) -> Result<Option<Cow<'a, [u8]>>, Error> {
5933 Ok(data.map(|data| Cow::Borrowed(data.as_bytes())))
5934 }
5935
5936 async fn serialize_with_headers<'a, 'h>(
5937 &'a self,
5938 topic: &'a str,
5939 headers: &'h mut RecordHeaders,
5940 data: Option<&'a String>,
5941 ) -> Result<Option<Cow<'a, [u8]>>, Error>
5942 where
5943 'a: 'h,
5944 {
5945 use crate::common::header::Header;
5946 let keys = headers.iter().map(|header| header.key().to_string()).collect();
5947 self.seen.lock().unwrap().push((keys, !headers.is_read_only()));
5948 headers
5949 .add_with_key_value(self.name, data.map(|data| data.as_bytes()))
5950 .map_err(Error::LocalIllegalState)?;
5951 Serializer::serialize(self, topic, data).await
5952 }
5953 }
5954
5955 /// A serializer may add headers (Java's `Serializer.serialize(String,
5956 /// Headers, T)` javadoc): `doSend` hands the record's headers to the key
5957 /// serializer, then the same headers to the value serializer
5958 /// (`KafkaProducer#doSend`), and sends them as they are after
5959 /// both (`KafkaProducer`). The value serializer sees the key serializer's
5960 /// header, and both added headers reach the batch, after the record's own.
5961 #[tokio::test]
5962 async fn test_serializer_added_headers_are_sent() {
5963 use crate::common::header::RecordHeader;
5964
5965 let seen = Arc::new(Mutex::new(Vec::new()));
5966 let accumulator = create_accumulator();
5967 let producer: KafkaProducer<String, String> = KafkaProducer::with_options(
5968 KafkaProducerOptionsBuilder::new()
5969 .set_config(&ProducerConfig::defaults())
5970 .set_key_serializer(Box::new(HeaderAddingSerializer { name: "key-header", seen: Arc::clone(&seen) }))
5971 .set_value_serializer(Box::new(HeaderAddingSerializer {
5972 name: "value-header",
5973 seen: Arc::clone(&seen),
5974 }))
5975 .set_metadata(create_metadata_with_topic(TOPIC, 1))
5976 .set_accumulator(Arc::clone(&accumulator))
5977 .set_time(default_time())
5978 .set_pending_requests(Arc::new(Mutex::new(PendingRequests::new())))
5979 .build()
5980 .expect("KafkaProducerOptionsBuilder::build: every mandatory parameter is set above"),
5981 );
5982
5983 let mut record: ProducerRecord<String, String> =
5984 ProducerRecord::with_key(TOPIC.to_string(), Some("key".to_string()), Some("value".to_string()));
5985 record
5986 .headers_mut()
5987 .add_with_header(RecordHeader::new("test".to_string(), Some(b"header-value".to_vec())))
5988 .unwrap();
5989
5990 producer.send(record).await.expect("the send should be accepted");
5991 assert_eq!(
5992 *seen.lock().unwrap(),
5993 [
5994 (vec!["test".to_string()], true),
5995 (vec!["test".to_string(), "key-header".to_string()], true),
5996 ],
5997 "the key serializer runs first; both get the record's writable headers"
5998 );
5999 assert_eq!(
6000 accumulator.record_headers_for_test(&TopicPartition::new(TOPIC.to_string(), 0)),
6001 vec![vec![
6002 ("test".to_string(), Some(b"header-value".to_vec())),
6003 ("key-header".to_string(), Some(b"key".to_vec())),
6004 ("value-header".to_string(), Some(b"value".to_vec())),
6005 ]]
6006 );
6007 }
6008
6009 /// `KafkaProducer.setReadOnly`, which
6010 /// `doSend` runs after both serializers (`KafkaProducer#doSend`): a header added after it
6011 /// fails with `RecordHeaders`' read-only message, as Java's
6012 /// `testHeadersSuccess` asserts after `send` (`KafkaProducerTest`),
6013 /// and the headers already there are kept.
6014 #[test]
6015 fn test_headers_are_read_only_after_serialization() {
6016 use crate::common::header::Header;
6017
6018 let mut headers = RecordHeaders::new();
6019 headers.add_with_key_value("test", Some(b"header2")).unwrap();
6020 KafkaProducer::<String, String>::set_read_only(&mut headers);
6021
6022 assert!(headers.is_read_only());
6023 let error = headers.add_with_key_value("test", Some(b"test")).unwrap_err();
6024 assert_eq!("RecordHeaders has been closed.", Error::LocalIllegalState(error).message());
6025 assert_eq!(
6026 Some(&b"header2"[..]),
6027 headers.last_header("test").and_then(|header| header.value())
6028 );
6029 }
6030
6031 /// The borrowed byte-slice `send`, the path of the C FFI's `send_batch` and
6032 /// single-record sends (so of the C and Python `KafkaProducer`), appends the
6033 /// record's headers as Java's `doSend` does (`KafkaProducer#doSend`,
6034 /// `KafkaProducer#doSend`), in order, with an empty and a null value kept apart. It
6035 /// used to append none.
6036 #[tokio::test]
6037 async fn test_borrowed_send_appends_the_record_headers() {
6038 use crate::common::header::Headers;
6039 use crate::common::header::RecordHeaders;
6040 use crate::common::serialization::ByteArraySerializer;
6041
6042 let metadata = create_metadata_with_topic(TOPIC, 1);
6043 let accumulator = create_accumulator();
6044 let producer: KafkaProducer<Vec<u8>, Vec<u8>> = KafkaProducer::with_options(
6045 KafkaProducerOptionsBuilder::new()
6046 .set_config(&ProducerConfig::defaults())
6047 .set_key_serializer(Box::new(ByteArraySerializer))
6048 .set_value_serializer(Box::new(ByteArraySerializer))
6049 .set_metadata(metadata)
6050 .set_accumulator(Arc::clone(&accumulator))
6051 .set_time(default_time())
6052 .set_pending_requests(Arc::new(Mutex::new(PendingRequests::new())))
6053 .build()
6054 .expect("KafkaProducerOptionsBuilder::build: every mandatory parameter is set above"),
6055 );
6056
6057 let mut headers = RecordHeaders::new();
6058 headers.add_with_key_value("trace", Some(b"abc")).unwrap();
6059 headers.add_with_key_value("empty", Some(b"")).unwrap();
6060 headers.add_with_key_value("null", None).unwrap();
6061 let record = ProducerRecord::with_partition_key_headers(
6062 TOPIC.to_string(),
6063 Some(0),
6064 Some(b"key".as_slice()),
6065 Some(b"value".as_slice()),
6066 headers,
6067 )
6068 .unwrap();
6069
6070 producer
6071 .send(record, None)
6072 .await
6073 .expect("the borrowed send should append the record");
6074 assert_eq!(
6075 accumulator.record_headers_for_test(&TopicPartition::new(TOPIC.to_string(), 0)),
6076 vec![vec![
6077 ("trace".to_string(), Some(b"abc".to_vec())),
6078 ("empty".to_string(), Some(Vec::new())),
6079 ("null".to_string(), None),
6080 ]]
6081 );
6082 }
6083
6084 /// Translated from `KafkaProducerTest.testFlushCompleteSendOfInflightBatches`.
6085 ///
6086 /// Tests that flush waits for all in-flight batches to complete. We send
6087 /// records, then simulate the sender completing them by aborting incomplete
6088 /// batches (which calls `done()` on all `ProduceRequestResult`s). Flush
6089 /// should then return immediately because all results are satisfied.
6090 #[tokio::test(flavor = "multi_thread")]
6091 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testFlushCompleteSendOfInflightBatches")]
6092 async fn test_flush_complete_send_of_inflight_batches() {
6093 let metadata = create_metadata_with_topic(TOPIC, 1);
6094 let accumulator = create_accumulator();
6095 let producer = create_producer(metadata, Arc::clone(&accumulator));
6096
6097 // Send multiple records
6098 let mut futures = Vec::new();
6099 for i in 0..5 {
6100 let record = ProducerRecord::new(TOPIC.to_string(), Some(format!("value{}", i)));
6101 let future = producer.send(record).await.unwrap();
6102 futures.push(future);
6103 }
6104
6105 // None should be done yet (no sender to complete them)
6106 for f in &futures {
6107 assert!(!f.is_done(), "Future should not be done before sender completes it");
6108 }
6109
6110 // Simulate the sender completing all batches by aborting them.
6111 // abort() calls complete_future_and_fire_callbacks which calls
6112 // produce_future.set() and produce_future.done(), unblocking flush.
6113 accumulator.abort_incomplete_batches().await;
6114
6115 // Now all futures should be done (abort marks ProduceRequestResults as done)
6116 for f in &futures {
6117 assert!(f.is_done(), "Future should be done after batch abort");
6118 }
6119
6120 // Flush should return immediately since all results are satisfied
6121 let result = producer.flush().await;
6122 assert!(result.is_ok(), "Flush should succeed after batches are completed");
6123 }
6124
6125 /// Translated from `KafkaProducerTest.testCloseWhenWaitingForMetadataUpdate`.
6126 ///
6127 /// Tests that closing the producer unblocks a send() that is waiting for
6128 /// metadata by verifying that after close(), the producer rejects new sends
6129 /// with IllegalState.
6130 #[tokio::test]
6131 async fn test_close_unblocks_pending_operations() {
6132 let metadata = create_metadata_with_topic(TOPIC, 1);
6133 let accumulator = create_accumulator();
6134 let producer = create_producer(metadata, accumulator);
6135
6136 // Close the producer
6137 producer.close().await.unwrap();
6138
6139 // Subsequent send should fail with IllegalState
6140 let record = ProducerRecord::new(TOPIC.to_string(), Some("value".to_string()));
6141 let result = producer.send(record).await;
6142 assert!(result.is_err());
6143 assert!(
6144 matches!(result.unwrap_err(), Error::LocalIllegalState(_)),
6145 "Expected IllegalState error after close"
6146 );
6147 }
6148
6149 /// Tests that initiate_close calls accumulator.close(), preventing new appends.
6150 ///
6151 /// Verifies Issue 2 fix: initiate_close must close the accumulator before
6152 /// setting running=false.
6153 #[tokio::test]
6154 async fn test_initiate_close_closes_accumulator() {
6155 let metadata = create_metadata_with_topic(TOPIC, 1);
6156 let accumulator = create_accumulator();
6157 let producer = create_producer(metadata, Arc::clone(&accumulator));
6158
6159 // Before close, we can send
6160 let record = ProducerRecord::new(TOPIC.to_string(), Some("value".to_string()));
6161 assert!(producer.send(record).await.is_ok());
6162
6163 // Initiate close
6164 producer.initiate_close();
6165
6166 // After initiate_close, the accumulator should be closed.
6167 // Attempting to send should fail because the accumulator rejects new appends.
6168 let record2 = ProducerRecord::new(TOPIC.to_string(), Some("value2".to_string()));
6169 // The error will come from ensure_not_closed since running=false
6170 let result = producer.send(record2).await;
6171 assert!(result.is_err(), "Send should fail after initiate_close");
6172 }
6173
6174 /// Tests that the negative timeout check is gone since Rust Duration is unsigned.
6175 ///
6176 /// Documents Issue 7 resolution: Rust's Duration cannot be negative,
6177 /// so the negative timeout check from Java is correctly omitted.
6178 #[test]
6179 fn test_duration_cannot_be_negative() {
6180 // This test documents that Rust's std::time::Duration cannot represent
6181 // negative values, so the Java test `closeWithNegativeTimestampShouldThrow`
6182 // is not applicable. Duration::ZERO is the smallest possible value.
6183 let zero = Duration::ZERO;
6184 assert_eq!(0, zero.as_millis());
6185 // There is no way to construct a negative Duration in Rust.
6186 // Duration::from_millis(u64) always produces a non-negative value.
6187 }
6188
6189 /// Tests that close with timeout 0 force-closes (no graceful drain).
6190 ///
6191 /// Verifies the force-close path in close_with_timeout when timeout is zero.
6192 #[tokio::test]
6193 async fn test_close_timeout_zero_force_closes() {
6194 let metadata = create_metadata_with_topic(TOPIC, 1);
6195 let accumulator = create_accumulator();
6196 let producer = create_producer(metadata, Arc::clone(&accumulator));
6197
6198 // Send a record
6199 let record = ProducerRecord::new(TOPIC.to_string(), Some("value".to_string()));
6200 let _ = producer.send(record).await;
6201 assert!(accumulator.has_undrained());
6202
6203 // Close with timeout 0 should force-close
6204 let result = producer.close_with_timeout(Duration::ZERO).await;
6205 assert!(result.is_ok());
6206
6207 // After force-close, the producer should not accept new sends
6208 let record2 = ProducerRecord::new(TOPIC.to_string(), Some("value2".to_string()));
6209 assert!(producer.send(record2).await.is_err());
6210 }
6211
6212 /// Regression test with no Java counterpart: a timeout whose length in
6213 /// milliseconds exceeds `i64::MAX` must result in a graceful close, as `close()`
6214 /// does.
6215 ///
6216 /// Truncating these values with `as i64` yields -1 ms for `Duration::MAX` and
6217 /// `Duration::from_millis(u64::MAX)`, -1000 ms for `Duration::from_secs(u64::MAX)`,
6218 /// and 0 ms for `Duration::from_secs(1 << 61)` (125 × 2^64 ms). A negative value
6219 /// matches neither branch of `close_with_timeout` and leaves the producer running;
6220 /// 0 forces the close instead of waiting. Java cannot reach either state:
6221 /// `Duration.toMillis()` throws on overflow and `close` rejects a negative timeout
6222 /// (`KafkaProducer#close(Duration)`).
6223 #[tokio::test]
6224 async fn test_close_with_timeout_beyond_i64_millis_closes_gracefully() {
6225 for timeout in [
6226 Duration::MAX,
6227 Duration::from_millis(u64::MAX),
6228 Duration::from_secs(u64::MAX),
6229 Duration::from_secs(1 << 61),
6230 ] {
6231 let metadata = create_metadata_with_topic(TOPIC, 1);
6232 let producer = create_producer(metadata, create_accumulator());
6233
6234 producer.close_with_timeout(timeout).await.expect("close");
6235
6236 assert!(
6237 !producer.running.load(Ordering::Acquire),
6238 "{:?}: close must stop the producer",
6239 timeout
6240 );
6241 assert!(
6242 !producer.force_close.load(Ordering::Acquire),
6243 "{:?}: close must be graceful, not forced",
6244 timeout
6245 );
6246 let record = ProducerRecord::new(TOPIC.to_string(), Some("value".to_string()));
6247 let error = producer.send(record).await.expect_err("a closed producer rejects sends");
6248 assert_eq!(
6249 error.message(),
6250 "Cannot perform operation after producer has been closed",
6251 "{:?}",
6252 timeout
6253 );
6254 }
6255 }
6256
6257 /// Tests that the callback is invoked with error on invalid topic.
6258 ///
6259 /// Translated from `KafkaProducerTest.testCallbackAndInterceptorHandleError`
6260 /// (the invalid topic variant).
6261 #[tokio::test]
6262 async fn test_callback_invoked_on_invalid_topic() {
6263 use crate::MetadataResponseData;
6264 use crate::common::protocol::ApiKeys;
6265 use crate::common::protocol::Errors;
6266 use crate::common::requests::MetadataResponse;
6267 use crate::metadata_response_data::{MetadataResponseBroker, MetadataResponseTopic};
6268
6269 let metadata = Arc::new(ProducerMetadata::new(
6270 100,
6271 1000,
6272 300_000,
6273 300_000,
6274 ClusterResourceListeners::new(),
6275 ));
6276
6277 // Create metadata with an invalid topic
6278 let invalid_topic = "topic with spaces";
6279 let mut data = MetadataResponseData::new();
6280 data.set_controller_id(0);
6281
6282 let mut broker = MetadataResponseBroker::new();
6283 broker.set_node_id(0);
6284 broker.set_host("localhost".to_string());
6285 broker.set_port(9092);
6286 data.set_brokers(vec![broker]);
6287
6288 let mut topic_resp = MetadataResponseTopic::new();
6289 topic_resp.set_name(Some(invalid_topic.to_string()));
6290 topic_resp.set_error_code(Errors::InvalidTopicError.code());
6291 topic_resp.set_is_internal(false);
6292 data.set_topics(vec![topic_resp]);
6293
6294 let response = MetadataResponse::with_version(data, ApiKeys::METADATA.latest_version());
6295 metadata.add(invalid_topic, 0);
6296 metadata.update_with_current_request_version(&response, false, 0);
6297
6298 let accumulator = create_accumulator();
6299 let producer = create_producer(metadata, accumulator);
6300
6301 let callback_invoked = Arc::new(AtomicBool::new(false));
6302 let got_error = Arc::new(AtomicBool::new(false));
6303 let inv = Arc::clone(&callback_invoked);
6304 let err = Arc::clone(&got_error);
6305
6306 let callback: Box<dyn Callback> = Box::new(move |_metadata: Option<&RecordMetadata>, error: Option<&Error>| {
6307 inv.store(true, Ordering::SeqCst);
6308 err.store(error.is_some(), Ordering::SeqCst);
6309 });
6310
6311 let record = ProducerRecord::new(invalid_topic.to_string(), Some("value".to_string()));
6312 let result = producer.send_with_callback(record, Some(callback)).await;
6313
6314 // Should return a failed future, not propagate the error
6315 assert!(result.is_ok(), "send() should return Ok with a failed future for InvalidTopic");
6316 let future = result.unwrap();
6317 assert!(future.is_done());
6318
6319 // Verify callback was invoked with the error
6320 assert!(callback_invoked.load(Ordering::SeqCst), "Callback should have been invoked");
6321 assert!(got_error.load(Ordering::SeqCst), "Callback should receive error");
6322
6323 // Verify the future contains the error
6324 let err = future.get().await.unwrap_err();
6325 assert!(
6326 matches!(err, Error::InvalidTopic(_)),
6327 "Expected InvalidTopic error, got: {:?}",
6328 err
6329 );
6330 // Java's `new InvalidTopicException(topic)` sets the message to the topic
6331 // name and leaves `invalidTopics()` empty (the `(String message)` ctor).
6332 assert_eq!(err.message(), "topic with spaces");
6333 match &err {
6334 Error::InvalidTopic(e) => assert!(e.invalid_topics().is_empty()),
6335 _ => unreachable!(),
6336 }
6337 }
6338
6339 /// Tests that multiple calls to close are safe even with timeout.
6340 ///
6341 /// Translated from `KafkaProducerTest.closeShouldBeIdempotent`.
6342 #[tokio::test]
6343 async fn test_close_with_timeout_idempotent() {
6344 let metadata = create_metadata_with_topic(TOPIC, 1);
6345 let accumulator = create_accumulator();
6346 let producer = create_producer(metadata, accumulator);
6347
6348 producer.close_with_timeout(Duration::from_secs(1)).await.unwrap();
6349 producer.close_with_timeout(Duration::from_secs(1)).await.unwrap();
6350 producer.close_with_timeout(Duration::ZERO).await.unwrap();
6351 }
6352
6353 /// Tests that different keys produce different partition assignments.
6354 ///
6355 /// Translated from partition-related KafkaProducerTest tests.
6356 #[test]
6357 fn test_different_keys_may_produce_different_partitions() {
6358 let metadata = create_metadata_with_topic(TOPIC, 100);
6359 let accumulator = create_accumulator();
6360 let producer = create_producer(metadata.clone(), accumulator);
6361
6362 let cluster = metadata.fetch();
6363 let mut partitions = std::collections::HashSet::new();
6364 for i in 0..20 {
6365 let key = format!("key-{}", i);
6366 let record = ProducerRecord::with_key(TOPIC.to_string(), Some(key.clone()), Some("v".to_string()));
6367 let p = producer.partition(&record, Some(key.as_bytes()), Some(b"v"), &cluster);
6368 partitions.insert(p);
6369 }
6370 // With 100 partitions and 20 different keys, we should get at least 2 different partitions
6371 assert!(
6372 partitions.len() >= 2,
6373 "Expected multiple partitions for different keys, got: {:?}",
6374 partitions
6375 );
6376 }
6377
6378 /// `definition-of-done.md` §10 / CLAUDE.md §13: enabling idempotence must not add
6379 /// a single per-record heap allocation to the send path.
6380 ///
6381 /// This is the audit clause aimed at the public `send` entry point.
6382 /// `RecordAccumulator::drain` has its own budget test
6383 /// (`test_drain_allocations_do_not_scale_with_the_record_count`), but that
6384 /// measures the wrong layer: Critic 44 issue 1 was a `String` + `Arc<str>`
6385 /// allocation per record in `KafkaProducer::do_send_bytes`, one call *above* it.
6386 ///
6387 /// Measured as a delta between a producer that holds a `TransactionManager` and
6388 /// one that does not, over a steady-state append (batch already created, topic
6389 /// info already interned). Pinning an absolute count would break on unrelated
6390 /// refactors; the delta is exactly the cost the transaction wiring adds, and it
6391 /// must be zero.
6392 #[tokio::test]
6393 async fn test_send_allocations_do_not_grow_when_idempotence_is_enabled() {
6394 async fn steady_state_send_allocations(with_transaction_manager: bool) -> usize {
6395 let metadata = create_metadata_with_topic(TOPIC, 1);
6396 let transaction_manager = if with_transaction_manager {
6397 let manager = TransactionManager::new(
6398 LogContext::empty(),
6399 None,
6400 60_000,
6401 100,
6402 Arc::new(ApiVersions::new()),
6403 metadata.metadata_arc(),
6404 false,
6405 );
6406 Some(Arc::new(Mutex::new(manager)))
6407 } else {
6408 None
6409 };
6410 // A large batch so every send below appends to the same batch.
6411 let accumulator = Arc::new(RecordAccumulator::new_for_test(
6412 1024 * 1024,
6413 Compression::none().build(),
6414 5,
6415 100,
6416 1000,
6417 120_000,
6418 PartitionerConfig::new(true, 0, false, "").unwrap(),
6419 default_time(),
6420 Arc::new(BufferPool::new_for_test(32 * 1024 * 1024, 1024 * 1024)),
6421 transaction_manager.clone(),
6422 ));
6423 let config = ProducerConfig::defaults();
6424 let producer = KafkaProducer::<String, String>::with_options(
6425 KafkaProducerOptionsBuilder::new()
6426 .set_config(&config)
6427 .set_key_serializer(Box::new(StringSerializer))
6428 .set_value_serializer(Box::new(StringSerializer))
6429 .set_metadata(Arc::clone(&metadata))
6430 .set_accumulator(accumulator)
6431 .set_time(default_time())
6432 .set_transaction_manager(transaction_manager)
6433 .set_pending_requests(Arc::new(Mutex::new(PendingRequests::new())))
6434 .build()
6435 .expect("KafkaProducerOptionsBuilder::build: every mandatory parameter is set above"),
6436 );
6437 let cluster = metadata.fetch();
6438
6439 // Warm up: create the topic info, the deque and the batch.
6440 for _ in 0..4 {
6441 producer
6442 .do_send_bytes(
6443 TOPIC,
6444 Some(0),
6445 Some(0),
6446 Some(b"k"),
6447 Some(b"v"),
6448 &[],
6449 None,
6450 0,
6451 0,
6452 &cluster,
6453 SendWait::MaxBlock,
6454 )
6455 .await
6456 .expect("append should succeed");
6457 }
6458
6459 {
6460 let _guard = crate::AllocTrackingGuard::new();
6461 crate::AllocTrackingGuard::reset();
6462 producer
6463 .do_send_bytes(
6464 TOPIC,
6465 Some(0),
6466 Some(0),
6467 Some(b"k"),
6468 Some(b"v"),
6469 &[],
6470 None,
6471 0,
6472 0,
6473 &cluster,
6474 SendWait::MaxBlock,
6475 )
6476 .await
6477 .expect("append should succeed");
6478 let count = crate::AllocTrackingGuard::count();
6479 assert!(count > 0, "the tracker must actually be measuring");
6480 count
6481 }
6482 }
6483
6484 let without = steady_state_send_allocations(false).await;
6485 let with = steady_state_send_allocations(true).await;
6486 assert_eq!(
6487 without, with,
6488 "enabling idempotence must add no per-record allocation to the send path; \
6489 got {without} without a TransactionManager vs {with} with one"
6490 );
6491 }
6492
6493 /// `definition-of-done.md` §10 / CLAUDE.md §13: dispatching through a custom
6494 /// `Partitioner<K, V>` must not add a single per-record heap allocation to the
6495 /// send path, matching the idempotence audit above
6496 /// (`test_send_allocations_do_not_grow_when_idempotence_is_enabled`).
6497 ///
6498 /// `partition: None` is passed to `do_send_bytes` (rather than the pre-resolved
6499 /// `Some(0)` the idempotence test uses) so `compute_partition` actually runs and,
6500 /// when a partitioner is configured, calls into it — otherwise this would measure
6501 /// nothing, since a pre-resolved partition short-circuits `compute_partition`
6502 /// before the partitioner is ever consulted.
6503 ///
6504 /// `RoundRobinPartitioner` is the concrete partitioner under measurement: its
6505 /// steady-state path is a `DashMap::get` on an already-present topic entry plus an
6506 /// `AtomicI32` increment (CLAUDE.md §13), so the delta versus no partitioner
6507 /// (built-in key-hash partitioning) must be zero.
6508 #[tokio::test]
6509 async fn test_send_allocations_do_not_grow_with_a_custom_partitioner() {
6510 async fn steady_state_send_allocations(with_partitioner: bool) -> usize {
6511 let partitioner: Option<Box<dyn Partitioner<String, String>>> = if with_partitioner {
6512 Some(Box::new(RoundRobinPartitioner::new()))
6513 } else {
6514 None
6515 };
6516 let metadata = create_metadata_with_topic(TOPIC, 1);
6517 // A large batch so every send below appends to the same batch.
6518 let accumulator = Arc::new(RecordAccumulator::new_for_test(
6519 1024 * 1024,
6520 Compression::none().build(),
6521 5,
6522 100,
6523 1000,
6524 120_000,
6525 PartitionerConfig::new(true, 0, false, "").unwrap(),
6526 default_time(),
6527 Arc::new(BufferPool::new_for_test(32 * 1024 * 1024, 1024 * 1024)),
6528 None,
6529 ));
6530 let config = ProducerConfig::defaults();
6531 let producer = KafkaProducer::<String, String>::with_options(
6532 KafkaProducerOptionsBuilder::new()
6533 .set_config(&config)
6534 .set_key_serializer(Box::new(StringSerializer))
6535 .set_value_serializer(Box::new(StringSerializer))
6536 .set_metadata(Arc::clone(&metadata))
6537 .set_accumulator(accumulator)
6538 .set_time(default_time())
6539 .set_pending_requests(Arc::new(Mutex::new(PendingRequests::new())))
6540 .set_partitioner(partitioner)
6541 .build()
6542 .expect("KafkaProducerOptionsBuilder::build: every mandatory parameter is set above"),
6543 );
6544 let cluster = metadata.fetch();
6545
6546 // Warm up: create the topic info, the deque and the batch, and (for the
6547 // partitioner case) the RoundRobinPartitioner's per-topic counter entry.
6548 for _ in 0..4 {
6549 producer
6550 .do_send_bytes(
6551 TOPIC,
6552 None,
6553 Some(0),
6554 Some(b"k"),
6555 Some(b"v"),
6556 &[],
6557 None,
6558 0,
6559 0,
6560 &cluster,
6561 SendWait::MaxBlock,
6562 )
6563 .await
6564 .expect("append should succeed");
6565 }
6566
6567 {
6568 let _guard = crate::AllocTrackingGuard::new();
6569 crate::AllocTrackingGuard::reset();
6570 producer
6571 .do_send_bytes(
6572 TOPIC,
6573 None,
6574 Some(0),
6575 Some(b"k"),
6576 Some(b"v"),
6577 &[],
6578 None,
6579 0,
6580 0,
6581 &cluster,
6582 SendWait::MaxBlock,
6583 )
6584 .await
6585 .expect("append should succeed");
6586 let count = crate::AllocTrackingGuard::count();
6587 assert!(count > 0, "the tracker must actually be measuring");
6588 count
6589 }
6590 }
6591
6592 let without = steady_state_send_allocations(false).await;
6593 let with = steady_state_send_allocations(true).await;
6594 assert_eq!(
6595 without, with,
6596 "dispatching through a custom Partitioner must add no per-record allocation \
6597 to the send path; got {without} without a partitioner vs {with} with one"
6598 );
6599 }
6600
6601 /// CLAUDE.md §13 / §14: with a custom partitioner, a `Vec<u8>` record goes
6602 /// through the serializers (the partitioner needs the typed key and value)
6603 /// and still allocates nothing for them: `ByteArraySerializer` borrows the
6604 /// record's own buffers, and its future fits the `InlineFuture` buffer. So
6605 /// the whole `send` costs exactly what appending the already-serialized
6606 /// bytes costs.
6607 #[tokio::test]
6608 async fn test_vec_send_with_a_custom_partitioner_allocates_nothing_for_the_serializers() {
6609 use crate::common::serialization::ByteArraySerializer;
6610
6611 let metadata = create_metadata_with_topic(TOPIC, 1);
6612 let accumulator = Arc::new(RecordAccumulator::new_for_test(
6613 1024 * 1024,
6614 Compression::none().build(),
6615 5,
6616 100,
6617 1000,
6618 120_000,
6619 PartitionerConfig::new(true, 0, false, "").unwrap(),
6620 default_time(),
6621 Arc::new(BufferPool::new_for_test(32 * 1024 * 1024, 1024 * 1024)),
6622 None,
6623 ));
6624 let config = ProducerConfig::defaults();
6625 let partitioner: Option<Box<dyn Partitioner<Vec<u8>, Vec<u8>>>> = Some(Box::new(RoundRobinPartitioner::new()));
6626 let producer = KafkaProducer::<Vec<u8>, Vec<u8>>::with_options(
6627 KafkaProducerOptionsBuilder::new()
6628 .set_config(&config)
6629 .set_key_serializer(Box::new(ByteArraySerializer))
6630 .set_value_serializer(Box::new(ByteArraySerializer))
6631 .set_metadata(Arc::clone(&metadata))
6632 .set_accumulator(accumulator)
6633 .set_time(default_time())
6634 .set_pending_requests(Arc::new(Mutex::new(PendingRequests::new())))
6635 .set_partitioner(partitioner)
6636 .build()
6637 .expect("KafkaProducerOptionsBuilder::build: every mandatory parameter is set above"),
6638 );
6639 let cluster = metadata.fetch();
6640 let record = || ProducerRecord::with_key(TOPIC.to_string(), Some(b"k".to_vec()), Some(b"v".to_vec()));
6641
6642 let send_bytes = || {
6643 producer.do_send_bytes(
6644 TOPIC,
6645 None,
6646 None,
6647 Some(b"k"),
6648 Some(b"v"),
6649 &[],
6650 None,
6651 0,
6652 0,
6653 &cluster,
6654 SendWait::MaxBlock,
6655 )
6656 };
6657
6658 // Warm up both paths: the topic info, the deque, the batch and the
6659 // partitioner's per-topic counter. Ten records, so the two measured
6660 // below (the 11th and 12th) do not land on a doubling of the batch's
6661 // per-record vectors, which would charge an amortized growth to one
6662 // side only.
6663 for _ in 0..5 {
6664 producer
6665 .do_send(record(), None, SendWait::MaxBlock)
6666 .await
6667 .expect("append should succeed");
6668 send_bytes().await.expect("append should succeed");
6669 }
6670
6671 let through_serializers = {
6672 let record = record();
6673 let _guard = crate::AllocTrackingGuard::new();
6674 crate::AllocTrackingGuard::reset();
6675 producer
6676 .do_send(record, None, SendWait::MaxBlock)
6677 .await
6678 .expect("append should succeed");
6679 crate::AllocTrackingGuard::count()
6680 };
6681 let bytes_only = {
6682 let _guard = crate::AllocTrackingGuard::new();
6683 crate::AllocTrackingGuard::reset();
6684 send_bytes().await.expect("append should succeed");
6685 crate::AllocTrackingGuard::count()
6686 };
6687 // `do_send` waits on the metadata before serializing, as Java's
6688 // `doSend` does; that step's own cost is not the serializers'.
6689 let metadata_wait = {
6690 let _guard = crate::AllocTrackingGuard::new();
6691 crate::AllocTrackingGuard::reset();
6692 producer
6693 .wait_on_metadata(TOPIC, None, 0, 0)
6694 .await
6695 .expect("the metadata is cached");
6696 crate::AllocTrackingGuard::count()
6697 };
6698 assert!(bytes_only > 0, "the tracker must actually be measuring");
6699 assert_eq!(
6700 bytes_only + metadata_wait,
6701 through_serializers,
6702 "serializing a Vec<u8> record must add no allocation to the send path; got \
6703 {through_serializers} through the serializers vs {bytes_only} for the bytes alone \
6704 plus {metadata_wait} for the metadata wait"
6705 );
6706 }
6707
6708 // =====================================================================
6709 // `KafkaProducerTest` transactional harness (Milestone 11, Phase 6)
6710 //
6711 // Java's `kafkaProducer(configs, keySer, valSer, metadata, client, interceptors,
6712 // time)` helper (`KafkaProducerTest#kafkaProducer`) builds a real `KafkaProducer`
6713 // over a `MockClient` and lets the constructor start a real Sender **thread**;
6714 // the test thread then pokes the `MockClient` beside it, relying on Java's
6715 // `MockClient` being internally synchronized.
6716 //
6717 // Rust cannot copy that directly: `Sender` owns its `C: KafkaClient` by value and
6718 // `MockClient` is a plain struct, so a `tokio::spawn`ed Sender would take the mock
6719 // with it and the test could no longer prepare responses or inspect requests. So
6720 // the `Sender` stays test-owned — exactly as `SenderTest` keeps it — and the
6721 // application call runs *concurrently with* a driver loop on the same task (see
6722 // [`drive`]). Everything the two share is shared the way production shares it: one
6723 // `TransactionManager`, one `PendingRequests`, one `RecordAccumulator`, one
6724 // `ProducerMetadata`, one `running` / `force_close` flag pair.
6725 // =====================================================================
6726
6727 /// Java's `NODE`, used as the coordinator in every
6728 /// `FindCoordinator` response below.
6729 fn coordinator_node() -> Node {
6730 Node::new(0, "host1".to_string(), 1000)
6731 }
6732
6733 /// The topic id [`TOPIC`] is published with, so a produce response can name the
6734 /// same one the request carried.
6735 fn topic_id() -> crate::common::Uuid {
6736 crate::common::Uuid::from_string("MKXx1fIkQy2J9jXHhK8m1w").expect("valid UUID")
6737 }
6738
6739 /// Publishes node `"0"`'s API versions with `transaction.version` finalized at
6740 /// `level`, which is what `maybeUpdateTransactionV2Enabled`
6741 /// (`TransactionManager#maybeUpdateTransactionV2Enabled`) reads.
6742 fn seed_transaction_version(api_versions: &Arc<ApiVersions>, level: i16) {
6743 use crate::NodeApiVersions;
6744 use crate::api_versions_response_data::{FinalizedFeatureKey, SupportedFeatureKey};
6745
6746 const FEATURE: &str = "transaction.version";
6747
6748 let mut supported = SupportedFeatureKey::new();
6749 supported.set_name(FEATURE.to_string());
6750 supported.set_max_version(level);
6751 supported.set_min_version(0);
6752
6753 let mut finalized = FinalizedFeatureKey::new();
6754 finalized.set_name(FEATURE.to_string());
6755 finalized.set_max_version_level(level);
6756 finalized.set_min_version_level(level);
6757
6758 api_versions.update(
6759 "0",
6760 NodeApiVersions::with_node_finalized_features_finalized_features_epoch(&[], &[supported], &[finalized], 0),
6761 );
6762 }
6763
6764 /// Java's `new MockTime()` started at `initial_ms`, with a frozen monotonic clock.
6765 fn mock_time(initial_ms: i64) -> Arc<MockTime> {
6766 Arc::new(MockTime::with_auto_tick_ms_current_time_ms_current_high_res_time_ns(
6767 0, initial_ms, 0,
6768 ))
6769 }
6770
6771 /// `RequestTestUtils.metadataUpdateWithIds(1, Map.of(topic, n), Map.of(topic, id))`,
6772 /// or `metadataUpdateWith(1, Map.of(topic, n))` (no ids) when `with_topic_ids` is
6773 /// `false`.
6774 fn topic_metadata_update(num_partitions: i32, with_topic_ids: bool) -> crate::common::requests::MetadataResponse {
6775 let topic_ids = if with_topic_ids {
6776 HashMap::from([(TOPIC.to_string(), topic_id())])
6777 } else {
6778 HashMap::new()
6779 };
6780 crate::common::requests::RequestTestUtils::metadata_update_with_ids(
6781 "kafka-cluster",
6782 1,
6783 &HashMap::new(),
6784 &HashMap::from([(TOPIC.to_string(), num_partitions)]),
6785 &|_| None,
6786 &topic_ids,
6787 )
6788 }
6789
6790 /// A `KafkaProducer` and the `Sender` that serves it, sharing every piece of
6791 /// state production shares.
6792 struct TxnProducerContext {
6793 producer: KafkaProducer<String, String>,
6794 sender: Sender<MockClient>,
6795 accumulator: Arc<RecordAccumulator>,
6796 metadata: Arc<ProducerMetadata>,
6797 transaction_manager: Arc<Mutex<TransactionManager>>,
6798 time: Arc<MockTime>,
6799 }
6800
6801 impl TxnProducerContext {
6802 /// Builds the context from `bootstrap.servers` plus `extra` configuration,
6803 /// mirroring the `configs` map each Java test assembles.
6804 ///
6805 /// `num_partitions` seeds the metadata for [`TOPIC`], standing in for Java's
6806 /// `RequestTestUtils.metadataUpdateWith(1, singletonMap("topic", 1))`.
6807 fn new(extra: &[(&str, &str)], num_partitions: i32) -> Self {
6808 Self::with_options(extra, num_partitions, false)
6809 }
6810
6811 /// As [`Self::new`], but with `transaction.version` finalized at level 2 so the
6812 /// manager enables KIP-890 Transaction V2 in `initTransactions`.
6813 ///
6814 /// Java seeds the same thing through `client.setNodeApiVersions(..)` plus
6815 /// `apiVersions.update(NODE.idString(), nodeApiVersions)`; only the second half
6816 /// is load-bearing, because `TransactionManager` reads `apiVersions` and never
6817 /// the client.
6818 fn transactional_v2(extra: &[(&str, &str)], num_partitions: i32) -> Self {
6819 Self::with_options(extra, num_partitions, true)
6820 }
6821
6822 fn with_options(extra: &[(&str, &str)], num_partitions: i32, transaction_v2: bool) -> Self {
6823 Self::with_metadata_seed(extra, num_partitions, transaction_v2, true, true)
6824 }
6825
6826 /// As [`Self::with_options`], choosing how the metadata starts out: whether
6827 /// [`TOPIC`] is already in the producer's tracked topic set (`metadata.add`),
6828 /// and whether the seeded snapshot carries its topic id. The KIP-1319 tests
6829 /// use an untracked topic, as Java's do, so `sendOffsetsToTransaction` has to
6830 /// refresh the metadata before it delegates.
6831 fn with_metadata_seed(
6832 extra: &[(&str, &str)],
6833 num_partitions: i32,
6834 transaction_v2: bool,
6835 track_topic: bool,
6836 with_topic_ids: bool,
6837 ) -> Self {
6838 let mut props = HashMap::from([("bootstrap.servers".to_string(), "localhost:9000".to_string())]);
6839 for (key, value) in extra {
6840 props.insert((*key).to_string(), (*value).to_string());
6841 }
6842 let config = ProducerConfig::new(&props).expect("valid config");
6843 let log_context = LogContext::new(format!("[Producer clientId={}] ", config.client_id));
6844 let time = mock_time(1_000);
6845 let api_versions = Arc::new(ApiVersions::new());
6846 if transaction_v2 {
6847 seed_transaction_version(&api_versions, 2);
6848 }
6849
6850 // The metadata comes first: the transaction manager reads its topic ids
6851 // (KIP-1319), as Java's constructor now builds it ahead of the manager.
6852 let metadata = Arc::new(ProducerMetadata::with_log_context(
6853 config.retry_backoff_ms,
6854 config.retry_backoff_max_ms,
6855 config.metadata_max_age_ms,
6856 config.metadata_max_idle_ms,
6857 ClusterResourceListeners::new(),
6858 log_context.clone(),
6859 ));
6860 let transaction_manager = KafkaProducer::<String, String>::configure_transaction_state(
6861 &config,
6862 &api_versions,
6863 &metadata.metadata_arc(),
6864 &log_context,
6865 )
6866 .expect("these tests always enable idempotence");
6867 let pending_requests = Arc::new(Mutex::new(PendingRequests::new()));
6868
6869 if track_topic {
6870 metadata.add(TOPIC, time.milliseconds());
6871 }
6872 // `metadata_update_with_ids` rather than `metadata_update_with`: the produce
6873 // path stamps the topic id from metadata onto the request, so a response has
6874 // to carry the same one to be matched back to its batch.
6875 let update = topic_metadata_update(num_partitions, with_topic_ids);
6876 metadata.update_with_current_request_version(&update, false, time.milliseconds());
6877
6878 let batch_size = config.batch_size.max(1);
6879 let metrics = Arc::new(Metrics::new());
6880 let buffer_pool = Arc::new(BufferPool::new(
6881 config.buffer_memory,
6882 batch_size as usize,
6883 Arc::clone(&metrics),
6884 Arc::clone(&time) as Arc<dyn Time>,
6885 KafkaProducer::<String, String>::PRODUCER_METRIC_GROUP_NAME,
6886 ));
6887 let accumulator = Arc::new(RecordAccumulator::with_log_context(
6888 batch_size,
6889 Compression::of(config.compression_type).build(),
6890 KafkaProducer::<String, String>::linger_ms(&config),
6891 config.retry_backoff_ms,
6892 config.retry_backoff_max_ms,
6893 config.delivery_timeout_ms,
6894 PartitionerConfig::new(
6895 config.partitioner_adaptive_partitioning_enable,
6896 config.partitioner_availability_timeout_ms,
6897 config.partitioner_rack_aware,
6898 &config.client_rack,
6899 )
6900 .unwrap(),
6901 Arc::clone(&metrics),
6902 KafkaProducer::<String, String>::PRODUCER_METRIC_GROUP_NAME,
6903 Arc::clone(&time) as Arc<dyn Time>,
6904 buffer_pool,
6905 Some(Arc::clone(&transaction_manager)),
6906 log_context.clone(),
6907 ));
6908
6909 let client = MockClient::with_static_nodes(vec![coordinator_node()], Arc::clone(&time) as Arc<dyn Time>);
6910 let wakeup = client.wakeup_notify();
6911 let running = Arc::new(AtomicBool::new(true));
6912 let force_close = Arc::new(AtomicBool::new(false));
6913
6914 let sender = Sender::new(
6915 client,
6916 Arc::clone(&metadata),
6917 Arc::clone(&accumulator),
6918 config.max_in_flight_requests_per_connection == 1,
6919 config.max_request_size,
6920 config.acks,
6921 config.retries,
6922 config.request_timeout_ms,
6923 config.retry_backoff_ms,
6924 SenderMetricsRegistry::new(Arc::clone(&metrics)),
6925 Arc::clone(&running),
6926 Arc::clone(&force_close),
6927 Arc::clone(&time) as Arc<dyn Time>,
6928 Some(Arc::clone(&transaction_manager)),
6929 Arc::clone(&pending_requests),
6930 log_context.clone(),
6931 );
6932
6933 let producer = KafkaProducer::with_options(
6934 KafkaProducerOptionsBuilder::new()
6935 .set_config(&config)
6936 .set_key_serializer(Box::new(StringSerializer))
6937 .set_value_serializer(Box::new(StringSerializer))
6938 .set_metadata(Arc::clone(&metadata))
6939 .set_accumulator(Arc::clone(&accumulator))
6940 .set_running(running)
6941 .set_force_close(force_close)
6942 .set_wakeup(wakeup)
6943 .set_time(Arc::clone(&time) as Arc<dyn Time>)
6944 .set_transaction_manager(Some(Arc::clone(&transaction_manager)))
6945 .set_pending_requests(pending_requests)
6946 .build()
6947 .expect("KafkaProducerOptionsBuilder::build: every mandatory parameter is set above"),
6948 );
6949
6950 Self { producer, sender, accumulator, metadata, transaction_manager, time }
6951 }
6952
6953 /// The default transactional setup: `transactional.id=some.id`, one topic
6954 /// partition.
6955 fn transactional() -> Self {
6956 Self::new(&[("transactional.id", TRANSACTIONAL_ID)], 1)
6957 }
6958
6959 /// Drives `Sender::run_once` `iterations` times with no application call in
6960 /// flight, for the assertions Java makes with a bare `sender.runOnce()`.
6961 async fn run_sender(&mut self, iterations: usize) {
6962 for _ in 0..iterations {
6963 run_once(&mut self.sender).await;
6964 }
6965 }
6966
6967 /// Queues the `FindCoordinator` + `InitProducerId` pair every
6968 /// `initTransactions` needs, in the order the `Sender` sends them.
6969 fn prepare_init_transactions(&mut self, error: Errors, producer_id: i64, epoch: i16) {
6970 let node = coordinator_node();
6971 self.sender
6972 .client_mut()
6973 .prepare_response(find_coordinator_response(Errors::None, TRANSACTIONAL_ID, &node));
6974 self.sender
6975 .client_mut()
6976 .prepare_response(init_producer_id_response(error, producer_id, epoch));
6977 }
6978
6979 /// The producer id and epoch the manager currently holds.
6980 fn producer_id_and_epoch(&self) -> (i64, i16) {
6981 let manager = self.transaction_manager.lock().unwrap();
6982 let id_and_epoch = manager.producer_id_and_epoch();
6983 (id_and_epoch.producer_id, id_and_epoch.epoch)
6984 }
6985 }
6986
6987 /// `Sender.runOnce()` with Java's catch-and-log
6988 /// (`Sender`) rather than a propagating `?`.
6989 async fn run_once(sender: &mut Sender<MockClient>) {
6990 if let Err(error) = sender.run_once().await {
6991 eprintln!("run_once: {}", error);
6992 }
6993 }
6994
6995 /// Runs `op` — a call on the application task — while driving the `Sender` the way
6996 /// Java's spawned I/O thread would.
6997 ///
6998 /// `tokio::join!` rather than `tokio::select!`: `select!` drops the losing future
6999 /// and its side effects (CLAUDE.md §11.6.1), which here would abandon a half-sent
7000 /// transactional request. `join!` polls both to completion and never drops either.
7001 ///
7002 /// The loop stops as soon as `op` resolves. It `yield_now()`s rather than sleeps:
7003 /// `MockClient::poll` returns immediately, so the yield is what lets the runtime
7004 /// run `op` and fire its `max.block.ms` timer, and it is also what keeps the
7005 /// injected `MockTime` advancing (each `run_once` reads the clock several times, so
7006 /// with `set_auto_tick_ms` the loop is the only thing that moves time — a Java
7007 /// `MockTime(1)` plus a hot Sender thread behaves the same way).
7008 ///
7009 /// A free function rather than a method on [`TxnProducerContext`] so callers can
7010 /// pass `&mut ctx.sender` and a future borrowing `&ctx.producer` at the same time.
7011 async fn drive<T>(sender: &mut Sender<MockClient>, op: impl std::future::Future<Output = T>) -> T {
7012 // A wall-clock deadline, NOT an iteration count. `drive`'s ops wait on
7013 // `await_result_timeout` (a real `tokio::time::timeout` on `max.block.ms`), so a
7014 // test that legitimately waits a timeout out busy-spins this loop —
7015 // `run_once` + `yield_now`, neither of which sleeps — for the whole real-time
7016 // window. That is an unbounded, machine-speed-dependent number of iterations,
7017 // especially when MockTime is frozen (no `set_auto_tick_ms`, as in
7018 // `test_init_transactions_response_after_timeout`). An iteration cap could not be
7019 // sized to let those tests pass *and* fail a genuine hang fast; a wall-clock
7020 // budget can. It comfortably exceeds the longest `max.block.ms` any drive-based
7021 // test configures while still surfacing a stuck regression, mirroring the intent
7022 // of `producer_test_utils::MAX_TRIES` (fail fast, do not hang) with the primitive
7023 // that fits `drive`'s real-time waits — an iteration count fits `run_until` there
7024 // only because it drives the Sender alone, with no concurrent real-time timeout.
7025 const DRIVE_BUDGET: std::time::Duration = std::time::Duration::from_secs(60);
7026 let done = AtomicBool::new(false);
7027 let op = async {
7028 let out = op.await;
7029 done.store(true, Ordering::SeqCst);
7030 out
7031 };
7032 let start = std::time::Instant::now();
7033 let driver = async {
7034 while !done.load(Ordering::SeqCst) {
7035 assert!(
7036 start.elapsed() < DRIVE_BUDGET,
7037 "drive: op did not complete within {DRIVE_BUDGET:?} — a stuck regression, not a hang"
7038 );
7039 run_once(sender).await;
7040 tokio::task::yield_now().await;
7041 }
7042 };
7043 let (out, ()) = tokio::join!(op, driver);
7044 out
7045 }
7046
7047 /// `producer.initTransactions()` driven to completion — the first line of most
7048 /// Java transactional tests.
7049 async fn init_transactions(ctx: &mut TxnProducerContext) {
7050 ctx.prepare_init_transactions(Errors::None, PRODUCER_ID, EPOCH);
7051 drive(&mut ctx.sender, ctx.producer.init_transactions())
7052 .await
7053 .expect("initTransactions succeeds");
7054 }
7055
7056 /// Java's `initProducerIdResponse(long producerId, short epoch, Errors error)`
7057 /// (`KafkaProducerTest#initProducerIdResponse`).
7058 fn init_producer_id_response(error: Errors, producer_id: i64, epoch: i16) -> ConcreteResponse {
7059 use crate::InitProducerIdResponseData;
7060 use crate::common::requests::InitProducerIdResponse;
7061
7062 let mut data = InitProducerIdResponseData::new();
7063 data.set_error_code(error.code())
7064 .set_producer_id(producer_id)
7065 .set_producer_epoch(epoch)
7066 .set_throttle_time_ms(0);
7067 ConcreteResponse::InitProducerId(InitProducerIdResponse::new(data))
7068 }
7069
7070 /// `FindCoordinatorResponse.prepareResponse(error, key, node)`.
7071 fn find_coordinator_response(error: Errors, key: &str, node: &Node) -> ConcreteResponse {
7072 use crate::common::requests::FindCoordinatorResponse;
7073
7074 ConcreteResponse::FindCoordinator(FindCoordinatorResponse::prepare_response(error, key, node))
7075 }
7076
7077 /// Java's `endTxnResponse(Errors error)` (`KafkaProducerTest#endTxnResponse`).
7078 fn end_txn_response(error: Errors) -> ConcreteResponse {
7079 use crate::EndTxnResponseData;
7080 use crate::common::requests::EndTxnResponse;
7081
7082 let mut data = EndTxnResponseData::new();
7083 data.set_error_code(error.code()).set_throttle_time_ms(0);
7084 ConcreteResponse::EndTxn(EndTxnResponse::new(data))
7085 }
7086
7087 // -- Transactional `KafkaProducerTest` methods --------------------------
7088
7089 /// Translated from `KafkaProducerTest.testInitTransactionTimeout`.
7090 ///
7091 /// The `FindCoordinator` is answered but the `InitProducerId` is not, so
7092 /// `initTransactions` expires at `max.block.ms`. A retry then succeeds — which is
7093 /// only possible because a timed-out `TransactionalRequestResult` is **not**
7094 /// acked (Java's `await` sets `isAcked` only after the latch opens,
7095 /// `TransactionalRequestResult#await`), so
7096 /// `handleCachedTransactionRequestResult` hands the same pending result back.
7097 #[tokio::test]
7098 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testInitTransactionTimeout")]
7099 async fn test_init_transaction_timeout() {
7100 let mut ctx = TxnProducerContext::new(&[("transactional.id", "bad-transaction"), ("max.block.ms", "500")], 1);
7101 // Coarser tick reaches the simulated 500ms deadline in fewer real
7102 // `drive()` iterations, reducing flakiness under CPU contention.
7103 ctx.time.set_auto_tick_ms(20);
7104 let node = coordinator_node();
7105 ctx.sender
7106 .client_mut()
7107 .prepare_response(find_coordinator_response(Errors::None, "bad-transaction", &node));
7108
7109 let error = drive(&mut ctx.sender, ctx.producer.init_transactions())
7110 .await
7111 .expect_err("no InitProducerId response is prepared");
7112 // AK 4.3.1: `assertFutureThrowsWithMessageContaining(TimeoutException, future,
7113 // INIT_TXN_TIMEOUT_MSG)`.
7114 assert!(matches!(error, Error::Timeout(_)), "expected a TimeoutException, got {error}");
7115 assert!(
7116 error.message().contains(KafkaProducer::<String, String>::INIT_TXN_TIMEOUT_MSG),
7117 "expected the InitTransactions timeout reason in the message, got {error}"
7118 );
7119
7120 // Retry initialization should work.
7121 ctx.sender
7122 .client_mut()
7123 .prepare_response(find_coordinator_response(Errors::None, "bad-transaction", &node));
7124 ctx.sender
7125 .client_mut()
7126 .prepare_response(init_producer_id_response(Errors::None, PRODUCER_ID, EPOCH));
7127 drive(&mut ctx.sender, ctx.producer.init_transactions())
7128 .await
7129 .expect("the retry succeeds");
7130 assert_eq!(ctx.producer_id_and_epoch(), (PRODUCER_ID, EPOCH));
7131 }
7132
7133 /// Translated from `KafkaProducerTest.testInitTransactionsResponseAfterTimeout`.
7134 ///
7135 /// Java submits `initTransactions` to an executor, waits for the `InitProducerId`
7136 /// to be in flight, advances the clock past `max.block.ms`, asserts the future
7137 /// threw, *then* answers the request and calls `initTransactions` again. The
7138 /// second call must return normally rather than raise — the response completed the
7139 /// cached result.
7140 ///
7141 /// The executor is not needed here: [`drive`] already runs the application call
7142 /// and the Sender concurrently, and its return is the future Java asserts on.
7143 #[tokio::test]
7144 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testInitTransactionsResponseAfterTimeout")]
7145 async fn test_init_transactions_response_after_timeout() {
7146 let mut ctx = TxnProducerContext::new(&[("transactional.id", "bad-transaction"), ("max.block.ms", "500")], 1);
7147 let node = coordinator_node();
7148 ctx.sender
7149 .client_mut()
7150 .prepare_response(find_coordinator_response(Errors::None, "bad-transaction", &node));
7151
7152 let error = drive(&mut ctx.sender, ctx.producer.init_transactions())
7153 .await
7154 .expect_err("the InitProducerId is unanswered");
7155 // AK 4.3.1: `assertFutureThrowsWithMessageContaining(TimeoutException, future,
7156 // INIT_TXN_TIMEOUT_MSG)` — the timeout message carries the init-txn reason.
7157 assert!(matches!(error, Error::Timeout(_)), "expected a TimeoutException, got {error}");
7158 assert!(
7159 error.message().contains(KafkaProducer::<String, String>::INIT_TXN_TIMEOUT_MSG),
7160 "expected the InitTransactions timeout reason in the message, got {error}"
7161 );
7162 assert!(
7163 ctx.sender.client().in_flight_request_count() > 0,
7164 "the InitProducerId must still be in flight, which is what the late response answers"
7165 );
7166
7167 // Java's `client.respond(..)`: answer the request that is already in flight.
7168 ctx.sender
7169 .client_mut()
7170 .respond(init_producer_id_response(Errors::None, PRODUCER_ID, EPOCH));
7171 drive(&mut ctx.sender, ctx.producer.init_transactions())
7172 .await
7173 .expect("the late response completed the cached result");
7174 assert_eq!(ctx.producer_id_and_epoch(), (PRODUCER_ID, EPOCH));
7175 }
7176
7177 /// Translated from `KafkaProducerTest.testInitTransactionWhileThrottled`.
7178 ///
7179 /// The coordinator is throttled for 5 s while `max.block.ms` is 10 s, so
7180 /// `awaitNodeReady` has to wait the node out before the `InitProducerId` goes.
7181 #[tokio::test]
7182 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testInitTransactionWhileThrottled")]
7183 async fn test_init_transaction_while_throttled() {
7184 let mut ctx = TxnProducerContext::new(&[("transactional.id", TRANSACTIONAL_ID), ("max.block.ms", "10000")], 1);
7185 // Java's `new MockTime(1)`: without a ticking clock `awaitNodeReady`'s
7186 // `now - startTime < timeout` never advances, because this test drives the
7187 // Sender itself and nothing else moves the clock.
7188 ctx.time.set_auto_tick_ms(1);
7189 let node = coordinator_node();
7190 ctx.sender.client_mut().throttle(&node, 5000);
7191 ctx.prepare_init_transactions(Errors::None, PRODUCER_ID, EPOCH);
7192
7193 drive(&mut ctx.sender, ctx.producer.init_transactions())
7194 .await
7195 .expect("initTransactions rides out the throttle");
7196 assert_eq!(ctx.producer_id_and_epoch(), (PRODUCER_ID, EPOCH));
7197 }
7198
7199 /// Translated from `KafkaProducerTest.testClusterAuthorizationFailure`.
7200 ///
7201 /// `CLUSTER_AUTHORIZATION_FAILED` on the `InitProducerId` is an authorization
7202 /// error, so the Sender's `shouldHandleAuthorizationError`
7203 /// (`Sender#shouldHandleAuthorizationError`) fails the pending requests, aborts the batches and
7204 /// transitions back to `UNINITIALIZED` — which is what makes the retry Java
7205 /// performs possible at all (PLAN §9.16 pass-1 finding 1).
7206 #[tokio::test]
7207 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testClusterAuthorizationFailure")]
7208 async fn test_cluster_authorization_failure() {
7209 let mut ctx = TxnProducerContext::new(
7210 &[
7211 ("transactional.id", "some-txn"),
7212 ("enable.idempotence", "true"),
7213 ("max.block.ms", "500"),
7214 ],
7215 1,
7216 );
7217 ctx.time.set_auto_tick_ms(1);
7218 let node = coordinator_node();
7219 ctx.sender
7220 .client_mut()
7221 .prepare_response(find_coordinator_response(Errors::None, "some-txn", &node));
7222 ctx.sender.client_mut().prepare_response(init_producer_id_response(
7223 Errors::ClusterAuthorizationFailed,
7224 PRODUCER_ID,
7225 EPOCH,
7226 ));
7227
7228 let error = drive(&mut ctx.sender, ctx.producer.init_transactions())
7229 .await
7230 .expect_err("the cluster authorization failure surfaces");
7231 assert_eq!(error.error(), Errors::ClusterAuthorizationFailed);
7232 assert!(
7233 ctx.transaction_manager.lock().unwrap().has_abortable_error(),
7234 "CLUSTER_AUTHORIZATION_FAILED is an abortable error on InitProducerId \
7235 (`TransactionManager.InitProducerIdHandler#handleResponse`)"
7236 );
7237
7238 // Java retries with
7239 // `TestUtils.retryOnExceptionWithTimeout(1000, 100, producer::initTransactions)`
7240 // because its Sender runs on an independent thread and the recovery has not
7241 // necessarily happened yet: the manager is in ABORTABLE_ERROR the instant the
7242 // result fails, and only the Sender's *next* `runOnce` takes
7243 // `shouldHandleAuthorizationError`'s path back to UNINITIALIZED
7244 // (`Sender#shouldHandleAuthorizationError`). Until it does, an attempt is rejected with "we are
7245 // in an error state".
7246 //
7247 // `drive` stops the moment the application call resolves, so that iteration is
7248 // asked for explicitly here — which makes the recovery deterministic rather
7249 // than raced, and lets the state be asserted directly instead of retried
7250 // around.
7251 ctx.run_sender(1).await;
7252 assert!(
7253 !ctx.transaction_manager.lock().unwrap().has_error(),
7254 "one runOnce must clear the abortable error via transitionToUninitialized"
7255 );
7256 assert!(
7257 !ctx.transaction_manager.lock().unwrap().has_producer_id(),
7258 "UNINITIALIZED means the producer id is gone too"
7259 );
7260
7261 // Only an `InitProducerId` is prepared, as in Java: `transitionToUninitialized`
7262 // does not forget the coordinator, so no second `FindCoordinator` is sent.
7263 ctx.sender
7264 .client_mut()
7265 .prepare_response(init_producer_id_response(Errors::None, PRODUCER_ID, EPOCH));
7266 drive(&mut ctx.sender, ctx.producer.init_transactions())
7267 .await
7268 .expect("the retry succeeds once the manager is back in UNINITIALIZED");
7269 assert_eq!(ctx.producer_id_and_epoch(), (PRODUCER_ID, EPOCH));
7270 ctx.producer.close().await.expect("close");
7271 }
7272
7273 /// Translated from `KafkaProducerTest.testAbortTransaction`.
7274 ///
7275 /// The whole `FindCoordinator` -> `InitProducerId` -> `EndTxn(ABORT)` sequence,
7276 /// with no records in the transaction.
7277 #[tokio::test]
7278 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testAbortTransaction")]
7279 async fn test_abort_transaction() {
7280 let mut ctx = TxnProducerContext::transactional();
7281 init_transactions(&mut ctx).await;
7282 ctx.producer.begin_transaction().expect("beginTransaction");
7283
7284 ctx.sender.client_mut().prepare_response(end_txn_response(Errors::None));
7285 drive(&mut ctx.sender, ctx.producer.abort_transaction())
7286 .await
7287 .expect("abortTransaction");
7288 }
7289
7290 /// Translated from
7291 /// `KafkaProducerTest.testOnlyCanExecuteCloseAfterInitTransactionsTimeout`.
7292 ///
7293 /// Nothing answers the `FindCoordinator`, so `initTransactions` expires at
7294 /// `max.block.ms=5`. After that failure every other transactional operation must
7295 /// be rejected, and only `close` is allowed.
7296 #[tokio::test]
7297 #[doc(
7298 alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testOnlyCanExecuteCloseAfterInitTransactionsTimeout"
7299 )]
7300 async fn test_only_can_execute_close_after_init_transactions_timeout() {
7301 let mut ctx = TxnProducerContext::new(&[("transactional.id", "bad-transaction"), ("max.block.ms", "5")], 1);
7302
7303 let error = drive(&mut ctx.sender, ctx.producer.init_transactions())
7304 .await
7305 .expect_err("nothing answers the FindCoordinator");
7306 // AK 4.3.1: `assertTrue(timeoutEx.getMessage().contains(INIT_TXN_TIMEOUT_MSG))`.
7307 assert!(
7308 error.message().contains(KafkaProducer::<String, String>::INIT_TXN_TIMEOUT_MSG),
7309 "expected the InitTransactions timeout reason in the message, got {error}"
7310 );
7311 assert_eq!(
7312 error.message(),
7313 format!(
7314 "Timeout expired after 5ms while awaiting InitProducerId. {}",
7315 KafkaProducer::<String, String>::INIT_TXN_TIMEOUT_MSG
7316 )
7317 );
7318
7319 // Other transactional operations are not allowed once the caller has taken the
7320 // error from a failed initTransactions: the manager is still INITIALIZING with
7321 // an unacked pending transition.
7322 let begin_error = ctx.producer.begin_transaction().expect_err("beginTransaction is rejected");
7323 assert_eq!(
7324 begin_error.message(),
7325 "Cannot attempt operation `beginTransaction` because the previous call to \
7326 `initTransactions` timed out and must be retried"
7327 );
7328
7329 ctx.producer
7330 .close_with_timeout(Duration::from_millis(0))
7331 .await
7332 .expect("close is the one allowed operation");
7333 }
7334
7335 /// Translated from `KafkaProducerTest.testPartitionAddedToTransaction`.
7336 ///
7337 /// Pins the producer-level transactional wiring: `doSend` must call
7338 /// `transactionManager.maybeAddPartition(tp)` after the append succeeds, with the
7339 /// partition the accumulator actually chose (`KafkaProducer#doSend`). The
7340 /// send's future must also still be pending — the record is buffered, not sent.
7341 ///
7342 /// # Deviation: a real manager instead of Mockito's `verify`
7343 ///
7344 /// Java builds the producer through `KafkaProducerTestContext`, which injects
7345 /// `mock(TransactionManager.class)` (`KafkaProducerTest.KafkaProducerTestContext#transactionManager`, passed at `KafkaProducerTest`), and asserts with
7346 /// `verify(ctx.transactionManager).maybeAddPartition(topicPartition)`.
7347 /// `TransactionManager` is a concrete struct here, so there is nothing to stub; the
7348 /// substitute is a **real** transactional manager driven into `IN_TRANSACTION`, and
7349 /// the assertion is `is_partition_pending_add` (`TransactionManager`) — the
7350 /// state `maybeAddPartition` exists to produce.
7351 ///
7352 /// That is stronger than `verify` on one axis: Mockito confirms the call was made,
7353 /// while this confirms it was made *and* had its effect. It is **not** stronger for
7354 /// carrying the right partition — `verify(..).maybeAddPartition(topicPartition)`
7355 /// checks the argument too, so that clause is parity, not superiority.
7356 ///
7357 /// The effect clause is a real gain, and holds because the state read is isolated.
7358 /// `is_partition_pending_add` reads the union
7359 /// `new_partitions_in_transaction ∪ pending_partitions_in_transaction`
7360 /// (`TransactionManager`), and in this crate:
7361 /// `new_partitions_in_transaction.insert` has exactly one call site, inside
7362 /// `maybe_add_partition`; and `pending_partitions_in_transaction` is only ever filled
7363 /// by `add_partitions_to_transaction_handler`'s
7364 /// `.extend(new_partitions_in_transaction.iter())`, so it is downstream of that same
7365 /// insert. The union is therefore non-empty only if `maybe_add_partition` ran, and
7366 /// the test asserts the negative pre-condition first.
7367 ///
7368 /// It is also why this test could be written at all rather than deferred like
7369 /// `SenderTest`'s one mock-injected entry, whose stub makes a real method *throw* and
7370 /// so has no real-state equivalent.
7371 ///
7372 /// # The one Java assertion not carried across
7373 ///
7374 /// Java opens with `assertEquals(future, producer.send(record))`, comparing
7375 /// the returned future against a **pre-built** `FutureRecordMetadata` that
7376 /// `expectAppend` installed by stubbing `ctx.accumulator.append(..)` and
7377 /// `ctx.partitioner.partition(..)` to return it. That assertion is about the
7378 /// *accumulator* mock, not the manager mock the deviation above discusses, and it is
7379 /// unrepresentable here for the same reason: with a real `RecordAccumulator` the
7380 /// future is created inside `append`, so there is no pre-known value to compare
7381 /// identity against. What survives of its intent — that `send` hands back the
7382 /// accumulator's own pending future rather than a completed or failed one — is
7383 /// asserted directly by `!future.is_done()` below.
7384 ///
7385 /// # Why this method was missing until Critic 46 issue 4
7386 ///
7387 /// It reaches the transactional path only through the injected mock, so its body
7388 /// names no public transactional method and no transactional config key — and the
7389 /// accounting block's marker set contained neither `TransactionManager` nor
7390 /// `maybeAddPartition`. Both are markers now; see the accounting block.
7391 #[tokio::test]
7392 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testPartitionAddedToTransaction")]
7393 async fn test_partition_added_to_transaction() {
7394 let mut ctx = TxnProducerContext::transactional();
7395 init_transactions(&mut ctx).await;
7396 ctx.producer.begin_transaction().expect("beginTransaction");
7397
7398 let partition = TopicPartition::new(TOPIC.to_string(), 0);
7399 assert!(
7400 !ctx.transaction_manager.lock().unwrap().is_partition_pending_add(&partition),
7401 "nothing may be pending before the send"
7402 );
7403
7404 let record = ProducerRecord::with_options(
7405 ProducerRecordOptionsBuilder::new()
7406 .set_topic(TOPIC.to_string())
7407 .set_value(Some("value".to_string()))
7408 .set_timestamp(Some(ctx.time.milliseconds()))
7409 .set_key(Some("key".to_string()))
7410 .build()
7411 .unwrap(),
7412 )
7413 .expect("a valid record");
7414 let future = ctx.producer.send(record).await.expect("send");
7415
7416 assert!(
7417 !future.is_done(),
7418 "the record is buffered in the accumulator, not sent — nothing has driven the Sender"
7419 );
7420 assert!(
7421 ctx.transaction_manager.lock().unwrap().is_partition_pending_add(&partition),
7422 "doSend must call maybeAddPartition with the partition the accumulator chose"
7423 );
7424 }
7425
7426 // =====================================================================
7427 // `maybeAddPartition`'s failure arm in `doSend`
7428 //
7429 // `TransactionManagerTest` covers what `maybeAddPartition` raises in each state
7430 // (`testFailIfNotReadyForSend*`, translated in
7431 // `transaction_manager.rs`). What follows covers the other half — how `doSend`
7432 // *surfaces* each of those, which only `KafkaProducer` decides — and it has no
7433 // Java counterpart: Java's own `KafkaProducerTest` reaches this arm once, through
7434 // a Mockito stub, in `testPartitionAddedToTransaction` above. These rows therefore
7435 // do NOT enter the Phase-6 accounting block's denominator.
7436 //
7437 // They exist because the arm shipped with a self-deadlock: written inline as
7438 // `if let Err(e) = tm.lock().unwrap().maybe_add_partition(..)`, the scrutinee's
7439 // `MutexGuard` outlives the whole success arm (edition 2024 only shortens it
7440 // across `else`), and the body re-locked the same non-reentrant
7441 // `std::sync::Mutex` through `handle_api_error` →
7442 // `maybe_transition_to_error_state`. Java's monitor is reentrant, so no Java test
7443 // could have caught it.
7444 // =====================================================================
7445
7446 /// How long [`bounded`] waits before calling a send wedged.
7447 ///
7448 /// Generous against a loaded CI box; the operations under test do no I/O and
7449 /// finish in microseconds.
7450 const DEADLOCK_BOUND: Duration = Duration::from_secs(10);
7451
7452 /// Runs `body` on its own thread and returns its value, failing the test if it
7453 /// does not finish within [`DEADLOCK_BOUND`].
7454 ///
7455 /// A `tokio::time::timeout` around the send would NOT bound these tests. The
7456 /// regression they guard blocks the thread inside `std::sync::Mutex::lock`, so the
7457 /// future never yields, the runtime never advances its timers, and the timeout can
7458 /// never fire — the symptom is a hung `cargo test`, not a failing one. Only a
7459 /// second thread can observe a wedged first one, so `body` gets a thread and its
7460 /// own current-thread runtime while the test thread waits on a channel.
7461 ///
7462 /// On a timeout the worker stays blocked forever, holding the producer it built.
7463 /// That is deliberate and harmless: it shares nothing with any other test, and
7464 /// libtest ends the process with `exit(2)` rather than joining stray threads.
7465 fn bounded<T: Send + 'static>(what: &str, body: impl FnOnce() -> T + Send + 'static) -> T {
7466 let (tx, rx) = std::sync::mpsc::channel();
7467 std::thread::spawn(move || {
7468 // `send` fails only if the receiver timed out and gave up; nothing to do.
7469 let _ = tx.send(body());
7470 });
7471 rx.recv_timeout(DEADLOCK_BOUND).unwrap_or_else(|_| {
7472 panic!(
7473 "{what} did not return within {DEADLOCK_BOUND:?} — the send path is wedged. \
7474 doSend's maybeAddPartition arm must release the TransactionManager guard \
7475 before handling the error, because handle_api_error re-locks it."
7476 )
7477 })
7478 }
7479
7480 /// Runs `body` on a fresh current-thread runtime inside [`bounded`].
7481 fn bounded_block_on<F>(what: &str, body: impl FnOnce() -> F + Send + 'static) -> F::Output
7482 where
7483 F: std::future::Future,
7484 F::Output: Send + 'static,
7485 {
7486 bounded(what, move || {
7487 tokio::runtime::Builder::new_current_thread()
7488 .enable_all()
7489 .build()
7490 .expect("a current-thread runtime")
7491 .block_on(body())
7492 })
7493 }
7494
7495 /// The record every test below sends; its contents are irrelevant because no send
7496 /// is expected to reach a batch's wire form.
7497 fn misuse_record() -> ProducerRecord<String, String> {
7498 ProducerRecord::with_key(TOPIC.to_string(), Some("key".to_string()), Some("value".to_string()))
7499 }
7500
7501 /// `send()` on a transactional producer that never called `initTransactions`.
7502 ///
7503 /// The state `TransactionManagerTest.testFailIfNotReadyForSendNoProducerId`
7504 /// asserts on, surfaced through `doSend`. `maybeAddPartition` raises
7505 /// `IllegalStateException` (`TransactionManager`), which is not a
7506 /// `KafkaException` at all, so it misses `catch (ApiException e)` *and*
7507 /// `catch (KafkaException e)`, reaches `catch (Exception e)`
7508 /// (`KafkaProducer`) and is rethrown out of `send()`.
7509 #[test]
7510 fn test_send_before_init_transactions_returns_illegal_state() {
7511 let error = bounded_block_on("send before initTransactions", || async {
7512 let ctx = TxnProducerContext::transactional();
7513 ctx.producer
7514 .send(misuse_record())
7515 .await
7516 .expect_err("an uninitialized transactional producer cannot send")
7517 });
7518
7519 assert!(
7520 matches!(error, Error::LocalIllegalState(_)),
7521 "an illegal-state error is not an API error, so it must be returned by send() \
7522 rather than reported through the future; got {error:?}"
7523 );
7524 assert_eq!(
7525 error.to_string(),
7526 format!(
7527 "LocalIllegalStateError: Cannot add partition {TOPIC}-0 to transaction before completing a call to initTransactions"
7528 )
7529 );
7530 }
7531
7532 /// `send()` on an initialized transactional producer with no open transaction.
7533 ///
7534 /// The state `TransactionManagerTest.testFailIfNotReadyForSendNoOngoingTransaction`
7535 /// asserts on, surfaced through `doSend`. Same
7536 /// `IllegalStateException` treatment as above, from
7537 /// `TransactionManager`, whose message carries the state and Java's
7538 /// double space before it.
7539 #[test]
7540 fn test_send_outside_transaction_returns_illegal_state() {
7541 let error = bounded_block_on("send outside a transaction", || async {
7542 let mut ctx = TxnProducerContext::transactional();
7543 init_transactions(&mut ctx).await;
7544 ctx.producer
7545 .send(misuse_record())
7546 .await
7547 .expect_err("a send needs an open transaction")
7548 });
7549
7550 assert!(
7551 matches!(error, Error::LocalIllegalState(_)),
7552 "expected the IllegalState to be returned by send(), got {error:?}"
7553 );
7554 assert_eq!(
7555 error.to_string(),
7556 format!("LocalIllegalStateError: Cannot add partition {TOPIC}-0 to transaction while in state READY")
7557 );
7558 }
7559
7560 /// `send()` after an abortable error, the state
7561 /// `TransactionManagerTest.testFailIfNotReadyForSendAfterAbortableError`
7562 /// asserts on.
7563 ///
7564 /// `maybeFailWithError` raises a **bare** `KafkaException`
7565 /// (`TransactionManager#maybeFailWithError`). `ApiException extends KafkaException`, not the
7566 /// other way round, so `catch (ApiException e)` does not match and
7567 /// `catch (KafkaException e)` (`KafkaProducer`) rethrows it — a
7568 /// block that, unlike the `ApiException` one, never calls
7569 /// `maybeTransitionToErrorState`. Hence the second assertion: routing this error to
7570 /// the `ApiException` arm would overwrite `lastError` with the "we are in an error
7571 /// state" wrapper and lose the cause the application needs.
7572 #[test]
7573 fn test_send_after_abortable_error_returns_error_without_overwriting_last_error() {
7574 let (error, last_error) = bounded_block_on("send after an abortable error", || async {
7575 let mut ctx = TxnProducerContext::transactional();
7576 init_transactions(&mut ctx).await;
7577 ctx.producer.begin_transaction().expect("beginTransaction");
7578 ctx.transaction_manager
7579 .lock()
7580 .unwrap()
7581 .transition_to_abortable_error(Error::with_message(Errors::InvalidTxnState, "cause"), Caller::App)
7582 .expect("IN_TRANSACTION -> ABORTABLE_ERROR is valid");
7583
7584 let error = ctx
7585 .producer
7586 .send(misuse_record())
7587 .await
7588 .expect_err("an abortable error blocks further sends");
7589 let last_error = ctx.transaction_manager.lock().unwrap().last_error().cloned();
7590 (error, last_error)
7591 });
7592
7593 assert_eq!(
7594 error.error(),
7595 Errors::UnknownServerError,
7596 "this crate spells Java's bare Kafka error as UnknownServerError; got {error:?}"
7597 );
7598 assert_eq!(
7599 error.message(),
7600 "Cannot execute transactional method because we are in an error state"
7601 );
7602 let last_error = last_error.expect("the manager keeps the abortable cause");
7603 assert_eq!(
7604 last_error.error(),
7605 Errors::InvalidTxnState,
7606 "the rethrowing catch block must not run maybeTransitionToErrorState, which \
7607 would replace the cause with the wrapper; got {last_error:?}"
7608 );
7609 }
7610
7611 /// `send()` after a fatal error, the state
7612 /// `TransactionManagerTest.testFailIfNotReadyForSendAfterFatalError`
7613 /// asserts on. Same bare-`KafkaException` treatment as the abortable case.
7614 #[test]
7615 fn test_send_after_fatal_error_returns_error() {
7616 // The closure also reports the manager's state: Java records fatality there
7617 // (`hasFatalError()` == `currentState == FATAL_ERROR`), not on the error.
7618 let (error, still_fatal) = bounded_block_on("send after a fatal error", || async {
7619 let mut ctx = TxnProducerContext::transactional();
7620 init_transactions(&mut ctx).await;
7621 ctx.transaction_manager
7622 .lock()
7623 .unwrap()
7624 .transition_to_fatal_error(
7625 Error::with_message(Errors::ClusterAuthorizationFailed, "cause"),
7626 Caller::App,
7627 )
7628 .expect("FATAL_ERROR is always reachable");
7629 let error = ctx
7630 .producer
7631 .send(misuse_record())
7632 .await
7633 .expect_err("a fatal error blocks further sends");
7634 let still_fatal = ctx.transaction_manager.lock().unwrap().has_fatal_error();
7635 (error, still_fatal)
7636 });
7637
7638 assert_eq!(error.error(), Errors::UnknownServerError, "got {error:?}");
7639 assert_eq!(
7640 error.message(),
7641 "Cannot execute transactional method because we are in an error state"
7642 );
7643 // Java records fatality in the state machine, not on the exception
7644 // (`TransactionManager.hasFatalError()` == `currentState == FATAL_ERROR`;
7645 // `lastError` is a plain RuntimeException). The application distinguishes a
7646 // dead producer by the exception *type* `maybe_fail_with_error` surfaces —
7647 // asserted above — so the regression guard is the state, checked here.
7648 assert!(still_fatal, "the producer must remain in the fatal state: {error:?}");
7649 }
7650
7651 /// `send()` on a purely **idempotent** producer whose manager is in a fatal state,
7652 /// the state
7653 /// `TransactionManagerTest.testFailIfNotReadyForSendIdempotentProducerFatalError`
7654 /// asserts on.
7655 ///
7656 /// `maybeFailWithError` runs before `maybeAddPartition`'s `isTransactional()` test
7657 /// (`TransactionManager#maybeAddPartition` vs `TransactionManager#maybeAddPartition`), so a producer with no
7658 /// `transactional.id` reaches the same arm.
7659 #[test]
7660 fn test_idempotent_send_after_fatal_error_returns_error() {
7661 let error = bounded_block_on("idempotent send after a fatal error", || async {
7662 let ctx = TxnProducerContext::new(&[], 1);
7663 ctx.transaction_manager
7664 .lock()
7665 .unwrap()
7666 .transition_to_fatal_error(Error::with_message(Errors::UnsupportedVersion, "cause"), Caller::App)
7667 .expect("FATAL_ERROR is always reachable");
7668 ctx.producer
7669 .send(misuse_record())
7670 .await
7671 .expect_err("a fatal error blocks further sends")
7672 });
7673
7674 assert_eq!(error.error(), Errors::UnknownServerError, "got {error:?}");
7675 }
7676
7677 /// The other side of the split: an `ApiException` out of `maybeAddPartition` still
7678 /// takes `catch (ApiException e)` and is reported through the future.
7679 ///
7680 /// This is the arm that actually re-locks the manager —
7681 /// `handle_api_error` → `maybe_transition_to_error_state` — so it is the direct
7682 /// regression test for the deadlock. `maybeFailWithError` re-raises a fenced
7683 /// producer as `ProducerFencedException` (`TransactionManager#maybeFailWithError`),
7684 /// which IS an `ApiException`.
7685 ///
7686 /// The manager is seeded **abortable**, not fatal, so that
7687 /// `maybeTransitionToErrorState` has somewhere to move it: `ProducerFenced` is in
7688 /// that method's fatal set (Java `TransactionManager#maybeTransitionToErrorState`), so the
7689 /// `ApiException` block drives `ABORTABLE_ERROR -> FATAL_ERROR`. Seeding
7690 /// `FATAL_ERROR` up front — as this test first did — makes the state assertion
7691 /// tautological: it would hold even if the arm were changed to `return Err(error)`
7692 /// and never call `maybe_transition_to_error_state` at all. Both `has_error()`
7693 /// arms reach the same `maybeFailWithError` branch, which keys on
7694 /// `last_error`'s code, so the error the send observes is unchanged.
7695 #[test]
7696 fn test_send_after_producer_fenced_fails_the_future() {
7697 let (send_result, fatal_before, fatal_after) = bounded_block_on("send after being fenced", || async {
7698 let mut ctx = TxnProducerContext::transactional();
7699 init_transactions(&mut ctx).await;
7700 ctx.producer.begin_transaction().expect("beginTransaction");
7701 ctx.transaction_manager
7702 .lock()
7703 .unwrap()
7704 .transition_to_abortable_error(Error::with_message(Errors::ProducerFenced, "fenced"), Caller::App)
7705 .expect("IN_TRANSACTION -> ABORTABLE_ERROR is valid");
7706 let fatal_before = ctx.transaction_manager.lock().unwrap().has_fatal_error();
7707
7708 let send_result = match ctx.producer.send(misuse_record()).await {
7709 Ok(future) => Ok(future.get().await.expect_err("the fenced send cannot be acked")),
7710 Err(error) => Err(error),
7711 };
7712 let fatal_after = ctx.transaction_manager.lock().unwrap().has_fatal_error();
7713 (send_result, fatal_before, fatal_after)
7714 });
7715
7716 let error = send_result.expect("an API error is reported through the future, not the call");
7717 assert_eq!(
7718 error.error(),
7719 Errors::ProducerFenced,
7720 "a producer-fenced error is an API error, so doSend returns a failed future; got {error:?}"
7721 );
7722 // `maybeFailWithError` re-raises rather than re-throwing `lastError`, so the
7723 // message is the fresh one `TransactionManager#maybeFailWithError` builds — not the "fenced" text the
7724 // test seeded.
7725 assert_eq!(
7726 error.message(),
7727 format!(
7728 "Producer with transactionalId '{TRANSACTIONAL_ID}' and \
7729 (producerId={PRODUCER_ID}, epoch={EPOCH}) has been fenced by another producer \
7730 with the same transactionalId"
7731 )
7732 );
7733 assert!(
7734 !fatal_before,
7735 "the manager starts abortable, so the transition below is observable"
7736 );
7737 assert!(
7738 fatal_after,
7739 "the API-error block runs maybeTransitionToErrorState, which moves a fenced \
7740 producer from ABORTABLE_ERROR to FATAL_ERROR; the rethrow path would not"
7741 );
7742 }
7743
7744 /// The `maybeAddPartition` `ApiException` arm of `do_send_bytes` passes `None`
7745 /// (not the record's callback) to `handle_api_error`, so the callback is
7746 /// **not** fired synchronously there.
7747 ///
7748 /// The callback was moved into `accumulator.append(..)` and registered with the
7749 /// record's future, which fires it exactly once when the batch is later
7750 /// completed/aborted. Passing it here too would double-fire (CLAUDE.md §11.5).
7751 /// (Java's `doSend` catch additionally fires the raw `callback` at
7752 /// `KafkaProducer#doSend`, but keeps `callback` as a reference separate from
7753 /// the `appendCallbacks` it registered, so Java can invoke it twice on this path;
7754 /// Rust's single-owner model fires it exactly once.)
7755 #[test]
7756 fn test_send_api_exception_after_append_fires_callback_exactly_once() {
7757 let (count_after_send, count_final) = bounded_block_on("fenced send with callback", || async {
7758 let mut ctx = TxnProducerContext::transactional();
7759 init_transactions(&mut ctx).await;
7760 ctx.producer.begin_transaction().expect("beginTransaction");
7761 // Seed abortable (not fatal) so `maybeAddPartition`'s `maybe_fail_with_error`
7762 // re-raises `ProducerFenced` — an `ApiException` — after the append succeeds.
7763 ctx.transaction_manager
7764 .lock()
7765 .unwrap()
7766 .transition_to_abortable_error(Error::with_message(Errors::ProducerFenced, "fenced"), Caller::App)
7767 .expect("IN_TRANSACTION -> ABORTABLE_ERROR is valid");
7768
7769 let count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
7770 let cb_count = Arc::clone(&count);
7771 let callback: Box<dyn Callback> =
7772 Box::new(move |_metadata: Option<&RecordMetadata>, _error: Option<&Error>| {
7773 cb_count.fetch_add(1, Ordering::SeqCst);
7774 });
7775
7776 // The append succeeds; `maybeAddPartition` then raises `ProducerFenced`,
7777 // taking the `handle_api_error(.., None)` arm.
7778 let future = ctx
7779 .producer
7780 .send_with_callback(misuse_record(), Some(callback))
7781 .await
7782 .expect("an ApiException is reported through the future, not the call");
7783 let error = future.get().await.expect_err("the fenced send cannot be acked");
7784 assert_eq!(error.error(), Errors::ProducerFenced, "got {error:?}");
7785
7786 // `handle_api_error` was passed `None`, so it did NOT fire the callback.
7787 let count_after_send = count.load(Ordering::SeqCst);
7788
7789 // The record still sits in the accumulator carrying the callback; the
7790 // accumulator fires it exactly once when the batch is aborted (as the
7791 // Sender's abort/close path would).
7792 ctx.accumulator.abort_incomplete_batches().await;
7793 (count_after_send, count.load(Ordering::SeqCst))
7794 });
7795
7796 assert_eq!(
7797 count_after_send, 0,
7798 "the maybeAddPartition ApiException path must NOT fire the callback (it is deferred to \
7799 the accumulator); firing it here would double-fire"
7800 );
7801 assert_eq!(
7802 count_final, 1,
7803 "the callback fires exactly once, from the accumulator when the batch is aborted"
7804 );
7805 }
7806
7807 /// Translated from
7808 /// `KafkaProducerTest.testCommitTransactionWithRecordTooLargeException`.
7809 ///
7810 /// A record larger than `max.request.size` fails its future with
7811 /// `RecordTooLargeException`, and because `doSend`'s `catch (ApiException e)` runs
7812 /// `maybeTransitionToErrorState` (`KafkaProducer#doSend`) the transaction
7813 /// is now abortable — so the following `commitTransaction` must fail rather than
7814 /// commit a partial transaction.
7815 #[tokio::test]
7816 #[doc(
7817 alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testCommitTransactionWithRecordTooLargeException"
7818 )]
7819 async fn test_commit_transaction_with_record_too_large_error() {
7820 let mut ctx =
7821 TxnProducerContext::new(&[("transactional.id", TRANSACTIONAL_ID), ("max.request.size", "1000")], 1);
7822 ctx.time.set_auto_tick_ms(1);
7823 init_transactions(&mut ctx).await;
7824 ctx.producer.begin_transaction().expect("beginTransaction");
7825
7826 let large_string = "*".repeat(1000);
7827 let record = ProducerRecord::with_key(TOPIC.to_string(), Some("large string".to_string()), Some(large_string));
7828 let future = ctx
7829 .producer
7830 .send(record)
7831 .await
7832 .expect("an API error is reported through the future, not the call");
7833 let send_error = future.get().await.expect_err("the record is too large");
7834 assert!(
7835 matches!(send_error, Error::RecordTooLarge(_)),
7836 "expected RecordTooLarge, got {:?}",
7837 send_error
7838 );
7839
7840 // Java asserts a bare `KafkaException`, which is what `maybeFailWithError`
7841 // (`TransactionManager#maybeFailWithError`) raises for a non-`IllegalStateException`
7842 // `lastError` — the original cause is wrapped, not re-raised.
7843 let commit_error = drive(&mut ctx.sender, ctx.producer.commit_transaction())
7844 .await
7845 .expect_err("the transaction is abortable after a failed send");
7846 assert_eq!(
7847 commit_error.message(),
7848 "Cannot execute transactional method because we are in an error state"
7849 );
7850 }
7851
7852 /// Translated from
7853 /// `KafkaProducerTest.testCommitTransactionWithMetadataTimeoutForMissingTopic`.
7854 ///
7855 /// # Deviation: how the metadata wait is made to expire
7856 ///
7857 /// Java stubs `metadata.fetch()` with Mockito to return an *empty* cluster and, on
7858 /// the sixth invocation, to jump `MockTime` forward by 70 s past a
7859 /// `max.block.ms` of 60 s. `ProducerMetadata` is a concrete type here, so there is
7860 /// no stub to install; the equivalent is a metadata instance that genuinely has no
7861 /// topic plus a `max.block.ms` short enough to expire in test time. What the test
7862 /// observes is unchanged: the send's future fails with a timeout, and the
7863 /// subsequent `commitTransaction` fails because the failed send made the
7864 /// transaction abortable.
7865 #[tokio::test]
7866 #[doc(
7867 alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testCommitTransactionWithMetadataTimeoutForMissingTopic"
7868 )]
7869 async fn test_commit_transaction_with_metadata_timeout_for_missing_topic() {
7870 let mut ctx = TxnProducerContext::new(
7871 &[("transactional.id", TRANSACTIONAL_ID), ("max.block.ms", "200")],
7872 // No partitions: `metadataUpdateWith(1, emptyMap())`, Java's `emptyCluster`.
7873 0,
7874 );
7875 init_transactions(&mut ctx).await;
7876 ctx.producer.begin_transaction().expect("beginTransaction");
7877
7878 let record = ProducerRecord::new(TOPIC.to_string(), Some("value".to_string()));
7879 let future = ctx
7880 .producer
7881 .send(record)
7882 .await
7883 .expect("a timeout is an API error and is reported through the future");
7884 let send_error = future.get().await.expect_err("the topic never appears in metadata");
7885 assert!(
7886 matches!(send_error, Error::Timeout(_)),
7887 "expected Timeout, got {:?}",
7888 send_error
7889 );
7890
7891 // Java asserts a bare `KafkaException` — see
7892 // `test_commit_transaction_with_record_too_large_error`.
7893 let commit_error = drive(&mut ctx.sender, ctx.producer.commit_transaction())
7894 .await
7895 .expect_err("the transaction is abortable after a failed send");
7896 assert_eq!(
7897 commit_error.message(),
7898 "Cannot execute transactional method because we are in an error state"
7899 );
7900 }
7901
7902 /// Translated from
7903 /// `KafkaProducerTest.testCommitTransactionWithMetadataTimeoutForPartitionOutOfRange`.
7904 ///
7905 /// As the previous test, but the metadata does contain the topic — with one
7906 /// partition — and the record names partition 2, so `waitOnMetadata` waits for a
7907 /// partition that never arrives. Same deviation on how the wait is made to expire.
7908 #[tokio::test]
7909 #[doc(
7910 alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testCommitTransactionWithMetadataTimeoutForPartitionOutOfRange"
7911 )]
7912 async fn test_commit_transaction_with_metadata_timeout_for_partition_out_of_range() {
7913 let mut ctx = TxnProducerContext::new(&[("transactional.id", TRANSACTIONAL_ID), ("max.block.ms", "200")], 1);
7914 init_transactions(&mut ctx).await;
7915 ctx.producer.begin_transaction().expect("beginTransaction");
7916
7917 let record = ProducerRecord::with_partition_key(TOPIC.to_string(), Some(2), None, Some("value".to_string()))
7918 .expect("a valid partition");
7919 let future = ctx
7920 .producer
7921 .send(record)
7922 .await
7923 .expect("a timeout is an API error and is reported through the future");
7924 let send_error = future.get().await.expect_err("partition 2 never appears in metadata");
7925 assert!(
7926 matches!(send_error, Error::Timeout(_)),
7927 "expected Timeout, got {:?}",
7928 send_error
7929 );
7930
7931 // Java asserts a bare `KafkaException` — see
7932 // `test_commit_transaction_with_record_too_large_error`.
7933 let commit_error = drive(&mut ctx.sender, ctx.producer.commit_transaction())
7934 .await
7935 .expect_err("the transaction is abortable after a failed send");
7936 assert_eq!(
7937 commit_error.message(),
7938 "Cannot execute transactional method because we are in an error state"
7939 );
7940 }
7941
7942 /// Translated from
7943 /// `KafkaProducerTest.testCommitTransactionWithSendToInvalidTopic`.
7944 ///
7945 /// An invalid topic name fails the send's future with `InvalidTopicException` and
7946 /// leaves the transaction abortable, so the commit fails.
7947 ///
7948 /// Java arranges the invalid topic through `client.prepareMetadataUpdate(..)`, a
7949 /// `MockClient` facility this port does not have; the metadata is seeded with the
7950 /// `INVALID_TOPIC_EXCEPTION` topic directly instead, which is the state that
7951 /// update would have produced.
7952 #[tokio::test]
7953 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testCommitTransactionWithSendToInvalidTopic")]
7954 async fn test_commit_transaction_with_send_to_invalid_topic() {
7955 use crate::MetadataResponseData;
7956 use crate::common::protocol::ApiKeys;
7957 use crate::common::requests::MetadataResponse;
7958 use crate::metadata_response_data::{MetadataResponseBroker, MetadataResponseTopic};
7959
7960 const INVALID_TOPIC: &str = "topic abc"; // Invalid topic name due to space.
7961
7962 let mut ctx = TxnProducerContext::new(&[("transactional.id", TRANSACTIONAL_ID), ("max.block.ms", "15000")], 1);
7963 init_transactions(&mut ctx).await;
7964 ctx.producer.begin_transaction().expect("beginTransaction");
7965
7966 let mut data = MetadataResponseData::new();
7967 data.set_controller_id(0);
7968 data.set_cluster_id(Some("test-cluster".to_string()));
7969 let mut broker = MetadataResponseBroker::new();
7970 broker.set_node_id(0);
7971 broker.set_host("localhost".to_string());
7972 broker.set_port(9092);
7973 data.set_brokers(vec![broker]);
7974 let mut invalid = MetadataResponseTopic::new();
7975 invalid.set_name(Some(INVALID_TOPIC.to_string()));
7976 invalid.set_error_code(Errors::InvalidTopicError.code());
7977 data.set_topics(vec![invalid]);
7978 let response = MetadataResponse::with_version(data, ApiKeys::METADATA.latest_version());
7979 ctx.metadata.add(INVALID_TOPIC, ctx.time.milliseconds());
7980 ctx.metadata
7981 .update_with_current_request_version(&response, false, ctx.time.milliseconds());
7982
7983 let record = ProducerRecord::new(INVALID_TOPIC.to_string(), Some("HelloKafka".to_string()));
7984 let future = ctx
7985 .producer
7986 .send(record)
7987 .await
7988 .expect("an invalid-topic error is reported through the future");
7989 let send_error = future.get().await.expect_err("the topic name is invalid");
7990 assert!(
7991 matches!(send_error, Error::InvalidTopic(_)),
7992 "expected InvalidTopic, got {:?}",
7993 send_error
7994 );
7995
7996 // Java asserts a bare `KafkaException` — see
7997 // `test_commit_transaction_with_record_too_large_error`.
7998 let commit_error = drive(&mut ctx.sender, ctx.producer.commit_transaction())
7999 .await
8000 .expect_err("the transaction is abortable after a failed send");
8001 assert_eq!(
8002 commit_error.message(),
8003 "Cannot execute transactional method because we are in an error state"
8004 );
8005 }
8006
8007 /// Translated from `KafkaProducerTest.testSendTxnOffsetsWithGroupId`.
8008 ///
8009 /// The offsets map Java passes is **empty**, so `sendOffsetsToTransaction` takes
8010 /// `KafkaProducer#sendOffsetsToTransaction`'s early exit and sends nothing at all: the
8011 /// `AddOffsetsToTxn`, second `FindCoordinator` and `TxnOffsetCommit` responses the
8012 /// Java test queues are never consumed, and only the `EndTxn` is. That is
8013 /// preserved rather than "fixed" — `testSendTxnOffsetsWithGroupIdTransactionV2`
8014 /// below is the sibling that passes a real offset.
8015 ///
8016 /// What the test does cover is that an empty map is a no-op even while the
8017 /// coordinator is throttled, and that the commit that follows succeeds.
8018 #[tokio::test]
8019 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testSendTxnOffsetsWithGroupId")]
8020 async fn test_send_txn_offsets_with_group_id() {
8021 let mut ctx = TxnProducerContext::new(&[("transactional.id", TRANSACTIONAL_ID), ("max.block.ms", "10000")], 1);
8022 ctx.time.set_auto_tick_ms(1);
8023 let node = coordinator_node();
8024 ctx.sender.client_mut().throttle(&node, 5000);
8025 init_transactions(&mut ctx).await;
8026 ctx.producer.begin_transaction().expect("beginTransaction");
8027
8028 let group_metadata = ConsumerGroupMetadataImpl::new("group");
8029 let sent_before = ctx.sender.client().request_count();
8030 drive(
8031 &mut ctx.sender,
8032 ctx.producer.send_offsets_to_transaction(HashMap::new(), &group_metadata),
8033 )
8034 .await
8035 .expect("an empty offsets map is a no-op");
8036 assert_eq!(
8037 ctx.sender.client().request_count(),
8038 sent_before,
8039 "an empty offsets map returns from `KafkaProducer#sendOffsetsToTransaction` before touching \
8040 the transaction state"
8041 );
8042
8043 ctx.sender.client_mut().prepare_response(end_txn_response(Errors::None));
8044 drive(&mut ctx.sender, ctx.producer.commit_transaction())
8045 .await
8046 .expect("commitTransaction");
8047 }
8048
8049 /// Translated from `KafkaProducerTest.testSendTxnOffsetsWithGroupMetadata`.
8050 ///
8051 /// Java's offsets map here is empty too, so as in
8052 /// [`test_send_txn_offsets_with_group_id`] nothing is sent; the group metadata it
8053 /// builds carries a generation id and member id, which is what the request matcher
8054 /// would have checked had a request gone out. The check that *is* reachable is that
8055 /// a fully populated `ConsumerGroupMetadata` passes
8056 /// `throwIfInvalidGroupMetadata` — `generationId > 0` **with** a known member id.
8057 #[tokio::test]
8058 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testSendTxnOffsetsWithGroupMetadata")]
8059 async fn test_send_txn_offsets_with_group_metadata() {
8060 let mut ctx = TxnProducerContext::new(&[("transactional.id", TRANSACTIONAL_ID), ("max.block.ms", "10000")], 1);
8061 ctx.time.set_auto_tick_ms(1);
8062 let node = coordinator_node();
8063 ctx.sender.client_mut().throttle(&node, 5000);
8064 init_transactions(&mut ctx).await;
8065 ctx.producer.begin_transaction().expect("beginTransaction");
8066
8067 let group_metadata = ConsumerGroupMetadataImpl::with_generation_id_member_id_group_instance_id(
8068 "group",
8069 5,
8070 "member",
8071 Some("instance".to_string()),
8072 );
8073 drive(
8074 &mut ctx.sender,
8075 ctx.producer.send_offsets_to_transaction(HashMap::new(), &group_metadata),
8076 )
8077 .await
8078 .expect("a populated group metadata is valid and an empty offsets map is a no-op");
8079
8080 ctx.sender.client_mut().prepare_response(end_txn_response(Errors::None));
8081 drive(&mut ctx.sender, ctx.producer.commit_transaction())
8082 .await
8083 .expect("commitTransaction");
8084 }
8085
8086 /// Translated from
8087 /// `KafkaProducerTest.testInvalidGenerationIdAndMemberIdCombinedInSendOffsets`, which calls the `verifyInvalidGroupMetadata` helper
8088 /// with `new ConsumerGroupMetadata("group", 2, UNKNOWN_MEMBER_ID,
8089 /// Optional.empty())`.
8090 ///
8091 /// `KafkaProducerTest.testNullGroupMetadataInSendOffsets` is the
8092 /// other caller of that helper, passing `null`. It is **not translated**: the
8093 /// parameter is a `ConsumerGroupMetadata` value in Rust, so a null cannot be
8094 /// constructed and the arm it exercises
8095 /// (`KafkaProducer`) is enforced by the type system rather than by
8096 /// a runtime check. This is the same reasoning that dropped the arm from
8097 /// [`KafkaProducer::throw_if_invalid_group_metadata`].
8098 #[tokio::test]
8099 #[doc(
8100 alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testInvalidGenerationIdAndMemberIdCombinedInSendOffsets"
8101 )]
8102 async fn test_invalid_generation_id_and_member_id_combined_in_send_offsets() {
8103 let mut ctx = TxnProducerContext::new(&[("transactional.id", TRANSACTIONAL_ID), ("max.block.ms", "10000")], 1);
8104 ctx.time.set_auto_tick_ms(1);
8105 let node = coordinator_node();
8106 ctx.sender.client_mut().throttle(&node, 5000);
8107 init_transactions(&mut ctx).await;
8108 ctx.producer.begin_transaction().expect("beginTransaction");
8109
8110 let group_metadata = ConsumerGroupMetadataImpl::with_generation_id_member_id_group_instance_id(
8111 "group",
8112 2,
8113 crate::common::requests::TxnOffsetCommitRequest::UNKNOWN_MEMBER_ID,
8114 None,
8115 );
8116 let error = ctx
8117 .producer
8118 .send_offsets_to_transaction(HashMap::new(), &group_metadata)
8119 .await
8120 .expect_err("generationId > 0 with an unknown member id is rejected");
8121 assert_eq!(
8122 error.message(),
8123 format!(
8124 "Passed in group metadata {} has generationId > 0 but the member.id is unknown",
8125 group_metadata
8126 )
8127 );
8128 }
8129
8130 /// A `ProduceResponse` for one partition, mirroring
8131 /// `KafkaProducerTest.produceResponse(TopicIdPartition, long, Errors, int, int)`.
8132 fn produce_response(partition: i32, base_offset: i64, error: Errors, log_start_offset: i64) -> ConcreteResponse {
8133 use crate::ProduceResponseData;
8134 use crate::common::requests::ProduceResponse;
8135 use crate::produce_response_data::{PartitionProduceResponse, TopicProduceResponse};
8136
8137 let mut partition_response = PartitionProduceResponse::new();
8138 partition_response.set_index(partition);
8139 partition_response.set_base_offset(base_offset);
8140 partition_response.set_error_code(error.code());
8141 partition_response.set_log_start_offset(log_start_offset);
8142
8143 let mut topic_response = TopicProduceResponse::new();
8144 topic_response.set_topic_id(topic_id());
8145 topic_response.set_name(TOPIC.to_string());
8146 topic_response.set_partition_responses(vec![partition_response]);
8147
8148 let mut data = ProduceResponseData::new();
8149 data.set_responses(vec![topic_response]);
8150 ConcreteResponse::Produce(ProduceResponse::new(data))
8151 }
8152
8153 /// Java's `addOffsetsToTxnResponse(Errors error)`
8154 /// (`KafkaProducerTest#addOffsetsToTxnResponse`).
8155 fn add_offsets_to_txn_response(error: Errors) -> ConcreteResponse {
8156 use crate::AddOffsetsToTxnResponseData;
8157 use crate::common::requests::AddOffsetsToTxnResponse;
8158
8159 let mut data = AddOffsetsToTxnResponseData::new();
8160 data.set_error_code(error.code()).set_throttle_time_ms(10);
8161 ConcreteResponse::AddOffsetsToTxn(AddOffsetsToTxnResponse::new(data))
8162 }
8163
8164 /// Java's `txnOffsetsCommitResponse(Map<TopicPartition, Errors>)`
8165 /// (`KafkaProducerTest#txnOffsetsCommitResponse`).
8166 fn txn_offsets_commit_response(errors: &[(TopicPartition, Errors)]) -> ConcreteResponse {
8167 use crate::common::requests::TxnOffsetCommitResponse;
8168
8169 let error_map: HashMap<TopicPartition, Errors> = errors.iter().cloned().collect();
8170 ConcreteResponse::TxnOffsetCommit(TxnOffsetCommitResponse::with_request_throttle_ms_response_data(
8171 10, &error_map,
8172 ))
8173 }
8174
8175 /// Translated from `KafkaProducerTest.testTransactionV2Produce`.
8176 ///
8177 /// The full Transaction V2 round trip: `FindCoordinator`, `InitProducerId`, one
8178 /// produce, `EndTxn`. No `AddPartitionsToTxn` is prepared or expected — under
8179 /// KIP-890 the broker adds the partition implicitly, which is exactly what
8180 /// `maybeAddPartition`'s V2 arm (`TransactionManager#maybeAddPartition`) relies on.
8181 #[tokio::test]
8182 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testTransactionV2Produce")]
8183 async fn test_transaction_v2_produce() {
8184 let mut ctx = TxnProducerContext::transactional_v2(&[("transactional.id", "some-txn")], 1);
8185 ctx.time.set_auto_tick_ms(1);
8186 let node = coordinator_node();
8187 ctx.sender
8188 .client_mut()
8189 .prepare_response(find_coordinator_response(Errors::None, "some-txn", &node));
8190 ctx.sender
8191 .client_mut()
8192 .prepare_response(init_producer_id_response(Errors::None, PRODUCER_ID, EPOCH));
8193 drive(&mut ctx.sender, ctx.producer.init_transactions())
8194 .await
8195 .expect("initTransactions");
8196 assert!(
8197 ctx.transaction_manager.lock().unwrap().is_transaction_v2_enabled(),
8198 "initTransactions ends with maybeUpdateTransactionV2Enabled(true)"
8199 );
8200
8201 ctx.producer.begin_transaction().expect("beginTransaction");
8202 ctx.sender
8203 .client_mut()
8204 .prepare_response(produce_response(0, 1, Errors::None, 0));
8205 ctx.sender.client_mut().prepare_response(end_txn_response(Errors::None));
8206
8207 let record = ProducerRecord::with_partition_key(
8208 TOPIC.to_string(),
8209 Some(0),
8210 Some("key".to_string()),
8211 Some("value".to_string()),
8212 )
8213 .expect("a valid partition");
8214 let future = ctx.producer.send(record).await.expect("send");
8215 let metadata = drive(&mut ctx.sender, future.get()).await.expect("the produce succeeds");
8216 assert_eq!(metadata.offset(), 1);
8217
8218 drive(&mut ctx.sender, ctx.producer.commit_transaction())
8219 .await
8220 .expect("commitTransaction");
8221 }
8222
8223 /// Translated from
8224 /// `KafkaProducerTest.testTransactionV2ProduceWithConcurrentTransactionError`.
8225 ///
8226 /// As [`test_transaction_v2_produce`], but the first produce is answered with
8227 /// `CONCURRENT_TRANSACTIONS`. That is retriable, so the batch is re-enqueued and
8228 /// the second response completes it — which is what makes the commit that follows
8229 /// succeed rather than fail on an abortable error.
8230 #[tokio::test]
8231 #[doc(
8232 alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testTransactionV2ProduceWithConcurrentTransactionError"
8233 )]
8234 async fn test_transaction_v2_produce_with_concurrent_transaction_error() {
8235 let mut ctx = TxnProducerContext::transactional_v2(&[("transactional.id", "some-txn")], 1);
8236 ctx.time.set_auto_tick_ms(1);
8237 let node = coordinator_node();
8238 ctx.sender
8239 .client_mut()
8240 .prepare_response(find_coordinator_response(Errors::None, "some-txn", &node));
8241 ctx.sender
8242 .client_mut()
8243 .prepare_response(init_producer_id_response(Errors::None, PRODUCER_ID, EPOCH));
8244 drive(&mut ctx.sender, ctx.producer.init_transactions())
8245 .await
8246 .expect("initTransactions");
8247 ctx.producer.begin_transaction().expect("beginTransaction");
8248
8249 ctx.sender
8250 .client_mut()
8251 .prepare_response(produce_response(0, 1, Errors::ConcurrentTransactions, 0));
8252 ctx.sender
8253 .client_mut()
8254 .prepare_response(produce_response(0, 1, Errors::None, 0));
8255 ctx.sender.client_mut().prepare_response(end_txn_response(Errors::None));
8256
8257 let record = ProducerRecord::with_partition_key(
8258 TOPIC.to_string(),
8259 Some(0),
8260 Some("key".to_string()),
8261 Some("value".to_string()),
8262 )
8263 .expect("a valid partition");
8264 let future = ctx.producer.send(record).await.expect("send");
8265 let metadata = drive(&mut ctx.sender, future.get()).await.expect("the retry succeeds");
8266 assert_eq!(metadata.offset(), 1);
8267
8268 drive(&mut ctx.sender, ctx.producer.commit_transaction())
8269 .await
8270 .expect("commitTransaction");
8271 }
8272
8273 /// Translated from `KafkaProducerTest.testSendTxnOffsetsWithGroupIdTransactionV2`.
8274 ///
8275 /// With Transaction V2 the client skips `AddOffsetsToTxn` and sends the
8276 /// `TxnOffsetCommit` straight away (`TransactionManager`), after a
8277 /// `FindCoordinator` for the *group* coordinator. Java's prepared sequence says the
8278 /// same thing: `FindCoordinator`, `InitProducerId`, `FindCoordinator`,
8279 /// `TxnOffsetCommit`, `EndTxn` — five responses with no `AddOffsetsToTxn` between
8280 /// the second and third, unlike its V1 sibling.
8281 #[tokio::test]
8282 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testSendTxnOffsetsWithGroupIdTransactionV2")]
8283 async fn test_send_txn_offsets_with_group_id_transaction_v2() {
8284 let mut ctx = TxnProducerContext::transactional_v2(
8285 &[("transactional.id", TRANSACTIONAL_ID), ("max.block.ms", "10000")],
8286 1,
8287 );
8288 ctx.time.set_auto_tick_ms(1);
8289 let node = coordinator_node();
8290 ctx.sender.client_mut().throttle(&node, 5000);
8291 ctx.prepare_init_transactions(Errors::None, PRODUCER_ID, EPOCH);
8292 drive(&mut ctx.sender, ctx.producer.init_transactions())
8293 .await
8294 .expect("initTransactions");
8295 assert!(ctx.transaction_manager.lock().unwrap().is_transaction_v2_enabled());
8296 ctx.producer.begin_transaction().expect("beginTransaction");
8297
8298 const GROUP_ID: &str = "group";
8299 let partition = TopicPartition::new(TOPIC.to_string(), 0);
8300 ctx.sender
8301 .client_mut()
8302 .prepare_response(find_coordinator_response(Errors::None, GROUP_ID, &node));
8303 ctx.sender
8304 .client_mut()
8305 .prepare_response(txn_offsets_commit_response(&[(partition.clone(), Errors::None)]));
8306 ctx.sender.client_mut().prepare_response(end_txn_response(Errors::None));
8307
8308 let group_metadata = ConsumerGroupMetadataImpl::new(GROUP_ID);
8309 let offsets = HashMap::from([(partition, OffsetAndMetadata::new(5).expect("a non-negative offset"))]);
8310 drive(
8311 &mut ctx.sender,
8312 ctx.producer.send_offsets_to_transaction(offsets, &group_metadata),
8313 )
8314 .await
8315 .expect("sendOffsetsToTransaction");
8316
8317 drive(&mut ctx.sender, ctx.producer.commit_transaction())
8318 .await
8319 .expect("commitTransaction");
8320 }
8321
8322 /// What Java's `MockClient.poll` does for metadata (`MockClient#poll`):
8323 /// whenever an update is requested, apply `update` — the queued
8324 /// `prepareMetadataUpdate` response, or the current snapshot standing in for
8325 /// `updateWithCurrentMetadata`. This crate's `MockClient` has no metadata
8326 /// updater, so the tests that need one run this beside the operation.
8327 async fn with_metadata_responder<T>(
8328 metadata: &ProducerMetadata,
8329 time: &MockTime,
8330 update: &crate::common::requests::MetadataResponse,
8331 op: impl std::future::Future<Output = T>,
8332 ) -> T {
8333 let done = AtomicBool::new(false);
8334 let op = async {
8335 let out = op.await;
8336 done.store(true, Ordering::SeqCst);
8337 out
8338 };
8339 let responder = async {
8340 while !done.load(Ordering::SeqCst) {
8341 if metadata.update_requested() {
8342 metadata.update_with_current_request_version(update, false, time.milliseconds());
8343 }
8344 tokio::task::yield_now().await;
8345 }
8346 };
8347 tokio::join!(op, responder).0
8348 }
8349
8350 /// The shared body of the two KIP-1319 `KafkaProducerTest` cases (6208dfc014,
8351 /// with 930ebc5608's frozen `MockTime`): a Transaction V2 producer whose
8352 /// metadata does not yet track [`TOPIC`] sends offsets for it. The refresh that
8353 /// `sendOffsetsToTransaction` triggers returns `refreshed`; the `TxnOffsetCommit`
8354 /// must then be at v6+ and carry the topic id.
8355 async fn send_offsets_to_transaction_negotiates_v6(initial_has_topic_id: bool) {
8356 let mut ctx = TxnProducerContext::with_metadata_seed(
8357 &[("transactional.id", "some.id"), ("max.block.ms", "10000")],
8358 1,
8359 true,
8360 false,
8361 initial_has_topic_id,
8362 );
8363 const GROUP_ID: &str = "group";
8364 let node = coordinator_node();
8365 let partition = TopicPartition::new(TOPIC.to_string(), 0);
8366
8367 ctx.sender
8368 .client_mut()
8369 .prepare_response(find_coordinator_response(Errors::None, "some.id", &node));
8370 ctx.sender
8371 .client_mut()
8372 .prepare_response(init_producer_id_response(Errors::None, 1, 5));
8373 ctx.sender
8374 .client_mut()
8375 .prepare_response(find_coordinator_response(Errors::None, "some.id", &node));
8376 ctx.sender.client_mut().prepare_response_matcher(
8377 Box::new(|request| {
8378 let crate::common::requests::AbstractRequest::TxnOffsetCommit(request) = request else {
8379 panic!("expected a TxnOffsetCommit request, got {request}");
8380 };
8381 assert_eq!(request.data().group_id, GROUP_ID);
8382 assert!(
8383 request.version() >= 6,
8384 "Expected TxnOffsetCommit at v6+, got {}",
8385 request.version()
8386 );
8387 assert_eq!(request.data().topics.len(), 1);
8388 assert_eq!(request.data().topics[0].topic_id, topic_id());
8389 true
8390 }),
8391 txn_offsets_commit_response(&[(partition.clone(), Errors::None)]),
8392 );
8393 ctx.sender.client_mut().prepare_response(end_txn_response(Errors::None));
8394
8395 drive(&mut ctx.sender, ctx.producer.init_transactions())
8396 .await
8397 .expect("initTransactions");
8398 ctx.producer.begin_transaction().expect("beginTransaction");
8399 assert!(!ctx.metadata.contains_topic(TOPIC), "the topic starts out untracked");
8400
8401 let refreshed = topic_metadata_update(1, true);
8402 let group_metadata = ConsumerGroupMetadataImpl::new(GROUP_ID);
8403 let offsets = HashMap::from([(partition, OffsetAndMetadata::new(5).expect("a non-negative offset"))]);
8404 let metadata = Arc::clone(&ctx.metadata);
8405 let time = Arc::clone(&ctx.time);
8406 drive(
8407 &mut ctx.sender,
8408 with_metadata_responder(
8409 &metadata,
8410 &time,
8411 &refreshed,
8412 ctx.producer.send_offsets_to_transaction(offsets, &group_metadata),
8413 ),
8414 )
8415 .await
8416 .expect("sendOffsetsToTransaction");
8417 assert!(ctx.metadata.contains_topic(TOPIC), "awaitTopicMetadata tracks the topic");
8418
8419 drive(&mut ctx.sender, ctx.producer.commit_transaction())
8420 .await
8421 .expect("commitTransaction");
8422 }
8423
8424 /// Translated from
8425 /// `KafkaProducerTest.testSendOffsetsToTransactionNegotiatesV6WhenMetadataKnowsTopicId`.
8426 #[tokio::test]
8427 #[doc(
8428 alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testSendOffsetsToTransactionNegotiatesV6WhenMetadataKnowsTopicId"
8429 )]
8430 async fn test_send_offsets_to_transaction_negotiates_v6_when_metadata_knows_topic_id() {
8431 send_offsets_to_transaction_negotiates_v6(true).await;
8432 }
8433
8434 /// Translated from
8435 /// `KafkaProducerTest.testSendOffsetsToTransactionTriggersMetadataRefreshThenNegotiatesV6`:
8436 /// the initial snapshot has the topic but no id; only the refresh
8437 /// `sendOffsetsToTransaction` triggers supplies it.
8438 #[tokio::test]
8439 #[doc(
8440 alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testSendOffsetsToTransactionTriggersMetadataRefreshThenNegotiatesV6"
8441 )]
8442 async fn test_send_offsets_to_transaction_triggers_metadata_refresh_then_negotiates_v6() {
8443 send_offsets_to_transaction_negotiates_v6(false).await;
8444 }
8445
8446 /// `awaitTopicMetadata` waits at most `max.block.ms` for the refresh it
8447 /// requests, and reports Java's `awaitUpdate` timeout when none arrives.
8448 ///
8449 /// `max.block.ms` also bounds the `initTransactions` that precedes the
8450 /// wait, and `drive` runs on real time: 500 ms, as the other tests that
8451 /// wait a timeout out use, leaves the mock round trips headroom when the
8452 /// whole suite runs under load, where 100 ms timed `initTransactions` out.
8453 #[tokio::test]
8454 async fn test_send_offsets_to_transaction_times_out_waiting_for_topic_metadata() {
8455 let mut ctx = TxnProducerContext::with_metadata_seed(
8456 &[("transactional.id", "some.id"), ("max.block.ms", "500")],
8457 1,
8458 true,
8459 false,
8460 false,
8461 );
8462 let node = coordinator_node();
8463 ctx.sender
8464 .client_mut()
8465 .prepare_response(find_coordinator_response(Errors::None, "some.id", &node));
8466 ctx.sender
8467 .client_mut()
8468 .prepare_response(init_producer_id_response(Errors::None, 1, 5));
8469 drive(&mut ctx.sender, ctx.producer.init_transactions())
8470 .await
8471 .expect("initTransactions");
8472 ctx.producer.begin_transaction().expect("beginTransaction");
8473
8474 let offsets = HashMap::from([(
8475 TopicPartition::new(TOPIC.to_string(), 0),
8476 OffsetAndMetadata::new(5).expect("a non-negative offset"),
8477 )]);
8478 let error = drive(
8479 &mut ctx.sender,
8480 ctx.producer
8481 .send_offsets_to_transaction(offsets, &ConsumerGroupMetadataImpl::new("group")),
8482 )
8483 .await
8484 .expect_err("no metadata update arrives");
8485 assert!(error.is_timeout_error(), "{error:?}");
8486 assert_eq!(error.message(), "Failed to update metadata after 500 ms.");
8487 assert!(
8488 !ctx.transaction_manager.lock().unwrap().has_pending_offset_commits(),
8489 "the manager is not reached when the metadata wait fails"
8490 );
8491 }
8492
8493 /// Reads a producer-level metric's current value. Translated from
8494 /// `KafkaProducerTest#getMetricValue`:
8495 /// `metrics.metric(metrics.metricName(name, "producer-metrics")).metricValue()`.
8496 fn get_metric_value(producer: &KafkaProducer<String, String>, name: &str) -> f64 {
8497 // `metric_value()` is a `Metric`-trait method (Java's `Metric.metricValue()`).
8498 use crate::common::Metric;
8499 let mn = producer.metrics.metric_name(name, "producer-metrics");
8500 producer
8501 .metrics
8502 .metric(&mn)
8503 .expect("metric present")
8504 .metric_value()
8505 .as_double()
8506 .expect("a double metric value")
8507 }
8508
8509 /// Translated from `KafkaProducerTest#getAndAssertDurationAtLeast`: reads the duration metric, asserts it is at least `floor`, and
8510 /// returns it.
8511 ///
8512 /// The producer times transaction control with its injected `time`, as Java
8513 /// does, so a test that installs `MockTime` with an auto-tick gets Java's
8514 /// durations: every `time.nanoseconds()` read advances the clock by one tick,
8515 /// and callers pass Java's `tick.toNanos()` floors unchanged.
8516 fn get_and_assert_duration_at_least(producer: &KafkaProducer<String, String>, name: &str, floor: f64) -> f64 {
8517 let value = get_metric_value(producer, name);
8518 assert!(value >= floor, "{name} duration {value} is below floor {floor}");
8519 value
8520 }
8521
8522 /// Translated from `KafkaProducerTest.assertDurationAtLeast`:
8523 /// asserts the duration is at least `floor`, discarding the value.
8524 fn assert_duration_at_least(producer: &KafkaProducer<String, String>, name: &str, floor: f64) {
8525 get_and_assert_duration_at_least(producer, name, floor);
8526 }
8527
8528 /// Translated from `KafkaProducerTest.testMeasureAbortTransactionDuration`.
8529 ///
8530 /// Runs two complete `beginTransaction` / `abortTransaction` cycles over one
8531 /// `initTransactions` — which proves `abortTransaction` leaves the manager in a
8532 /// state a *second* transaction can start from — and asserts, as Java does, that
8533 /// `txn-abort-time-ns-total` is positive after the first abort and strictly larger
8534 /// after the second. Java's `assertTrue(first > 0)` / `assertTrue(second > first)`
8535 /// carry over unchanged: the producer times with the injected `MockTime`, whose
8536 /// one-millisecond auto-tick (Java's `new MockTime(1)`) advances every read, so
8537 /// each abort records a strictly positive duration and the cumulative total grows.
8538 #[tokio::test]
8539 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testMeasureAbortTransactionDuration")]
8540 async fn test_measure_abort_transaction_duration() {
8541 let mut ctx = TxnProducerContext::transactional();
8542 ctx.time.set_auto_tick_ms(1);
8543 init_transactions(&mut ctx).await;
8544
8545 // Java captures `first` after abort #0 (`first > 0`) and asserts abort #1 is
8546 // `> first`. A zero-initialised `previous` folds both into one strict-growth
8547 // check: attempt 0 asserts `value > 0`, attempt 1 asserts `value > first`.
8548 let mut previous = 0.0_f64;
8549 for attempt in 0..2 {
8550 ctx.sender.client_mut().prepare_response(end_txn_response(Errors::None));
8551 ctx.producer
8552 .begin_transaction()
8553 .unwrap_or_else(|error| panic!("beginTransaction {}: {}", attempt, error));
8554 drive(&mut ctx.sender, ctx.producer.abort_transaction())
8555 .await
8556 .unwrap_or_else(|error| panic!("abortTransaction {}: {}", attempt, error));
8557
8558 let value = get_metric_value(&ctx.producer, "txn-abort-time-ns-total");
8559 assert!(
8560 value > previous,
8561 "txn-abort-time-ns-total must grow after abort {attempt}: {value} <= {previous}"
8562 );
8563 previous = value;
8564 }
8565 }
8566
8567 /// Translated from `KafkaProducerTest.testMeasureTransactionDurations`.
8568 ///
8569 /// Asserts every transaction-control latency sensor Java asserts —
8570 /// `txn-init-time-ns-total`, `txn-begin-time-ns-total`,
8571 /// `txn-send-offsets-time-ns-total`, `txn-commit-time-ns-total` — around a full V1
8572 /// offsets round trip run **twice** over one `initTransactions`: `AddOffsetsToTxn`,
8573 /// a `FindCoordinator` for the group, `TxnOffsetCommit`, `EndTxn` — and, on the
8574 /// second pass, no second `FindCoordinator`, because the group coordinator is
8575 /// already known. That asymmetry is Java's too (the second batch of prepared
8576 /// responses omits it).
8577 ///
8578 /// Round 1 captures each duration and asserts it is at least one tick; round 2
8579 /// asserts it grew by at least one more ([`get_and_assert_duration_at_least`] /
8580 /// [`assert_duration_at_least`]) — Java's `tick.toNanos()` floors, reachable
8581 /// because the producer times with the injected one-second-tick `MockTime`.
8582 ///
8583 /// Kafka 4.4 pre-tracks the topic (`metadata.add("topic", ..)`) so
8584 /// that `sendOffsetsToTransaction`'s KIP-1319 `awaitTopicMetadata` finds it
8585 /// already tracked and does not wait for a refresh, which under the one-second
8586 /// auto-tick would exhaust `max.block.ms`. [`TxnProducerContext::new`] tracks
8587 /// [`TOPIC`] the same way.
8588 #[tokio::test]
8589 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testMeasureTransactionDurations")]
8590 async fn test_measure_transaction_durations() {
8591 let mut ctx = TxnProducerContext::new(&[("transactional.id", TRANSACTIONAL_ID), ("max.block.ms", "10000")], 1);
8592 // Java's `new MockTime(Duration.ofSeconds(1).toMillis())` — a one-second tick
8593 // on every read of the clock the producer times transaction control with.
8594 const TICK_MS: i64 = 1000;
8595 // Java's `tick.toNanos()`.
8596 const TICK_NANOS: f64 = (TICK_MS * 1_000_000) as f64;
8597 ctx.time.set_auto_tick_ms(TICK_MS);
8598 init_transactions(&mut ctx).await;
8599
8600 // Java: assertDurationAtLeast(producer, "txn-init-time-ns-total", tick.toNanos()).
8601 assert_duration_at_least(&ctx.producer, "txn-init-time-ns-total", TICK_NANOS);
8602
8603 const GROUP_ID: &str = "group";
8604 let node = coordinator_node();
8605 let partition = TopicPartition::new(TOPIC.to_string(), 0);
8606
8607 // Round-1 captures; round 2's `first + tick` floors hold because these are
8608 // CumulativeSum sensors — the round-2 total is the round-1 total plus a
8609 // second delta of at least one tick.
8610 let mut begin_first = 0.0_f64;
8611 let mut send_off_first = 0.0_f64;
8612 let mut commit_first = 0.0_f64;
8613
8614 for (attempt, offset) in [(0usize, 5i64), (1, 10)] {
8615 ctx.sender
8616 .client_mut()
8617 .prepare_response(add_offsets_to_txn_response(Errors::None));
8618 if attempt == 0 {
8619 ctx.sender
8620 .client_mut()
8621 .prepare_response(find_coordinator_response(Errors::None, GROUP_ID, &node));
8622 }
8623 ctx.sender
8624 .client_mut()
8625 .prepare_response(txn_offsets_commit_response(&[(partition.clone(), Errors::None)]));
8626 ctx.sender.client_mut().prepare_response(end_txn_response(Errors::None));
8627
8628 ctx.producer
8629 .begin_transaction()
8630 .unwrap_or_else(|error| panic!("beginTransaction {}: {}", attempt, error));
8631 // Java: getAndAssertDurationAtLeast (round 1) / assertDurationAtLeast (round 2).
8632 if attempt == 0 {
8633 begin_first = get_and_assert_duration_at_least(&ctx.producer, "txn-begin-time-ns-total", TICK_NANOS);
8634 } else {
8635 assert_duration_at_least(&ctx.producer, "txn-begin-time-ns-total", begin_first + TICK_NANOS);
8636 }
8637
8638 let group_metadata = ConsumerGroupMetadataImpl::new(GROUP_ID);
8639 let offsets = HashMap::from([(
8640 partition.clone(),
8641 OffsetAndMetadata::new(offset).expect("a non-negative offset"),
8642 )]);
8643 drive(
8644 &mut ctx.sender,
8645 ctx.producer.send_offsets_to_transaction(offsets, &group_metadata),
8646 )
8647 .await
8648 .unwrap_or_else(|error| panic!("sendOffsetsToTransaction {}: {}", attempt, error));
8649 if attempt == 0 {
8650 send_off_first =
8651 get_and_assert_duration_at_least(&ctx.producer, "txn-send-offsets-time-ns-total", TICK_NANOS);
8652 } else {
8653 assert_duration_at_least(&ctx.producer, "txn-send-offsets-time-ns-total", send_off_first + TICK_NANOS);
8654 }
8655
8656 drive(&mut ctx.sender, ctx.producer.commit_transaction())
8657 .await
8658 .unwrap_or_else(|error| panic!("commitTransaction {}: {}", attempt, error));
8659 if attempt == 0 {
8660 commit_first = get_and_assert_duration_at_least(&ctx.producer, "txn-commit-time-ns-total", TICK_NANOS);
8661 } else {
8662 assert_duration_at_least(&ctx.producer, "txn-commit-time-ns-total", commit_first + TICK_NANOS);
8663 }
8664 }
8665 }
8666
8667 /// A producer whose `Sender` is **spawned**, exactly as
8668 /// `KafkaProducer::with_client_options` does in production.
8669 ///
8670 /// Needed by the three `testCloseIsForcedOn*` methods, whose subject is the
8671 /// force-close path in `Sender::run`'s tail (`Sender#run`): it runs only
8672 /// once the run loop itself exits, which driving `run_once` cannot reach. The price
8673 /// is Java's — the `MockClient` moves into the task, so `prepare` queues every
8674 /// response up front and the test cannot inspect the mock afterwards.
8675 fn spawned_transactional_producer(
8676 extra: &[(&str, &str)],
8677 prepare: impl FnOnce(&mut MockClient),
8678 ) -> KafkaProducer<String, String> {
8679 spawned_transactional_producer_with_exit_hook(extra, prepare, None)
8680 }
8681
8682 /// As [`spawned_transactional_producer`], but runs `on_exit` on the Sender's own
8683 /// thread once `Sender::run` has returned.
8684 ///
8685 /// This is what makes "did `close` actually join the Sender?" observable — see
8686 /// [`test_close_joins_the_sender_after_forcing`].
8687 fn spawned_transactional_producer_with_exit_hook(
8688 extra: &[(&str, &str)],
8689 prepare: impl FnOnce(&mut MockClient),
8690 on_exit: Option<Arc<dyn Fn() + Send + Sync>>,
8691 ) -> KafkaProducer<String, String> {
8692 let mut props = HashMap::from([("bootstrap.servers".to_string(), "localhost:9000".to_string())]);
8693 for (key, value) in extra {
8694 props.insert((*key).to_string(), (*value).to_string());
8695 }
8696 let config = ProducerConfig::new(&props).expect("valid config");
8697 let log_context = LogContext::new(format!("[Producer clientId={}] ", config.client_id));
8698 let time = mock_time(1_000);
8699 let api_versions = Arc::new(ApiVersions::new());
8700
8701 let metadata = Arc::new(ProducerMetadata::with_log_context(
8702 config.retry_backoff_ms,
8703 config.retry_backoff_max_ms,
8704 config.metadata_max_age_ms,
8705 config.metadata_max_idle_ms,
8706 ClusterResourceListeners::new(),
8707 log_context.clone(),
8708 ));
8709 let transaction_manager = KafkaProducer::<String, String>::configure_transaction_state(
8710 &config,
8711 &api_versions,
8712 &metadata.metadata_arc(),
8713 &log_context,
8714 )
8715 .expect("these tests always enable idempotence");
8716 metadata.add(TOPIC, time.milliseconds());
8717 let update = crate::common::requests::RequestTestUtils::metadata_update_with(
8718 1,
8719 &HashMap::from([(TOPIC.to_string(), 1)]),
8720 );
8721 metadata.update_with_current_request_version(&update, false, time.milliseconds());
8722
8723 let batch_size = config.batch_size.max(1);
8724 let metrics = Arc::new(Metrics::new());
8725 let buffer_pool = Arc::new(BufferPool::new(
8726 config.buffer_memory,
8727 batch_size as usize,
8728 Arc::clone(&metrics),
8729 Arc::clone(&time) as Arc<dyn Time>,
8730 KafkaProducer::<String, String>::PRODUCER_METRIC_GROUP_NAME,
8731 ));
8732 let accumulator = Arc::new(RecordAccumulator::with_log_context(
8733 batch_size,
8734 Compression::of(config.compression_type).build(),
8735 KafkaProducer::<String, String>::linger_ms(&config),
8736 config.retry_backoff_ms,
8737 config.retry_backoff_max_ms,
8738 config.delivery_timeout_ms,
8739 PartitionerConfig::new(
8740 config.partitioner_adaptive_partitioning_enable,
8741 config.partitioner_availability_timeout_ms,
8742 config.partitioner_rack_aware,
8743 &config.client_rack,
8744 )
8745 .unwrap(),
8746 Arc::clone(&metrics),
8747 KafkaProducer::<String, String>::PRODUCER_METRIC_GROUP_NAME,
8748 Arc::clone(&time) as Arc<dyn Time>,
8749 buffer_pool,
8750 Some(Arc::clone(&transaction_manager)),
8751 log_context,
8752 ));
8753
8754 let mut client = MockClient::with_static_nodes(vec![coordinator_node()], Arc::clone(&time) as Arc<dyn Time>);
8755 prepare(&mut client);
8756 let wakeup = client.wakeup_notify();
8757 let running = Arc::new(AtomicBool::new(true));
8758 let force_close = Arc::new(AtomicBool::new(false));
8759 let pending_requests = Arc::new(Mutex::new(PendingRequests::new()));
8760
8761 let mut sender = Sender::new(
8762 client,
8763 Arc::clone(&metadata),
8764 Arc::clone(&accumulator),
8765 config.max_in_flight_requests_per_connection == 1,
8766 config.max_request_size,
8767 config.acks,
8768 config.retries,
8769 config.request_timeout_ms,
8770 config.retry_backoff_ms,
8771 SenderMetricsRegistry::new(Arc::clone(&metrics)),
8772 Arc::clone(&running),
8773 Arc::clone(&force_close),
8774 Arc::clone(&time) as Arc<dyn Time>,
8775 Some(Arc::clone(&transaction_manager)),
8776 Arc::clone(&pending_requests),
8777 LogContext::empty(),
8778 );
8779
8780 // # Why the Sender gets its own thread and runtime here
8781 //
8782 // `with_client_options` would `tokio::task::spawn` it onto the test's runtime, and that
8783 // deadlocks: `Sender::run` over a `MockClient` never awaits anything that is
8784 // pending — `MockClient::poll` returns immediately — so the task never yields.
8785 // In tokio only a worker parks on the time driver, and the sole awake worker is
8786 // then stuck inside that task, so **no timer in the whole runtime ever fires**:
8787 // the test's own `sleep` never returns and `close` is never called. (Diagnosed
8788 // from a thread sample: the second worker sitting in `park_condvar` while the
8789 // first spun in `run_once`.) A real `NetworkClient` cannot cause this, because
8790 // its `poll` awaits the selector.
8791 //
8792 // `spawn_blocking` plus a private current-thread runtime gives the Sender its
8793 // own OS thread and its own driver — which is also closer to Java, where the
8794 // Sender genuinely *is* a separate thread (`ioThread`). The returned handle is
8795 // still a `tokio::task::JoinHandle<()>`, so `close`'s join works unchanged.
8796 let sender_handle = tokio::task::spawn_blocking(move || {
8797 tokio::runtime::Builder::new_current_thread()
8798 .enable_all()
8799 .build()
8800 .expect("a current-thread runtime for the Sender")
8801 .block_on(sender.run());
8802 if let Some(on_exit) = on_exit {
8803 on_exit();
8804 }
8805 });
8806
8807 KafkaProducer::with_options(
8808 KafkaProducerOptionsBuilder::new()
8809 .set_config(&config)
8810 .set_key_serializer(Box::new(StringSerializer))
8811 .set_value_serializer(Box::new(StringSerializer))
8812 .set_metadata(metadata)
8813 .set_accumulator(accumulator)
8814 .set_running(running)
8815 .set_force_close(force_close)
8816 .set_wakeup(wakeup)
8817 .set_sender_handle(Some(sender_handle))
8818 .set_time(Arc::clone(&time) as Arc<dyn Time>)
8819 .set_transaction_manager(Some(transaction_manager))
8820 .set_pending_requests(pending_requests)
8821 .build()
8822 .expect("KafkaProducerOptionsBuilder::build: every mandatory parameter is set above"),
8823 )
8824 }
8825
8826 /// The body all three `testCloseIsForcedOn*` methods share: start
8827 /// `initTransactions` on another task, let its request go out, then `close` with a
8828 /// one-second timeout and assert `close` **returned** instead of blocking behind the
8829 /// pending request.
8830 ///
8831 /// Java writes it three times over, submitting to an `ExecutorService` and waiting
8832 /// on a `CountDownLatch`; a spawned task and its `JoinHandle` are the direct
8833 /// equivalent. `client.waitForRequests(1, 2000)` becomes a short sleep: the
8834 /// `MockClient` has moved into the Sender task, so its request count is no longer
8835 /// observable from here.
8836 ///
8837 /// # What is asserted, and why not more
8838 ///
8839 /// Java's last line is `assertionDoneLatch.await(5000, MILLISECONDS)` and it
8840 /// **discards the boolean result**, so the test does not in fact require
8841 /// `initTransactions` to have returned by then — and it cannot have, on either
8842 /// side: the request that is in flight when `close` runs was already dequeued from
8843 /// `pendingRequests` by `nextRequest`, so `TransactionManager.close`'s
8844 /// `pendingRequests.forEach(handler -> handler.fail(..))`
8845 /// (`TransactionManager#close`) has nothing to fail, and neither
8846 /// `NetworkClient.close` nor `MockClient.close` runs completion handlers. The
8847 /// caller is released by its own `max.block.ms` instead, 60 s later by default.
8848 ///
8849 /// So the assertion is the one the test names: `close` is *forced* — it returns
8850 /// within its own timeout rather than waiting on a request that will never be
8851 /// answered. The `initTransactions` task is then given a bounded window purely so
8852 /// a run that *does* complete has its error inspected, exactly as far as Java goes.
8853 ///
8854 /// `multi_thread` is required at the call sites. `Sender::run` over a `MockClient`
8855 /// never blocks — `MockClient::poll` returns immediately — so on the default
8856 /// current-thread runtime it would starve the task doing the closing.
8857 async fn assert_close_forces_pending_transactional_request(producer: KafkaProducer<String, String>) {
8858 let producer = Arc::new(producer);
8859 let init = {
8860 let producer = Arc::clone(&producer);
8861 tokio::task::spawn(async move { producer.init_transactions().await })
8862 };
8863
8864 tokio::time::sleep(Duration::from_millis(200)).await;
8865 let started = std::time::Instant::now();
8866 producer.close_with_timeout(Duration::from_millis(1000)).await.expect("close");
8867 let elapsed = started.elapsed();
8868 assert!(
8869 elapsed < Duration::from_secs(5),
8870 "close must be forced after its 1000 ms timeout, not blocked behind the \
8871 pending transactional request; it took {:?}",
8872 elapsed
8873 );
8874
8875 // Java's ignored `assertionDoneLatch.await(5000, ..)`. If the call did return,
8876 // it must have returned an error — never a successful initTransactions.
8877 if let Ok(joined) = tokio::time::timeout(Duration::from_millis(500), init).await {
8878 let result = joined.expect("the initTransactions task did not panic");
8879 assert!(result.is_err(), "initTransactions cannot succeed once the producer is closed");
8880 }
8881 }
8882
8883 // -- Rust-side regression tests, no Java counterpart --------------------
8884 //
8885 // `close`'s join is a Rust-only failure mode: Java's `ioThread.join()` cannot "lose"
8886 // its thread, whereas `tokio::time::timeout(t, handle)` consumes the `JoinHandle` and
8887 // drops it on expiry. Both tests below exist to make reverting
8888 // `await_sender_handle`'s `&mut` fail — which Critic 46 issue 6 showed the three
8889 // translated `testCloseIsForcedOn*` tests do **not**: with the bug restored,
8890 // `await_sender_handle_indefinitely` finds `None` and returns *sooner*, so every
8891 // assertion in those tests still passes.
8892
8893 /// The mechanism: an expired [`KafkaProducer::await_sender_handle`] must leave the
8894 /// handle in place for [`KafkaProducer::await_sender_handle_indefinitely`] to join.
8895 ///
8896 /// Java's `close` force-closes and *then* joins unconditionally
8897 /// (`KafkaProducer#close`), and CLAUDE.md §11.4 requires the Rust
8898 /// translation to await the handle rather than merely signal it. Passing the handle
8899 /// by value into `tokio::time::timeout` breaks that silently: `timeout` takes
8900 /// ownership and drops it on `Elapsed`.
8901 #[tokio::test]
8902 async fn test_await_sender_handle_keeps_the_handle_when_it_expires() {
8903 let ctx = TxnProducerContext::transactional();
8904
8905 // A task that outlives the wait below, so the wait is guaranteed to expire.
8906 let handle = tokio::task::spawn(async { tokio::time::sleep(Duration::from_secs(30)).await });
8907 *ctx.producer.sender_handle.lock().unwrap() = Some(handle);
8908
8909 let completed = ctx.producer.await_sender_handle(Duration::from_millis(50)).await;
8910 assert!(!completed, "the task outlives the wait, so it cannot have completed");
8911 assert!(
8912 ctx.producer.sender_handle.lock().unwrap().is_some(),
8913 "an expired wait must put the handle back — otherwise close's later \
8914 unconditional join has nothing to join and returns while the Sender runs"
8915 );
8916
8917 // And the retained handle is still the live one: aborting the task and waiting
8918 // again resolves against it, rather than short-circuiting on `None`.
8919 ctx.producer.sender_handle.lock().unwrap().as_ref().expect("retained").abort();
8920 assert!(
8921 ctx.producer.await_sender_handle(Duration::from_secs(5)).await,
8922 "the retained handle must be awaitable, not a husk"
8923 );
8924 }
8925
8926 /// The contract: `close_with_timeout` must not return before the Sender has finished.
8927 ///
8928 /// Same setup as the three `testCloseIsForcedOn*` translations — a transactional
8929 /// request left in flight so the graceful wait expires and `close` force-closes —
8930 /// but with the Sender's exit made *observable*: the harness runs an exit hook on
8931 /// the Sender's own thread once `Sender::run` returns, after a deliberate 300 ms of
8932 /// shutdown cost.
8933 ///
8934 /// The sleep is instrumentation, not padding. Real shutdown work takes time (the
8935 /// force-close path fails pending requests, aborts batches, closes the client), but
8936 /// with a `MockClient` it is instant, which would leave the assertion racing the task
8937 /// instead of observing the join. 300 ms against a 1000 ms graceful timeout is a
8938 /// wide, one-sided margin: with the join the flag is necessarily set; without it
8939 /// `close` returns ~300 ms early.
8940 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8941 async fn test_close_joins_the_sender_after_forcing() {
8942 let exited = Arc::new(AtomicBool::new(false));
8943 let producer = {
8944 let exited = Arc::clone(&exited);
8945 spawned_transactional_producer_with_exit_hook(
8946 &[("transactional.id", "this-is-a-transactional-id")],
8947 |_client| {},
8948 Some(Arc::new(move || {
8949 std::thread::sleep(Duration::from_millis(300));
8950 exited.store(true, Ordering::SeqCst);
8951 })),
8952 )
8953 };
8954
8955 let producer = Arc::new(producer);
8956 let init = {
8957 let producer = Arc::clone(&producer);
8958 tokio::task::spawn(async move { producer.init_transactions().await })
8959 };
8960 tokio::time::sleep(Duration::from_millis(200)).await;
8961
8962 producer.close_with_timeout(Duration::from_millis(1000)).await.expect("close");
8963 assert!(
8964 exited.load(Ordering::SeqCst),
8965 "close returned before the Sender task finished: the graceful wait expired, \
8966 so close force-closed and must then have joined the handle \
8967 (the `ioThread.join()` after the force close in `KafkaProducer#close(Duration, boolean)`, \
8968 CLAUDE.md §11.4)"
8969 );
8970
8971 init.abort();
8972 }
8973
8974 /// Regression test with no Java counterpart: `close_with_timeout(Duration::MAX)`
8975 /// must deliver the records buffered in the accumulator before returning, as Java's
8976 /// `close(Duration.ofMillis(Long.MAX_VALUE))` does (`KafkaProducer#close()`).
8977 ///
8978 /// A 30 s `linger.ms` with a frozen clock keeps the record buffered, so only a
8979 /// graceful close can release it: `initiate_close` closes the accumulator, which
8980 /// makes the batch ready, and the Sender's shutdown loop drains it before exiting
8981 /// (`Sender#run`).
8982 ///
8983 /// The Sender runs on a dedicated thread and runtime, for the reason given in
8984 /// [`spawned_transactional_producer_with_exit_hook`].
8985 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8986 async fn test_close_with_duration_max_delivers_buffered_records() {
8987 // Idempotent (the default) rather than transactional: the Sender requires only
8988 // a producer id before draining the batch.
8989 let mut ctx = TxnProducerContext::new(&[("linger.ms", "30000")], 1);
8990 ctx.sender
8991 .client_mut()
8992 .prepare_response(init_producer_id_response(Errors::None, PRODUCER_ID, EPOCH));
8993 ctx.sender
8994 .client_mut()
8995 .prepare_response(produce_response(0, 1, Errors::None, 0));
8996
8997 let record = ProducerRecord::with_partition_key(
8998 TOPIC.to_string(),
8999 Some(0),
9000 Some("key".to_string()),
9001 Some("value".to_string()),
9002 )
9003 .expect("a valid partition");
9004 let future = ctx.producer.send(record).await.expect("send");
9005 assert!(!future.is_done(), "linger.ms keeps the record buffered until close");
9006
9007 let TxnProducerContext { producer, mut sender, .. } = ctx;
9008 let sender_handle = tokio::task::spawn_blocking(move || {
9009 tokio::runtime::Builder::new_current_thread()
9010 .enable_all()
9011 .build()
9012 .expect("a current-thread runtime for the Sender")
9013 .block_on(sender.run());
9014 });
9015 *producer.sender_handle.lock().unwrap() = Some(sender_handle);
9016
9017 tokio::time::timeout(Duration::from_secs(10), producer.close_with_timeout(Duration::MAX))
9018 .await
9019 .expect("close returns once the buffered record is delivered")
9020 .expect("close");
9021
9022 assert!(future.is_done(), "close returned before the buffered record completed");
9023 let metadata = future.get().await.expect("the record is delivered, not aborted");
9024 assert_eq!(metadata.offset(), 1);
9025 }
9026
9027 /// Translated from `KafkaProducerTest.testTransactionalMethodThrowsWhenSenderClosed`.
9028 #[tokio::test]
9029 async fn test_transactional_method_throws_when_sender_closed() {
9030 let ctx = TxnProducerContext::new(&[("transactional.id", "this-is-a-transactional-id")], 1);
9031 ctx.producer.close().await.expect("close");
9032 let error = ctx.producer.init_transactions().await.expect_err("the producer is closed");
9033 assert_eq!(error.message(), "Cannot perform operation after producer has been closed");
9034 }
9035
9036 /// Translated from `KafkaProducerTest.testCloseIsForcedOnPendingFindCoordinator`.
9037 ///
9038 /// No response is prepared, so the `FindCoordinator` is the request left in flight
9039 /// when `close` runs.
9040 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9041 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testCloseIsForcedOnPendingFindCoordinator")]
9042 async fn test_close_is_forced_on_pending_find_coordinator() {
9043 let producer =
9044 spawned_transactional_producer(&[("transactional.id", "this-is-a-transactional-id")], |_client| {});
9045 assert_close_forces_pending_transactional_request(producer).await;
9046 }
9047
9048 /// Translated from `KafkaProducerTest.testCloseIsForcedOnPendingInitProducerId`.
9049 ///
9050 /// The `FindCoordinator` is answered, so the `InitProducerId` is the request left
9051 /// in flight.
9052 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9053 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testCloseIsForcedOnPendingInitProducerId")]
9054 async fn test_close_is_forced_on_pending_init_producer_id() {
9055 let producer =
9056 spawned_transactional_producer(&[("transactional.id", "this-is-a-transactional-id")], |client| {
9057 client.prepare_response(find_coordinator_response(
9058 Errors::None,
9059 "this-is-a-transactional-id",
9060 &coordinator_node(),
9061 ));
9062 });
9063 assert_close_forces_pending_transactional_request(producer).await;
9064 }
9065
9066 /// Translated from `KafkaProducerTest.testCloseIsForcedOnPendingAddOffsetRequest`.
9067 ///
9068 /// In Apache Kafka 4.2 this method's body is **identical** to
9069 /// `testCloseIsForcedOnPendingInitProducerId`'s — it prepares one
9070 /// `FindCoordinator` and submits `initTransactions`, never reaching an
9071 /// `AddOffsetsToTxn` despite the name. Translated as written rather than
9072 /// "corrected": inventing the `sendOffsetsToTransaction` the name implies would be
9073 /// a different test from the one Java runs.
9074 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9075 #[doc(alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testCloseIsForcedOnPendingAddOffsetRequest")]
9076 async fn test_close_is_forced_on_pending_add_offset_request() {
9077 let producer =
9078 spawned_transactional_producer(&[("transactional.id", "this-is-a-transactional-id")], |client| {
9079 client.prepare_response(find_coordinator_response(
9080 Errors::None,
9081 "this-is-a-transactional-id",
9082 &coordinator_node(),
9083 ));
9084 });
9085 assert_close_forces_pending_transactional_request(producer).await;
9086 }
9087
9088 // =====================================================================
9089 // PHASE-6 TEST ACCOUNTING — the transactional `KafkaProducerTest` methods
9090 //
9091 // SCOPE CRITERION. A `KafkaProducerTest.java` method is in scope for Phase 6 iff its
9092 // body (or a helper it calls) mentions one of the five public transactional methods,
9093 // `TRANSACTIONAL_ID_CONFIG`, `TransactionManager`, `maybeAddPartition`, or the
9094 // `verifyInvalidGroupMetadata` helper. That is the marker set below, exactly.
9095 //
9096 // Two corrections to the criterion as first written (Critic 46 issue 4):
9097 //
9098 // - It said "one of the **three** transactional config keys" while the marker set
9099 // held one. The narrow *program* was right and the prose wrong: adding
9100 // `ENABLE_IDEMPOTENCE_CONFIG` yields 36 rows and drags in 8 plainly
9101 // non-transactional tests that merely set `enable.idempotence=false`
9102 // (`testMetadataFetch`, `testMetadataExpiry`, `testMetadataTimeoutWith*`,
9103 // `testMetadataWithPartitionOutOfRange`, `testTopicRefreshInMetadata`,
9104 // `testFlushCompleteSendOfInflightBatches`, `shouldNotInvokeFlushInCallback`).
9105 // "Three" described no realizable marker set, so the prose now matches the
9106 // program.
9107 // - `TransactionManager` / `maybeAddPartition` were **missing**, which made every
9108 // mock-injected transactional test invisible. `testPartitionAddedToTransaction`
9109 // reaches the transactional path only through
9110 // `KafkaProducerTestContext`'s `mock(TransactionManager.class)`
9111 // (`KafkaProducerTest.KafkaProducerTestContext`), so it named no marker at all and was absent from the denominator.
9112 // It was also a real coverage gap, not only bookkeeping: `maybe_add_partition`'s
9113 // production call site in this file had no test.
9114 //
9115 // The classifier lesson generalises, and is the same one the sibling `sender.rs`
9116 // block now records: a completeness check is only as strong as the assumption its
9117 // classifier makes, and when the Java side and the Rust side share that assumption
9118 // the resulting diff is **vacuous** for whatever they agree to ignore. Here the
9119 // shared assumption was "a transactional test names a transactional symbol", which a
9120 // Mockito-injected test does not.
9121 //
9122 // DERIVATION. The splitter counts braces rather than matching a declaration
9123 // regexp, because several of these bodies contain anonymous classes and lambdas
9124 // whose members would otherwise end a block early. Run from the repo root; awk
9125 // version 20200816, exit 0:
9126 //
9127 // T=kafka/clients/src/test/java/org/apache/kafka/clients/producer/KafkaProducerTest.java
9128 // M='initTransactions,beginTransaction,commitTransaction,abortTransaction,'
9129 // M="$M"'sendOffsetsToTransaction,verifyInvalidGroupMetadata,TRANSACTIONAL_ID_CONFIG,'
9130 // M="$M"'TransactionManager,maybeAddPartition'
9131 // awk -v MARKERS="$M" '
9132 // BEGIN { n = split(MARKERS, m, ","); depth = 0; inm = 0 }
9133 // {
9134 // line = $0
9135 // if (depth == 1 && !inm && line ~ /^ [a-zA-Z@<].*\(.*\{[ \t]*$/ &&
9136 // line !~ /^ (class|enum|interface|static \{)/) {
9137 // match(line, /[a-zA-Z0-9_]+\(/)
9138 // if (RSTART > 0) { name = substr(line, RSTART, RLENGTH-1); start = NR; inm = 1; delete hard }
9139 // }
9140 // if (inm) { for (i = 1; i <= n; i++) if (index(line, m[i]) > 0) hard[m[i]] = 1 }
9141 // o = gsub(/\{/, "{", line); c = gsub(/\}/, "}", line); depth += o - c
9142 // if (inm && depth <= 1) {
9143 // hits = ""
9144 // for (i = 1; i <= n; i++) if (m[i] in hard) hits = hits (hits == "" ? "" : "+") m[i]
9145 // if (hits != "") printf "%d\t%s\n", start, name
9146 // inm = 0
9147 // }
9148 // }' "$T"
9149 //
9150 // It prints **29** rows. The inline `for (i = 1; i <= n; i++)` walks `MARKERS` in
9151 // declaration order rather than `for (k in hard)`, so the output is reproducible on
9152 // any awk (the sibling blocks in `sender.rs` / `transaction_manager.rs` record why
9153 // that matters). An earlier revision credited an `emit` function for that property;
9154 // there is none in *this* program — the logic is inline, and `emit` belongs to the
9155 // sibling blocks (Critic 46 issue 4).
9156 //
9157 // ARITHMETIC. 29 printed rows = 1 helper (`verifyInvalidGroupMetadata`, 2000 — not a
9158 // test method) + **28** test methods. §Phase-6 says 27; 28 is the corrected
9159 // denominator, the extra being `testPartitionAddedToTransaction` (2423; Critic 46 issue 4).
9160 // Those 28 partition with no overlap into
9161 //
9162 // 23 translated here
9163 // + 1 not translated, justified (`testNullGroupMetadataInSendOffsets`)
9164 // + 4 translated in Phase 1, in `producer_config.rs`
9165 // = 28.
9166 //
9167 // Each of the three groups is listed in full below, so the sum can be checked
9168 // against the lists rather than taken on trust.
9169 //
9170 // TRANSLATED HERE (23 Rust tests):
9171 // 1290 testInitTransactionsResponseAfterTimeout
9172 // -> test_init_transactions_response_after_timeout
9173 // 1329 testInitTransactionTimeout -> test_init_transaction_timeout
9174 // 1364 testInitTransactionWhileThrottled -> test_init_transaction_while_throttled
9175 // 1390 testClusterAuthorizationFailure -> test_cluster_authorization_failure
9176 // 1419 testAbortTransaction -> test_abort_transaction
9177 // 1444 testTransactionV2ProduceWithConcurrentTransactionError
9178 // -> test_transaction_v2_produce_with_concurrent_transaction_error
9179 // 1503 testMeasureAbortTransactionDuration -> test_measure_abort_transaction_duration
9180 // 1533 testCommitTransactionWithRecordTooLargeException
9181 // -> test_commit_transaction_with_record_too_large_error
9182 // 1563 testCommitTransactionWithMetadataTimeoutForMissingTopic
9183 // -> test_commit_transaction_with_metadata_timeout_for_missing_topic
9184 // 1600 testCommitTransactionWithMetadataTimeoutForPartitionOutOfRange
9185 // -> test_commit_transaction_with_metadata_timeout_for_partition_out_of_range
9186 // 1637 testCommitTransactionWithSendToInvalidTopic
9187 // -> test_commit_transaction_with_send_to_invalid_topic
9188 // 1677 testSendTxnOffsetsWithGroupId -> test_send_txn_offsets_with_group_id
9189 // 1715 testSendTxnOffsetsWithGroupIdTransactionV2
9190 // -> test_send_txn_offsets_with_group_id_transaction_v2
9191 // 1772 testTransactionV2Produce -> test_transaction_v2_produce
9192 // 1842 testMeasureTransactionDurations -> test_measure_transaction_durations
9193 // 1895 testSendTxnOffsetsWithGroupMetadata -> test_send_txn_offsets_with_group_metadata
9194 // 1950 testInvalidGenerationIdAndMemberIdCombinedInSendOffsets
9195 // -> test_invalid_generation_id_and_member_id_combined_in_send_offsets
9196 // 2054 testOnlyCanExecuteCloseAfterInitTransactionsTimeout
9197 // -> test_only_can_execute_close_after_init_transactions_timeout
9198 // 2163 testTransactionalMethodThrowsWhenSenderClosed
9199 // -> test_transactional_method_throws_when_sender_closed
9200 // 2182 testCloseIsForcedOnPendingFindCoordinator
9201 // -> test_close_is_forced_on_pending_find_coordinator
9202 // 2210 testCloseIsForcedOnPendingInitProducerId
9203 // -> test_close_is_forced_on_pending_init_producer_id
9204 // 2239 testCloseIsForcedOnPendingAddOffsetRequest
9205 // -> test_close_is_forced_on_pending_add_offset_request
9206 // 2423 testPartitionAddedToTransaction -> test_partition_added_to_transaction
9207 // The mock-injected one. Translated with a real manager and
9208 // `is_partition_pending_add` in place of Mockito's `verify`; the test's own
9209 // rustdoc argues why that is stronger rather than weaker.
9210 //
9211 // NOT TRANSLATED, JUSTIFIED (1):
9212 // 1944 testNullGroupMetadataInSendOffsets — passes `null` for the
9213 // `ConsumerGroupMetadata`. The Rust parameter is a value, so the argument
9214 // cannot be constructed and the arm it exercises
9215 // (`KafkaProducer`) is enforced by the type system instead of a
9216 // runtime check. Recorded again on
9217 // `test_invalid_generation_id_and_member_id_combined_in_send_offsets`, the
9218 // other caller of the same Java helper, which *is* translated.
9219 //
9220 // TRANSLATED IN PHASE 1, in `producer_config.rs` (4): these reference
9221 // `TRANSACTIONAL_ID_CONFIG` only as an input to
9222 // `postProcessAndValidateIdempotenceConfigs`, so they belong to `ProducerConfig`,
9223 // not to this file. Verified present, not assumed:
9224 //
9225 // $ grep -c 'fn test_overwrite_acks_and_retries_for_idempotent_producers\|fn test_acks_and_idempotence_for_idempotent_producers\|fn test_retries_and_idempotence_for_idempotent_producers\|fn test_inflight_requests_and_idempotence_for_idempotent_producers' src/producer/producer_config.rs
9226 // 4
9227 //
9228 // 222 testOverwriteAcksAndRetriesForIdempotentProducers
9229 // 238 testAcksAndIdempotenceForIdempotentProducers
9230 // 341 testRetriesAndIdempotenceForIdempotentProducers
9231 // 413 testInflightRequestsAndIdempotenceForIdempotentProducers
9232 //
9233 // TRANSLATED IN FULL, ASSERTIONS INCLUDED (2 methods):
9234 // `testMeasureAbortTransactionDuration` and `testMeasureTransactionDurations`.
9235 // `KafkaProducerMetrics` and `org.apache.kafka.common.metrics` are translated and
9236 // wired (`KafkaProducer::{init_transactions, begin_transaction,
9237 // send_offsets_to_transaction, commit_transaction, abort_transaction}` now call the
9238 // matching `record_*` sensor), so every `getMetricValue(producer,
9239 // "txn-*-time-ns-total")` assertion is translated via `get_metric_value` /
9240 // `get_and_assert_duration_at_least` / `assert_duration_at_least`, against Java's
9241 // own `tick.toNanos()` floors: the producer times with the injected `MockTime`.
9242 // =====================================================================
9243
9244 // -- `configureTransactionState` tests ----------------------------------
9245 //
9246 // These began life covering the temporary MILESTONE-11 GUARD in `new`.
9247 // Phase 4 turned the idempotence cases from rejections into constructions, and
9248 // Phase 6 removed the guard's last (transactional) arm — so what they now cover
9249 // is `configureTransactionState` across its three
9250 // outcomes: no manager, an idempotent manager, and a transactional manager.
9251
9252 fn guard_props(extra: &[(&str, &str)]) -> HashMap<String, String> {
9253 let mut props = HashMap::from([("bootstrap.servers".to_string(), "localhost:9999".to_string())]);
9254 for (key, value) in extra {
9255 props.insert((*key).to_string(), (*value).to_string());
9256 }
9257 props
9258 }
9259
9260 fn from_guard_props(props: &HashMap<String, String>) -> Result<(), Error> {
9261 let config = ProducerConfig::new(props)?;
9262 KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer).map(|_| ())
9263 }
9264
9265 /// KIP-1123 (KAFKA-19193): `partitioner.rack.aware` and `client.rack` reach the
9266 /// accumulator's `PartitionerConfig` (`KafkaProducer#KafkaProducer`), and a blank
9267 /// rack in rack-aware mode fails construction with `PartitionerConfig`'s
9268 /// `ConfigException`, wrapped like every other construction failure. Java checks
9269 /// it even with a custom partitioner, so the `RoundRobinPartitioner` case fails
9270 /// too.
9271 #[tokio::test]
9272 async fn test_partitioner_rack_awareness_wiring() {
9273 let props = guard_props(&[("partitioner.rack.aware", "true"), ("client.rack", "rack0")]);
9274 let config = ProducerConfig::new(&props).expect("valid config");
9275 let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
9276 .expect("a rack-aware producer with a rack constructs");
9277 assert_eq!((true, "rack0"), producer.accumulator.partitioner_rack_for_test());
9278
9279 let props = guard_props(&[("client.rack", "rack0")]);
9280 let config = ProducerConfig::new(&props).expect("valid config");
9281 let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
9282 .expect("default config constructs");
9283 assert_eq!((false, "rack0"), producer.accumulator.partitioner_rack_for_test());
9284
9285 for partitioner_type in [None, Some(ProducerConfig::ROUND_ROBIN_PARTITIONER)] {
9286 let mut extra = vec![("partitioner.rack.aware", "true")];
9287 if let Some(name) = partitioner_type {
9288 extra.push(("partitioner.type", name));
9289 }
9290 let config = ProducerConfig::new(&guard_props(&extra)).expect("valid config");
9291 let error = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
9292 .expect_err("rack-aware without client.rack must fail construction");
9293 assert_eq!(error.message(), "Failed to construct kafka producer", "{partitioner_type:?}");
9294 let cause: &Error = std::error::Error::source(&error)
9295 .and_then(|e| e.downcast_ref::<Error>())
9296 .expect("the cause is a crate Error");
9297 assert!(matches!(cause, Error::Config(_)), "got {cause:?}");
9298 assert_eq!(
9299 cause.message(),
9300 "client.rack must be provided if partitioner.rack.aware is enabled"
9301 );
9302 }
9303 }
9304
9305 /// KAFKA-2121 for the KIP-1123 check: a custom partitioner is configured before
9306 /// `PartitionerConfig` rejects a blank `client.rack`, so the failed construction
9307 /// must close it, as Java's constructor `catch` -> `close(0, true)` does
9308 /// (`KafkaProducer#KafkaProducer` -> `KafkaProducer#close`). Critic 96 F1.
9309 #[test]
9310 fn test_partitioner_closed_when_construction_fails_after_configure() {
9311 let _guard = MockPartitioner::lock_counters();
9312 MockPartitioner::reset_counters();
9313
9314 let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap();
9315 runtime.block_on(async {
9316 let partitioner: Box<dyn Partitioner<String, String>> = Box::new(MockPartitioner::new());
9317 let config = ProducerConfig::new(&guard_props(&[("partitioner.rack.aware", "true")]))
9318 .expect("valid config")
9319 .set_partitioner(partitioner);
9320 let init_count = AtomicUsize::new(0);
9321 let serializer_closes = Arc::new(AtomicUsize::new(0));
9322 let error = KafkaProducer::<String, String>::new(
9323 config,
9324 MockSerializer::new(&init_count, &serializer_closes),
9325 MockSerializer::new(&init_count, &serializer_closes),
9326 )
9327 .expect_err("rack-aware without client.rack must fail construction");
9328 assert_eq!(error.message(), "Failed to construct kafka producer");
9329
9330 assert_eq!(1, MockPartitioner::init_count().load(Ordering::SeqCst));
9331 assert_eq!(1, MockPartitioner::close_count().load(Ordering::SeqCst));
9332 // `close(Duration.ofMillis(0), true)` closes the serializers too
9333 // (`KafkaProducer#close`).
9334 assert_eq!(2, serializer_closes.load(Ordering::SeqCst));
9335 });
9336
9337 MockPartitioner::reset_counters();
9338 }
9339
9340 /// KIP-1332 (`KafkaProducer#KafkaProducer`): `buffer.memory.allocation.strategy=incremental`
9341 /// (any case) builds a `ChunkedRecordAccumulator` over a pool serving 16 KiB chunks when
9342 /// `batch.size` is at least one chunk, and falls back to the full strategy, with a pool of
9343 /// `batch.size` buffers, when it is not. The default is the full strategy.
9344 #[tokio::test]
9345 async fn test_buffer_memory_allocation_strategy_wiring() {
9346 use crate::producer::internals::buffer_pool::AllocationMode;
9347 type Case<'a> = (&'a [(&'a str, &'a str)], bool, usize);
9348 let cases: [Case; 5] = [
9349 (&[], false, 16384),
9350 (&[("buffer.memory.allocation.strategy", "incremental")], true, 16384),
9351 (
9352 &[
9353 ("buffer.memory.allocation.strategy", "Incremental"),
9354 ("batch.size", "1048576"),
9355 ],
9356 true,
9357 16384,
9358 ),
9359 // Below one chunk: the warning, then the full strategy.
9360 (
9361 &[
9362 ("buffer.memory.allocation.strategy", "incremental"),
9363 ("batch.size", "16383"),
9364 ],
9365 false,
9366 16383,
9367 ),
9368 (
9369 &[
9370 ("buffer.memory.allocation.strategy", "incremental"),
9371 ("batch.size", "0"),
9372 ],
9373 false,
9374 1,
9375 ),
9376 ];
9377 for (extra, incremental, poolable_size) in cases {
9378 let config = ProducerConfig::new(&guard_props(extra)).expect("valid config");
9379 let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
9380 .expect("the producer constructs");
9381 assert_eq!(incremental, producer.chunked_accumulator.is_some(), "{extra:?}");
9382 let pool = producer.accumulator.buffer_pool_for_test();
9383 let mode = if incremental {
9384 AllocationMode::Incremental
9385 } else {
9386 AllocationMode::Full
9387 };
9388 assert_eq!(mode, pool.allocation_mode(), "{extra:?}");
9389 assert_eq!(poolable_size, pool.poolable_size(), "{extra:?}");
9390 if let Some(chunked) = &producer.chunked_accumulator {
9391 // The chunked accumulator extends the very accumulator the Sender drains.
9392 assert!(Arc::ptr_eq(chunked.base(), &producer.accumulator), "{extra:?}");
9393 }
9394 }
9395 }
9396
9397 /// KIP-1332: the incremental strategy does not support compression yet, so the combination
9398 /// fails construction with Java's `ConfigException` (`KafkaProducer#KafkaProducer`), wrapped
9399 /// like every other construction failure. Below one chunk the strategy falls back to the full
9400 /// one first, so compression is allowed there, as in Java.
9401 #[cfg(feature = "gzip")]
9402 #[tokio::test]
9403 async fn test_incremental_allocation_strategy_rejects_compression() {
9404 let config = ProducerConfig::new(&guard_props(&[
9405 ("buffer.memory.allocation.strategy", "incremental"),
9406 ("compression.type", "gzip"),
9407 ]))
9408 .expect("valid config");
9409 let error = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
9410 .expect_err("incremental with compression must fail construction");
9411 assert_eq!(error.message(), "Failed to construct kafka producer");
9412 let cause: &Error = std::error::Error::source(&error)
9413 .and_then(|e| e.downcast_ref::<Error>())
9414 .expect("the cause is a crate Error");
9415 assert!(matches!(cause, Error::Config(_)), "got {cause:?}");
9416 assert_eq!(
9417 cause.message(),
9418 "The incremental buffer.memory.allocation.strategy does not support compression yet. \
9419 compression.type must be set to none."
9420 );
9421
9422 let config = ProducerConfig::new(&guard_props(&[
9423 ("buffer.memory.allocation.strategy", "incremental"),
9424 ("compression.type", "gzip"),
9425 ("batch.size", "1024"),
9426 ]))
9427 .expect("valid config");
9428 let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
9429 .expect("the full-strategy fallback supports compression");
9430 assert!(producer.chunked_accumulator.is_none());
9431 }
9432
9433 /// KAFKA-2121 for the KIP-1332 check: the compression `ConfigException` is raised inside
9434 /// `build_accumulator`, after a custom partitioner is configured, so the failed construction
9435 /// closes it (`KafkaProducer#KafkaProducer` -> `KafkaProducer`).
9436 #[cfg(feature = "lz4")]
9437 #[test]
9438 fn test_partitioner_closed_when_incremental_compression_is_rejected() {
9439 let _guard = MockPartitioner::lock_counters();
9440 MockPartitioner::reset_counters();
9441
9442 let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap();
9443 runtime.block_on(async {
9444 let partitioner: Box<dyn Partitioner<String, String>> = Box::new(MockPartitioner::new());
9445 let config = ProducerConfig::new(&guard_props(&[
9446 ("buffer.memory.allocation.strategy", "incremental"),
9447 ("compression.type", "lz4"),
9448 ]))
9449 .expect("valid config")
9450 .set_partitioner(partitioner);
9451 let error = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
9452 .expect_err("incremental with compression must fail construction");
9453 assert_eq!(error.message(), "Failed to construct kafka producer");
9454
9455 assert_eq!(1, MockPartitioner::init_count().load(Ordering::SeqCst));
9456 assert_eq!(1, MockPartitioner::close_count().load(Ordering::SeqCst));
9457 });
9458
9459 MockPartitioner::reset_counters();
9460 }
9461
9462 /// Java wraps the whole constructor in `catch (Throwable t)` and rethrows
9463 /// `new KafkaException("Failed to construct kafka producer", t)`
9464 /// (`KafkaProducer#KafkaProducer`), so a caller has one class and one message to
9465 /// guard construction with whatever went wrong inside. Every failure used to
9466 /// escape raw, and one of them (`Error::local_illegal_argument` from the channel
9467 /// builder) answered `false` to `is_kafka_error()`.
9468 #[test]
9469 fn construction_failures_are_wrapped_as_a_kafka_error() {
9470 let props = HashMap::from([("bootstrap.servers".to_string(), "not-a-host-port".to_string())]);
9471 let config = ProducerConfig::new(&props).expect("the config itself parses");
9472 let error = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
9473 .expect_err("an unparseable bootstrap.servers entry must fail construction");
9474
9475 assert_eq!(error.message(), "Failed to construct kafka producer");
9476 // Java's replacement is a bare `KafkaException`.
9477 assert!(error.is_kafka_error(), "Java's replacement is a Kafka error: {error:?}");
9478 assert!(!error.is_api_error(), "a bare Kafka error is not an API error: {error:?}");
9479 // The real cause is carried, not stringified into the message.
9480 assert!(
9481 error.source().is_some(),
9482 "the underlying failure must be the wrapper's cause, not lost"
9483 );
9484 // The cause is `parseAndValidateAddresses`' `ConfigException`, with its
9485 // single-message text.
9486 let cause: &Error = std::error::Error::source(&error)
9487 .and_then(|e| e.downcast_ref::<Error>())
9488 .expect("the cause is a crate Error");
9489 assert!(matches!(cause, Error::Config(_)), "got {cause:?}");
9490 assert_eq!(cause.message(), "Invalid url in bootstrap.servers: not-a-host-port");
9491 }
9492
9493 /// Translated from `KafkaProducerTest.testProducerBootstrapResolutionExceptionPropagated`
9494 /// (KIP-909): with a positive `bootstrap.resolve.timeout.ms` and an
9495 /// unresolvable bootstrap host, `partitionsFor` fails with the
9496 /// `BootstrapResolutionException` once the budget runs out (the metadata
9497 /// wait is woken by it, not left to `max.block.ms`), and so does every later
9498 /// call.
9499 #[tokio::test(flavor = "multi_thread")]
9500 #[doc(
9501 alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testProducerBootstrapResolutionExceptionPropagated"
9502 )]
9503 async fn test_producer_bootstrap_resolution_error_propagated() {
9504 let props = HashMap::from([
9505 (
9506 ProducerConfig::BOOTSTRAP_SERVERS_CONFIG.to_string(),
9507 "unresolvable.invalid:9092".to_string(),
9508 ),
9509 (
9510 ProducerConfig::BOOTSTRAP_RESOLVE_TIMEOUT_MS_CONFIG.to_string(),
9511 "3000".to_string(),
9512 ),
9513 ]);
9514 let producer = KafkaProducer::<String, String>::new(
9515 ProducerConfig::new(&props).unwrap(),
9516 StringSerializer,
9517 StringSerializer,
9518 )
9519 .expect("a positive timeout defers resolution, so construction succeeds");
9520
9521 let expected = "Failed to resolve bootstrap servers after 3000ms. \
9522 Please check your bootstrap.servers configuration and DNS settings.";
9523 let start = std::time::Instant::now();
9524 let max_wait = Duration::from_millis(15000);
9525 loop {
9526 assert!(
9527 start.elapsed() < max_wait,
9528 "Expected the BootstrapResolutionError within {}ms",
9529 max_wait.as_millis()
9530 );
9531 match tokio::time::timeout(max_wait, producer.partitions_for("test-topic")).await {
9532 Ok(Err(Error::BootstrapResolution(e))) => {
9533 assert_eq!(e.message(), expected);
9534 break;
9535 },
9536 Ok(_) => {},
9537 Err(_) => panic!("partitions_for must be woken by the bootstrap failure"),
9538 }
9539 }
9540
9541 // After the first failure, any further API call must also return it. This guards against
9542 // accidentally clearing the bootstrap error from the metadata layer.
9543 match producer.partitions_for("test-topic").await {
9544 Err(Error::BootstrapResolution(e)) => assert_eq!(e.message(), expected),
9545 other => panic!("expected the bootstrap failure, got {other:?}"),
9546 }
9547 producer.close_with_timeout(Duration::ZERO).await.unwrap();
9548 }
9549
9550 /// Translated from
9551 /// `KafkaProducerTest.testProducerSendOffsetsToTransactionBootstrapResolutionExceptionPropagated`
9552 /// (KIP-909 + KIP-1319, with 0720ba1141's group-metadata mock): sendOffsetsToTransaction became
9553 /// metadata-aware in KIP-1319, so it can also block on bootstrap and must surface the
9554 /// `BootstrapResolutionException`. The first call waits on the refresh and is woken by the
9555 /// failure; the second finds the topic already tracked and hits the no-refresh branch's check.
9556 #[tokio::test(flavor = "multi_thread")]
9557 #[doc(
9558 alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testProducerSendOffsetsToTransactionBootstrapResolutionExceptionPropagated"
9559 )]
9560 async fn test_producer_send_offsets_to_transaction_bootstrap_resolution_error_propagated() {
9561 let props = HashMap::from([
9562 (
9563 ProducerConfig::BOOTSTRAP_SERVERS_CONFIG.to_string(),
9564 "unresolvable.invalid:9092".to_string(),
9565 ),
9566 (
9567 ProducerConfig::BOOTSTRAP_RESOLVE_TIMEOUT_MS_CONFIG.to_string(),
9568 "3000".to_string(),
9569 ),
9570 (ProducerConfig::TRANSACTIONAL_ID_CONFIG.to_string(), "test-tx-id".to_string()),
9571 ]);
9572 let producer = KafkaProducer::<String, String>::new(
9573 ProducerConfig::new(&props).unwrap(),
9574 StringSerializer,
9575 StringSerializer,
9576 )
9577 .expect("a positive timeout defers resolution, so construction succeeds");
9578
9579 let group_metadata = ConsumerGroupMetadataImpl::new("test-group");
9580 let offsets = || {
9581 HashMap::from([(
9582 TopicPartition::new("test-topic".to_string(), 0),
9583 OffsetAndMetadata::new(0).expect("a non-negative offset"),
9584 )])
9585 };
9586 let expected = "Failed to resolve bootstrap servers after 3000ms. \
9587 Please check your bootstrap.servers configuration and DNS settings.";
9588 for attempt in 0..2 {
9589 match tokio::time::timeout(
9590 Duration::from_secs(15),
9591 producer.send_offsets_to_transaction(offsets(), &group_metadata),
9592 )
9593 .await
9594 {
9595 Ok(Err(Error::BootstrapResolution(e))) => assert_eq!(e.message(), expected, "attempt {attempt}"),
9596 Ok(other) => panic!("attempt {attempt}: expected the bootstrap failure, got {other:?}"),
9597 Err(_) => {
9598 panic!("attempt {attempt}: send_offsets_to_transaction must be woken by the bootstrap failure")
9599 },
9600 }
9601 }
9602 producer.close_with_timeout(Duration::ZERO).await.unwrap();
9603 }
9604
9605 /// Critic 92, Issue 2: the C / Python producer destroy path closes the
9606 /// producer and drops the runtime it built. With a slow bootstrap lookup in
9607 /// flight that now returns within Java's bound; with the lookup on that
9608 /// runtime's blocking pool the drop waited for every remaining lookup.
9609 #[test]
9610 fn test_close_and_runtime_drop_are_bounded_while_a_bootstrap_resolution_is_in_flight() {
9611 let servers: Vec<String> = (0..3)
9612 .map(|i| format!("host{i}{}:9092", crate::ClientUtils::SLOW_TEST_HOST_SUFFIX))
9613 .collect();
9614 let props = HashMap::from([
9615 (ProducerConfig::BOOTSTRAP_SERVERS_CONFIG.to_string(), servers.join(",")),
9616 (
9617 ProducerConfig::BOOTSTRAP_RESOLVE_TIMEOUT_MS_CONFIG.to_string(),
9618 "120000".to_string(),
9619 ),
9620 ]);
9621 let runtime = tokio::runtime::Builder::new_multi_thread().enable_all().build().unwrap();
9622 let producer = runtime.block_on(async {
9623 KafkaProducer::<String, String>::new(
9624 ProducerConfig::new(&props).unwrap(),
9625 StringSerializer,
9626 StringSerializer,
9627 )
9628 .expect("a positive timeout defers resolution, so construction succeeds")
9629 });
9630 let started = std::time::Instant::now();
9631 runtime.block_on(async {
9632 // Let the sender poll so the resolution is surely in flight.
9633 tokio::time::sleep(Duration::from_millis(100)).await;
9634 producer.close_with_timeout(Duration::ZERO).await.unwrap();
9635 });
9636 drop(producer);
9637 drop(runtime);
9638 let elapsed = started.elapsed();
9639 assert!(elapsed < Duration::from_secs(4), "close + runtime drop took {elapsed:?}");
9640 }
9641
9642 /// Translated from
9643 /// `KafkaProducerTest.testProducerConstructorFailsWithConfigExceptionOnUnresolvableBootstrapWhenTimeoutZero`
9644 /// (KAFKA-20939): the default `bootstrap.resolve.timeout.ms=0` resolves DNS
9645 /// synchronously in the constructor, so an unresolvable host fails
9646 /// construction with the `ConfigException` wrapped in the constructor's
9647 /// `KafkaException`.
9648 #[test]
9649 #[doc(
9650 alias = "org.apache.kafka.clients.producer.KafkaProducerTest#testProducerConstructorFailsWithConfigExceptionOnUnresolvableBootstrapWhenTimeoutZero"
9651 )]
9652 fn test_producer_constructor_fails_with_config_error_on_unresolvable_bootstrap_when_timeout_zero() {
9653 let props = HashMap::from([(
9654 ProducerConfig::BOOTSTRAP_SERVERS_CONFIG.to_string(),
9655 "unresolvable.invalid:9092".to_string(),
9656 )]);
9657 let config = ProducerConfig::new(&props).unwrap();
9658 assert_eq!(config.bootstrap_resolve_timeout_ms, 0);
9659 let error = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
9660 .expect_err("construction must fail");
9661 assert_eq!(error.message(), "Failed to construct kafka producer");
9662 let cause: &Error = std::error::Error::source(&error)
9663 .and_then(|e| e.downcast_ref::<Error>())
9664 .expect("the cause is a crate Error");
9665 assert!(matches!(cause, Error::Config(_)), "got {cause:?}");
9666 assert_eq!(cause.message(), "No resolvable bootstrap urls given in bootstrap.servers");
9667 }
9668
9669 /// `doSend`'s inner `catch (KafkaException e)` around `waitOnMetadata`
9670 /// (`KafkaProducer#doSend`) relabels a `close()` racing an in-flight send:
9671 ///
9672 /// ```java
9673 /// if (metadata.isClosed())
9674 /// throw new KafkaException("Producer closed while send in progress", e);
9675 /// ```
9676 ///
9677 /// Rust had neither layer: `await_update` ignored `is_closed()` and `do_send`
9678 /// had no counterpart to this catch, so a caller shutting down while a first
9679 /// send resolved metadata saw a stall followed by a misleading retriable
9680 /// timeout.
9681 #[tokio::test]
9682 async fn closing_the_metadata_during_a_send_reports_the_close_not_a_timeout() {
9683 // A producer whose metadata knows nothing, so `send` has to wait for it.
9684 let metadata = Arc::new(ProducerMetadata::new(
9685 100,
9686 1000,
9687 300_000,
9688 300_000,
9689 ClusterResourceListeners::new(),
9690 ));
9691 let accumulator = create_accumulator();
9692 let config = ProducerConfig { max_block_ms: 30_000, ..ProducerConfig::defaults() };
9693 let producer = create_producer_with_config(config, Arc::clone(&metadata), accumulator);
9694
9695 metadata.close();
9696
9697 let started = std::time::Instant::now();
9698 let record = ProducerRecord::with_key(TOPIC.to_string(), Some("k".to_string()), Some("v".to_string()));
9699 let error = producer
9700 .send(record)
9701 .await
9702 .expect_err("a bare Kafka error is not an API error, so send() returns Err");
9703
9704 assert_eq!(error.message(), "Producer closed while send in progress");
9705 assert!(error.is_kafka_error(), "got {error:?}");
9706 assert!(!error.is_api_error(), "got {error:?}");
9707 assert!(
9708 !error.is_timeout_error(),
9709 "the close must not be reported as a timeout: {error:?}"
9710 );
9711 assert_eq!(
9712 error.source().expect("the awaitUpdate error is the cause").message(),
9713 "Requested metadata update after close"
9714 );
9715 assert!(
9716 started.elapsed() < std::time::Duration::from_secs(5),
9717 "it must fail fast, not wait out max.block.ms"
9718 );
9719 }
9720
9721 /// The default configuration must construct. `#[tokio::test]`: a successful
9722 /// `new` spawns the Sender task and so needs a runtime. The spawned task
9723 /// attempts to reach localhost:9999, fails harmlessly, and is dropped with the
9724 /// test.
9725 #[tokio::test]
9726 async fn test_guard_allows_default_config() {
9727 from_guard_props(&guard_props(&[])).expect("default config must still construct");
9728 }
9729
9730 /// Explicit `enable.idempotence=true` is accepted as of Phase 4, and the
9731 /// producer really is idempotent: it holds a `TransactionManager`.
9732 #[tokio::test]
9733 async fn test_explicit_enable_idempotence_builds_a_transaction_manager() {
9734 let props = guard_props(&[("enable.idempotence", "true")]);
9735 let config = ProducerConfig::new(&props).expect("valid config");
9736 let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
9737 .expect("explicit idempotence is supported as of Phase 4");
9738 let transaction_manager = producer
9739 .transaction_manager
9740 .as_ref()
9741 .expect("configureTransactionState builds a manager when enable.idempotence is true");
9742 let manager = transaction_manager.lock().unwrap();
9743 assert!(!manager.is_transactional());
9744 assert!(
9745 !manager.has_producer_id(),
9746 "the producer id is acquired asynchronously by the Sender task"
9747 );
9748 }
9749
9750 /// `enable.idempotence=false` builds no manager at all, mirroring Java's `null`
9751 /// return from `configureTransactionState` (`KafkaProducer#configureTransactionState`).
9752 #[tokio::test]
9753 async fn test_disabled_idempotence_builds_no_transaction_manager() {
9754 let props = guard_props(&[("enable.idempotence", "false")]);
9755 let config = ProducerConfig::new(&props).expect("valid config");
9756 let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
9757 .expect("disabling idempotence is allowed");
9758 assert!(producer.transaction_manager.is_none());
9759 }
9760
9761 /// `transactional.id` is accepted as of Phase 6, and the producer really is
9762 /// transactional: `configureTransactionState` passes the id through to the
9763 /// manager (`KafkaProducer#configureTransactionState`) and `isTransactional()` reports it.
9764 ///
9765 /// This replaces the Phase-1 `test_guard_rejects_transactional_id`, whose whole
9766 /// subject — the `new` guard of PLAN §7.1 — is what this phase deleted.
9767 #[tokio::test]
9768 async fn test_transactional_id_builds_a_transactional_manager() {
9769 let props = guard_props(&[("transactional.id", "my-txn")]);
9770 let config = ProducerConfig::new(&props).expect("valid config");
9771 let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
9772 .expect("transactional.id is supported as of Phase 6");
9773 let transaction_manager = producer
9774 .transaction_manager
9775 .as_ref()
9776 .expect("configureTransactionState builds a manager when transactional.id is set");
9777 let manager = transaction_manager.lock().unwrap();
9778 assert!(manager.is_transactional());
9779 assert_eq!(manager.transactional_id(), Some("my-txn"));
9780 }
9781
9782 /// A producer with no `transactional.id` rejects every transactional method
9783 /// with Java's `throwIfNoTransactionManager` message — but only when there is
9784 /// no manager at all, i.e. `enable.idempotence=false`. An *idempotent* producer
9785 /// has a manager and is rejected one level down; the next test covers that.
9786 ///
9787 /// Java's message is built at `KafkaProducer`.
9788 #[tokio::test]
9789 async fn test_transactional_methods_without_a_manager() {
9790 let props = guard_props(&[("enable.idempotence", "false")]);
9791 let config = ProducerConfig::new(&props).expect("valid config");
9792 let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
9793 .expect("disabling idempotence is allowed");
9794
9795 const EXPECTED: &str = "Cannot use transactional methods without enabling transactions \
9796 by setting the transactional.id configuration property";
9797
9798 let expect_no_manager = |error: Error, method: &str| {
9799 assert_eq!(error.message(), EXPECTED, "{} reported the wrong error", method);
9800 };
9801 expect_no_manager(producer.init_transactions().await.expect_err("no manager"), "init_transactions");
9802 expect_no_manager(producer.begin_transaction().expect_err("no manager"), "begin_transaction");
9803 expect_no_manager(
9804 producer.commit_transaction().await.expect_err("no manager"),
9805 "commit_transaction",
9806 );
9807 expect_no_manager(producer.abort_transaction().await.expect_err("no manager"), "abort_transaction");
9808 let group_metadata = ConsumerGroupMetadataImpl::new("group");
9809 expect_no_manager(
9810 producer
9811 .send_offsets_to_transaction(
9812 HashMap::from([(
9813 TopicPartition::new(TOPIC.to_string(), 0),
9814 OffsetAndMetadata::new(1).expect("a non-negative offset"),
9815 )]),
9816 &group_metadata,
9817 )
9818 .await
9819 .expect_err("no manager"),
9820 "send_offsets_to_transaction",
9821 );
9822 }
9823
9824 /// An idempotent (non-transactional) producer *does* hold a manager, so Java's
9825 /// `throwIfNoTransactionManager` passes and the rejection comes from
9826 /// `TransactionManager.ensureTransactional()`
9827 /// with a different message. Pinning both messages is what proves
9828 /// [`KafkaProducer::transaction_manager_or_error`] does not additionally test
9829 /// `is_transactional()`.
9830 #[tokio::test]
9831 async fn test_transactional_methods_on_an_idempotent_producer() {
9832 let props = guard_props(&[("enable.idempotence", "true")]);
9833 let config = ProducerConfig::new(&props).expect("valid config");
9834 let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
9835 .expect("explicit idempotence is supported as of Phase 4");
9836
9837 const EXPECTED: &str = "Transactional method invoked on a non-transactional producer.";
9838 assert_eq!(
9839 producer.init_transactions().await.expect_err("not transactional").message(),
9840 EXPECTED
9841 );
9842 assert_eq!(producer.begin_transaction().expect_err("not transactional").message(), EXPECTED);
9843 assert_eq!(
9844 producer.commit_transaction().await.expect_err("not transactional").message(),
9845 EXPECTED
9846 );
9847 assert_eq!(
9848 producer.abort_transaction().await.expect_err("not transactional").message(),
9849 EXPECTED
9850 );
9851 }
9852
9853 /// `metadata.cluster.check.enable` (KIP-1242) reaches the producer's
9854 /// `NetworkClient`, as `ClientUtils.createNetworkClient` passes it
9855 /// (`ClientUtils#createNetworkClient`). Goes through the REAL constructor (DoD #12)
9856 /// against an in-process fake broker and observes the wire: the bootstrap
9857 /// connection's ApiVersions carries no cluster id (it is not known yet), and
9858 /// the connection to the partition leader (node 0, learned from the Metadata
9859 /// response) carries `ClusterId`/`NodeId` only when the check is enabled —
9860 /// the default. Java's broker-backed analogue is
9861 /// `ClientRebootstrapTest.testRebootstrapOnMetadataClusterCheckFail`.
9862 #[tokio::test(flavor = "multi_thread")]
9863 async fn metadata_cluster_check_enable_reaches_the_network_client() {
9864 use crate::producer::Producer;
9865 use crate::test_fake_broker::{FAKE_BROKER_NODE_ID, FakeBroker, ReceivedApiVersions};
9866
9867 const TOPIC: &str = "cluster-check-topic";
9868 for (configured, check) in [(None, true), (Some("false"), false), (Some("true"), true)] {
9869 let broker = FakeBroker::start("fake-cluster", TOPIC).await;
9870 // Non-idempotent, so the Sender does not first wait on an
9871 // InitProducerId the fake never answers.
9872 let mut props = HashMap::from([
9873 ("bootstrap.servers".to_string(), broker.bootstrap()),
9874 ("enable.idempotence".to_string(), "false".to_string()),
9875 ]);
9876 if let Some(value) = configured {
9877 props.insert("metadata.cluster.check.enable".to_string(), value.to_string());
9878 }
9879 let config = ProducerConfig::new(&props).expect("valid config");
9880 let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
9881 .expect("constructs against the fake broker");
9882 // The send waits for the topic's metadata, then the batch drains to
9883 // the leader, which opens the second connection. The Produce request
9884 // is never answered; only the ApiVersions requests matter.
9885 let _pending = producer
9886 .send(ProducerRecord::new(TOPIC.to_string(), Some("value".to_string())))
9887 .await
9888 .expect("metadata arrives from the fake broker");
9889 let seen = broker.await_api_versions_requests(2, Duration::from_secs(30)).await;
9890 assert_eq!(2, seen.len(), "metadata.cluster.check.enable = {configured:?}: {seen:?}");
9891 let unchecked = ReceivedApiVersions { version: 5, cluster_id: None, node_id: -1 };
9892 assert_eq!(unchecked, seen[0], "bootstrap connection, check = {check}");
9893 let expected_leader = if check {
9894 ReceivedApiVersions {
9895 version: 5,
9896 cluster_id: Some("fake-cluster".to_string()),
9897 node_id: FAKE_BROKER_NODE_ID,
9898 }
9899 } else {
9900 unchecked
9901 };
9902 assert_eq!(expected_leader, seen[1], "leader connection, check = {check}");
9903 producer.close_with_timeout(Duration::ZERO).await.expect("close");
9904 }
9905 }
9906
9907 /// A closed and dropped producer frees its buffer pool and metadata. The
9908 /// accumulator's gauges capture the pool and the Sender's `metadata-age`
9909 /// gauge the metadata; before the registry's "count" gauge and the
9910 /// sensors' registry back-reference were weak, the registry kept both
9911 /// alive (the pool with its cached free-list buffers) for the life of the
9912 /// process. Java's garbage collector frees the same graph.
9913 #[tokio::test(flavor = "multi_thread")]
9914 async fn test_closed_producer_is_freed() {
9915 let config = ProducerConfig::new(&guard_props(&[])).expect("valid config");
9916 let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
9917 .expect("the producer constructs");
9918 let pool = Arc::downgrade(producer.accumulator.buffer_pool_for_test());
9919 let metadata = Arc::downgrade(&producer.metadata);
9920 let metrics = Arc::downgrade(&producer.metrics);
9921 producer.close().await.expect("close");
9922 drop(producer);
9923
9924 // Spawned tasks holding a clone end shortly after the close.
9925 let deadline = std::time::Instant::now() + Duration::from_secs(5);
9926 while (pool.strong_count() > 0 || metadata.strong_count() > 0 || metrics.strong_count() > 0)
9927 && std::time::Instant::now() < deadline
9928 {
9929 tokio::time::sleep(Duration::from_millis(10)).await;
9930 }
9931 assert_eq!(
9932 (0, 0, 0),
9933 (pool.strong_count(), metadata.strong_count(), metrics.strong_count()),
9934 "BufferPool / ProducerMetadata / Metrics still referenced"
9935 );
9936 }
9937
9938 /// A producer dropped without `close` still delivers the record it holds, as
9939 /// Java's does: Java's I/O thread is a daemon (built in the `KafkaProducer` constructor) that
9940 /// keeps sending after nothing references the producer. Dropping it used to
9941 /// force-close the sender, which failed the record with "Producer is closed
9942 /// forcefully.".
9943 #[tokio::test(flavor = "multi_thread")]
9944 async fn a_producer_dropped_without_close_delivers_the_record_it_holds() {
9945 use crate::producer::Producer;
9946 use crate::test_fake_broker::FakeBroker;
9947
9948 const TOPIC: &str = "dropped-producer-topic";
9949 let broker = FakeBroker::start_acknowledging_produce("fake-cluster", TOPIC).await;
9950 let props = HashMap::from([
9951 ("bootstrap.servers".to_string(), broker.bootstrap()),
9952 // Non-idempotent, so the Sender does not first wait on an
9953 // InitProducerId the fake never answers.
9954 ("enable.idempotence".to_string(), "false".to_string()),
9955 // The record stays in the accumulator until the producer is dropped.
9956 ("linger.ms".to_string(), "60000".to_string()),
9957 ]);
9958 let config = ProducerConfig::new(&props).expect("valid config");
9959 let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
9960 .expect("constructs against the fake broker");
9961 let future = producer
9962 .send(ProducerRecord::new(TOPIC.to_string(), Some("value".to_string())))
9963 .await
9964 .expect("metadata arrives from the fake broker");
9965 assert!(!future.is_done(), "the record must still be held when the producer is dropped");
9966
9967 drop(producer);
9968
9969 let metadata = future
9970 .get_with_timeout(Duration::from_secs(30))
9971 .await
9972 .expect("the record must be delivered after the drop");
9973 assert_eq!(0, metadata.offset());
9974 }
9975
9976 /// A broker with `connections.max.reauth.ms` set gives each SASL session a
9977 /// lifetime and closes a connection whose session has expired when it next
9978 /// receives a request other than a re-authentication. Java's client
9979 /// re-authenticates the connection before then (KIP-368), so its requests
9980 /// keep flowing over the same connection. This client used to ignore the
9981 /// lifetime: the broker closed its connection every lifetime, and the
9982 /// requests in flight failed and were retried.
9983 #[tokio::test(flavor = "multi_thread")]
9984 async fn a_sasl_producer_reauthenticates_before_its_session_expires() {
9985 use crate::producer::Producer;
9986 use crate::test_fake_broker::FakeBroker;
9987
9988 const TOPIC: &str = "reauthentication-topic";
9989 const SESSION_LIFETIME_MS: u64 = 500;
9990 let broker = FakeBroker::start_acknowledging_produce("fake-cluster", TOPIC).await;
9991 broker.set_sasl_session_lifetime_ms(SESSION_LIFETIME_MS as i64);
9992 let props = HashMap::from([
9993 ("bootstrap.servers".to_string(), broker.bootstrap()),
9994 // Non-idempotent, so the Sender does not first wait on an
9995 // InitProducerId the fake never answers.
9996 ("enable.idempotence".to_string(), "false".to_string()),
9997 // A record whose connection is closed under it fails instead of
9998 // being retried, so a lost connection shows up as a failed send too.
9999 ("retries".to_string(), "0".to_string()),
10000 ("security.protocol".to_string(), "SASL_PLAINTEXT".to_string()),
10001 ("sasl.mechanism".to_string(), "PLAIN".to_string()),
10002 (
10003 "sasl.jaas.config".to_string(),
10004 "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"u\" password=\"p\";"
10005 .to_string(),
10006 ),
10007 ]);
10008 let config = ProducerConfig::new(&props).expect("valid config");
10009 let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
10010 .expect("constructs against the fake broker");
10011
10012 // One record at a time for four session lifetimes.
10013 let deadline = std::time::Instant::now() + Duration::from_millis(4 * SESSION_LIFETIME_MS);
10014 let mut sent = 0;
10015 while std::time::Instant::now() < deadline {
10016 producer
10017 .send(ProducerRecord::new(TOPIC.to_string(), Some(format!("value-{sent}"))))
10018 .await
10019 .expect("metadata arrives from the fake broker")
10020 .get_with_timeout(Duration::from_secs(10))
10021 .await
10022 .unwrap_or_else(|e| panic!("record {sent} must be acknowledged: {e:?}"));
10023 sent += 1;
10024 tokio::time::sleep(Duration::from_millis(20)).await;
10025 }
10026 producer.close().await.expect("close");
10027
10028 assert_eq!(
10029 0,
10030 broker.expired_connections_killed(),
10031 "the broker had to close a connection whose session expired; {sent} records sent"
10032 );
10033 // The bootstrap and the leader connection each authenticated once; the
10034 // leader connection, used throughout, re-authenticated about once a
10035 // lifetime.
10036 let authentications = broker.sasl_authentications();
10037 assert!(
10038 authentications >= 2 + 3,
10039 "{authentications} SASL authentications in four session lifetimes"
10040 );
10041 }
10042
10043 /// As [`a_sasl_producer_reauthenticates_before_its_session_expires`], with
10044 /// up to five requests in flight on the leader's connection when its
10045 /// session falls due, so responses to them arrive while it re-authenticates.
10046 /// The authenticator sets each aside and the selector hands it to the
10047 /// network client, at most one per channel per pass, as Java's does once
10048 /// per `poll()`.
10049 #[tokio::test(flavor = "multi_thread")]
10050 async fn a_sasl_producer_with_requests_in_flight_reauthenticates() {
10051 use crate::producer::Producer;
10052 use crate::test_fake_broker::FakeBroker;
10053
10054 const TOPIC: &str = "reauthentication-in-flight-topic";
10055 const SESSION_LIFETIME_MS: u64 = 500;
10056 let broker = FakeBroker::start_acknowledging_produce("fake-cluster", TOPIC).await;
10057 broker.set_sasl_session_lifetime_ms(SESSION_LIFETIME_MS as i64);
10058 let props = HashMap::from([
10059 ("bootstrap.servers".to_string(), broker.bootstrap()),
10060 ("enable.idempotence".to_string(), "false".to_string()),
10061 ("retries".to_string(), "0".to_string()),
10062 ("linger.ms".to_string(), "0".to_string()),
10063 ("batch.size".to_string(), "16".to_string()),
10064 ("max.in.flight.requests.per.connection".to_string(), "5".to_string()),
10065 ("security.protocol".to_string(), "SASL_PLAINTEXT".to_string()),
10066 ("sasl.mechanism".to_string(), "PLAIN".to_string()),
10067 (
10068 "sasl.jaas.config".to_string(),
10069 "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"u\" password=\"p\";"
10070 .to_string(),
10071 ),
10072 ]);
10073 let config = ProducerConfig::new(&props).expect("valid config");
10074 let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
10075 .expect("constructs against the fake broker");
10076
10077 let deadline = std::time::Instant::now() + Duration::from_millis(4 * SESSION_LIFETIME_MS);
10078 let mut sent = 0;
10079 while std::time::Instant::now() < deadline {
10080 // A burst of one-record batches, each its own request.
10081 let mut futures = Vec::new();
10082 for _ in 0..20 {
10083 futures.push(
10084 producer
10085 .send(ProducerRecord::new(TOPIC.to_string(), Some(format!("value-{sent}"))))
10086 .await
10087 .expect("metadata arrives from the fake broker"),
10088 );
10089 sent += 1;
10090 }
10091 for future in futures {
10092 future
10093 .get_with_timeout(Duration::from_secs(10))
10094 .await
10095 .unwrap_or_else(|e| panic!("every record must be acknowledged: {e:?}"));
10096 }
10097 }
10098 producer.close().await.expect("close");
10099
10100 assert_eq!(0, broker.expired_connections_killed(), "{sent} records sent");
10101 assert!(broker.sasl_authentications() >= 2 + 3, "{}", broker.sasl_authentications());
10102 }
10103
10104 /// `buffer.memory=0` builds a producer, as Java's `atLeast(0)` validator
10105 /// and `BufferPool` constructor allow, and each send fails in
10106 /// `KafkaProducer#ensureValidRecordSize` with Java's
10107 /// message, under both allocation strategies. Construction used to panic
10108 /// with "Buffer pool memory must be positive".
10109 #[tokio::test(flavor = "multi_thread")]
10110 async fn a_producer_with_no_buffer_memory_fails_each_send_as_java_does() {
10111 use crate::producer::Producer;
10112 use crate::test_fake_broker::FakeBroker;
10113
10114 const TOPIC: &str = "no-buffer-memory-topic";
10115 let broker = FakeBroker::start("fake-cluster", TOPIC).await;
10116 for strategy in [
10117 ProducerConfig::BUFFER_MEMORY_ALLOCATION_STRATEGY_FULL,
10118 ProducerConfig::BUFFER_MEMORY_ALLOCATION_STRATEGY_INCREMENTAL,
10119 ] {
10120 let props = HashMap::from([
10121 ("bootstrap.servers".to_string(), broker.bootstrap()),
10122 (ProducerConfig::BUFFER_MEMORY_CONFIG.to_string(), "0".to_string()),
10123 (
10124 ProducerConfig::BUFFER_MEMORY_ALLOCATION_STRATEGY_CONFIG.to_string(),
10125 strategy.to_string(),
10126 ),
10127 // Non-idempotent, so the Sender does not first wait on an
10128 // InitProducerId the fake never answers.
10129 ("enable.idempotence".to_string(), "false".to_string()),
10130 ]);
10131 let config = ProducerConfig::new(&props).expect("buffer.memory=0 is a valid config");
10132 let producer = KafkaProducer::<String, String>::new(config, StringSerializer, StringSerializer)
10133 .expect("a producer with no buffer memory constructs");
10134 assert_eq!(
10135 strategy == ProducerConfig::BUFFER_MEMORY_ALLOCATION_STRATEGY_INCREMENTAL,
10136 producer.chunked_accumulator.is_some(),
10137 "{strategy}: the accumulator this strategy selects"
10138 );
10139 let future = producer
10140 .send(ProducerRecord::new(TOPIC.to_string(), Some("value".to_string())))
10141 .await
10142 .expect("metadata arrives from the fake broker");
10143 let err = future.get().await.expect_err("no record fits in no buffer memory");
10144 assert!(matches!(err, Error::RecordTooLarge(_)), "{strategy}: {err:?}");
10145 let size = AbstractRecords::estimate_size_in_bytes_upper_bound(
10146 RecordBatch::CURRENT_MAGIC_VALUE,
10147 CompressionType::None,
10148 None,
10149 Some(b"value"),
10150 &[],
10151 );
10152 assert_eq!(
10153 format!(
10154 "The message is {size} bytes when serialized which is larger than the total memory buffer you \
10155 have configured with the buffer.memory configuration."
10156 ),
10157 err.message(),
10158 "{strategy}"
10159 );
10160 producer.close_with_timeout(Duration::ZERO).await.expect("close");
10161 }
10162 }
10163}