confluent_kafka/producer/producer.rs
1// Copyright 2025 Confluent Inc.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! The Producer trait — the interface for the KafkaProducer.
16//!
17//! Translated from `org.apache.kafka.clients.producer.Producer`.
18
19use std::collections::HashMap;
20use std::time::Duration;
21
22use crate::common::Error;
23use crate::common::KafkaFuture;
24use crate::common::PartitionInfo;
25use crate::common::TopicPartition;
26use crate::common::metrics::MetricsView;
27use crate::consumer::ConsumerGroupMetadata;
28use crate::consumer::OffsetAndMetadata;
29use crate::producer::Callback;
30use crate::producer::ProducerRecord;
31use crate::producer::RecordMetadata;
32
33/// The interface for the [`KafkaProducer`](super::KafkaProducer).
34// Every asynchronous method returns `impl Future + Send` rather than being
35// declared `async fn`, so that generic code can move the returned futures across
36// tasks and box them as `dyn Future + Send`. That guarantee is what lets
37// `DynProducer` be blanket-implemented for every `Producer`. Implementations may
38// still be written with `async fn`.
39#[doc(alias = "org.apache.kafka.clients.producer.Producer")]
40pub trait Producer<K, V>: Send + Sync {
41 /// Needs to be called before any other methods when the `transactional.id`
42 /// is set in the configuration.
43 ///
44 /// See [`KafkaProducer::init_transactions`](super::KafkaProducer::init_transactions).
45 ///
46 /// # Errors
47 ///
48 /// - `Error::is_local_illegal_state_error` if no `transactional.id` has been configured
49 /// - `Error::is_unsupported_version_error` fatal error indicating the broker does not support
50 /// transactions (i.e. if its version is lower than 0.11.0.0)
51 /// - `Error::is_authorization_error` error indicating that the configured transactional.id is
52 /// not authorized, or the idempotent producer id is unavailable. See the error for more
53 /// details. User may retry this function call after fixing the permission.
54 /// - `Error::is_kafka_error` if the producer has encountered a previous fatal error or for any
55 /// other unexpected error
56 /// - `Error::is_timeout_error` if the time taken for initialize the transaction has surpassed
57 /// `max.block.ms`.
58 /// - `Error::is_interrupt_error` if the task is interrupted while blocked
59 ///
60 /// # Translation notes
61 ///
62 /// A Rust task is never interrupted: dropping the future cancels the wait.
63 #[doc(alias = "org.apache.kafka.clients.producer.Producer#initTransactions")]
64 fn init_transactions(&self) -> impl Future<Output = Result<(), Error>> + Send;
65
66 /// Should be called before the start of each new transaction. Note that
67 /// prior to the first invocation of this method, you must invoke
68 /// [`init_transactions`](Self::init_transactions) exactly one time.
69 ///
70 /// See [`KafkaProducer::begin_transaction`](super::KafkaProducer::begin_transaction).
71 ///
72 /// # Errors
73 ///
74 /// - `Error::is_local_illegal_state_error` if no `transactional.id` has been configured or if
75 /// `Self::init_transactions` has not yet been invoked
76 /// - `Error::is_producer_fenced_error` if another producer with the same transactional.id is
77 /// active
78 /// - `Error::is_invalid_producer_epoch_error` if the producer has attempted to produce with an
79 /// old epoch to the partition leader. See the error for more details
80 /// - `Error::is_unsupported_version_error` fatal error indicating the broker does not support
81 /// transactions (i.e. if its version is lower than 0.11.0.0)
82 /// - `Error::is_authorization_error` fatal error indicating that the configured
83 /// `transactional.id` is not authorized. See the error for more details
84 /// - `Error::is_kafka_error` if the producer has encountered a previous fatal error or for any
85 /// other unexpected error
86 ///
87 /// # Translation notes
88 ///
89 /// Synchronous, as Java's `beginTransaction` never blocks.
90 #[doc(alias = "org.apache.kafka.clients.producer.Producer#beginTransaction")]
91 fn begin_transaction(&self) -> Result<(), Error>;
92
93 /// Sends a list of specified offsets to the consumer group coordinator, and
94 /// also marks those offsets as part of the current transaction. These
95 /// offsets will be considered committed only if the transaction is
96 /// committed successfully.
97 ///
98 /// See [`KafkaProducer::send_offsets_to_transaction`](super::KafkaProducer::send_offsets_to_transaction).
99 ///
100 /// # Arguments
101 ///
102 /// - `offsets` - the offsets to commit, the next message your application will consume for
103 /// each partition
104 /// - `group_metadata` - the consumer group metadata, extracted from the consumer via
105 /// `KafkaConsumer::group_metadata`
106 ///
107 /// # Errors
108 ///
109 /// - `Error::is_local_illegal_state_error` if no transactional.id has been configured or no
110 /// transaction has been started.
111 /// - `Error::is_producer_fenced_error` fatal error indicating another producer with the same
112 /// transactional.id is active
113 /// - `Error::is_unsupported_version_error` fatal error indicating the broker does not support
114 /// transactions (i.e. if its version is lower than 0.11.0.0) or the broker doesn't support
115 /// the latest version of transactional API with all consumer group metadata (i.e. if its
116 /// version is lower than 2.5.0).
117 /// - `Error::is_unsupported_for_message_format_error` fatal error indicating the message format
118 /// used for the offsets topic on the broker does not support transactions
119 /// - `Error::is_authorization_error` fatal error indicating that the configured
120 /// transactional.id is not authorized, or the consumer group id is not authorized.
121 /// - `Error::is_consumer_commit_failed_error` if the commit failed and cannot be retried (e.g.
122 /// if the consumer has been kicked out of the group). Users should handle this by aborting
123 /// the transaction.
124 /// - `Error::is_fenced_instance_id_error` if this producer instance gets fenced by broker due
125 /// to a mis-configured consumer instance id within group metadata.
126 /// - `Error::is_invalid_producer_epoch_error` if the producer has attempted to produce with an
127 /// old epoch to the partition leader. See the error for more details
128 /// - `Error::is_kafka_error` if the producer has encountered a previous fatal or abortable
129 /// error, or for any other unexpected error
130 /// - `Error::is_timeout_error` if the combined time taken for resolving topic metadata and
131 /// sending the offsets has surpassed `max.block.ms`.
132 /// - `Error::is_interrupt_error` if the task is interrupted while blocked
133 /// - `Error::is_bootstrap_resolution_error` if DNS resolution of the bootstrap servers fails
134 /// within `bootstrap.resolve.timeout.ms`
135 ///
136 /// # Translation notes
137 ///
138 /// A Rust task is never interrupted: dropping the future cancels the wait.
139 #[doc(alias = "org.apache.kafka.clients.producer.Producer#sendOffsetsToTransaction")]
140 fn send_offsets_to_transaction(
141 &self,
142 offsets: HashMap<TopicPartition, OffsetAndMetadata>,
143 group_metadata: &dyn ConsumerGroupMetadata,
144 ) -> impl Future<Output = Result<(), Error>> + Send;
145
146 /// Commits the ongoing transaction. This method will flush any unsent
147 /// records before actually committing the transaction.
148 ///
149 /// See [`KafkaProducer::commit_transaction`](super::KafkaProducer::commit_transaction).
150 ///
151 /// # Errors
152 ///
153 /// - `Error::is_local_illegal_state_error` if no transactional.id has been configured or no
154 /// transaction has been started
155 /// - `Error::is_producer_fenced_error` fatal error indicating another producer with the same
156 /// transactional.id is active
157 /// - `Error::is_unsupported_version_error` fatal error indicating the broker does not support
158 /// transactions (i.e. if its version is lower than 0.11.0.0)
159 /// - `Error::is_authorization_error` fatal error indicating that the configured
160 /// transactional.id is not authorized. See the error for more details
161 /// - `Error::is_invalid_producer_epoch_error` if the producer has attempted to produce with an
162 /// old epoch to the partition leader. See the error for more details
163 /// - `Error::is_kafka_error` if the producer has encountered a previous fatal or abortable
164 /// error, or for any other unexpected error
165 /// - `Error::is_timeout_error` if the time taken for committing the transaction has surpassed
166 /// `max.block.ms`.
167 /// - `Error::is_interrupt_error` if the task is interrupted while blocked
168 ///
169 /// # Translation notes
170 ///
171 /// A Rust task is never interrupted: dropping the future cancels the wait.
172 #[doc(alias = "org.apache.kafka.clients.producer.Producer#commitTransaction")]
173 fn commit_transaction(&self) -> impl Future<Output = Result<(), Error>> + Send;
174
175 /// Aborts the ongoing transaction. Any unflushed produce messages will be
176 /// aborted when this call is made.
177 ///
178 /// See [`KafkaProducer::abort_transaction`](super::KafkaProducer::abort_transaction).
179 ///
180 /// # Errors
181 ///
182 /// - `Error::is_local_illegal_state_error` if no transactional.id has been configured or no
183 /// transaction has been started
184 /// - `Error::is_producer_fenced_error` fatal error indicating another producer with the same
185 /// transactional.id is active
186 /// - `Error::is_invalid_producer_epoch_error` if the producer has attempted to produce with an
187 /// old epoch to the partition leader. See the error for more details
188 /// - `Error::is_unsupported_version_error` fatal error indicating the broker does not support
189 /// transactions (i.e. if its version is lower than 0.11.0.0)
190 /// - `Error::is_authorization_error` fatal error indicating that the configured
191 /// transactional.id is not authorized. See the error for more details
192 /// - `Error::is_kafka_error` if the producer has encountered a previous fatal error or for any
193 /// other unexpected error
194 /// - `Error::is_timeout_error` if the time taken for aborting the transaction has surpassed
195 /// `max.block.ms`.
196 /// - `Error::is_interrupt_error` if the task is interrupted while blocked
197 ///
198 /// # Translation notes
199 ///
200 /// A Rust task is never interrupted: dropping the future cancels the wait.
201 #[doc(alias = "org.apache.kafka.clients.producer.Producer#abortTransaction")]
202 fn abort_transaction(&self) -> impl Future<Output = Result<(), Error>> + Send;
203
204 /// Asynchronously send a record to a topic. Equivalent to
205 /// `send_with_callback(record, None)`. See
206 /// [`send_with_callback`](Producer::send_with_callback) for details.
207 ///
208 /// # Arguments
209 ///
210 /// - `record` - The record to send. If the topic or the partition specified in it cannot be
211 /// found in metadata within `max.block.ms`, the returned future will time out when retrieved.
212 ///
213 /// # Returns
214 ///
215 /// A future which will eventually contain the response information.
216 ///
217 /// # Errors
218 ///
219 /// - `Error::is_local_illegal_state_error` if a transactional.id has been configured and no
220 /// transaction has been started, or when send is invoked after producer has been closed.
221 /// - `Error::is_interrupt_error` If the task is interrupted while blocked
222 /// - `Error::is_serialization_error` If the key or value are not valid objects given the
223 /// configured serializers
224 /// - `Error::is_bootstrap_resolution_error` if DNS resolution of the bootstrap servers fails
225 /// within `bootstrap.resolve.timeout.ms`
226 /// - `Error::is_kafka_error` If a Kafka related error occurs that does not belong to the public
227 /// API errors.
228 ///
229 /// # Translation notes
230 ///
231 /// A Rust task is never interrupted: dropping the future cancels the wait.
232 #[doc(alias = "org.apache.kafka.clients.producer.Producer#send")]
233 fn send(
234 &self,
235 record: ProducerRecord<K, V>,
236 ) -> impl Future<Output = Result<KafkaFuture<RecordMetadata>, Error>> + Send;
237
238 /// Asynchronously send a record to a topic and invoke the provided callback
239 /// when the send has been acknowledged.
240 ///
241 /// The send is asynchronous and this method will return immediately (except
242 /// for rare cases described below) once the record has been stored in the
243 /// buffer of records waiting to be sent. This allows sending many records in
244 /// parallel without blocking to wait for the response after each one. Can
245 /// block for the following cases: 1) For the first record being sent to the
246 /// cluster by this client for the given topic. In this case it will block for
247 /// up to `max.block.ms` milliseconds while waiting for topic's metadata if
248 /// Kafka cluster is unreachable; 2) Allocating a buffer if buffer pool
249 /// doesn't have any free buffers.
250 ///
251 /// **Reducing first-send latency:** You can reduce the latency of the first
252 /// send by preloading the metadata with
253 /// [`partitions_for`](Self::partitions_for). However, be aware that metadata
254 /// cache will be cleared to free up resources after `metadata.max.idle.ms`
255 /// of inactivity, so subsequent sends after a long idle period will still
256 /// experience delays.
257 ///
258 /// The result of the send is a [`RecordMetadata`] specifying the partition
259 /// the record was sent to, the offset it was assigned and the timestamp of
260 /// the record. If the producer is configured with acks = 0, the
261 /// `RecordMetadata` will have offset = -1 because the producer does not wait
262 /// for the acknowledgement from the broker. If CreateTime is used by the
263 /// topic, the timestamp will be the user provided timestamp or the record
264 /// send time if the user did not specify a timestamp for the record. If
265 /// LogAppendTime is used for the topic, the timestamp will be the Kafka
266 /// broker local time when the message is appended.
267 ///
268 /// Since the send call is asynchronous it returns a future for the
269 /// `RecordMetadata` that will be assigned to this record. Awaiting this
270 /// future waits until the associated request completes and then returns
271 /// the metadata for the record or the error that occurred while sending the
272 /// record. Fully non-blocking usage can make use of the `callback` parameter
273 /// to provide a callback that will be invoked when the request is complete.
274 ///
275 /// Callbacks for records being sent to the same partition are guaranteed to
276 /// execute in order.
277 ///
278 /// When used as part of a transaction, it is not necessary to define a
279 /// callback or check the result of the future in order to detect errors
280 /// from `send`. If any of the send calls failed with an irrecoverable error,
281 /// the final [`commit_transaction`](Self::commit_transaction) call will fail
282 /// and return the error from the last failed send. When this happens, your
283 /// application should call [`abort_transaction`](Self::abort_transaction)
284 /// to reset the state and continue to send data.
285 ///
286 /// Some transactional send errors cannot be resolved with a call to
287 /// `abort_transaction`. In particular, if a transactional send finishes with
288 /// a `ProducerFencedError`, a `OutOfOrderSequenceError`, a
289 /// `UnsupportedVersionError`, or an `AuthorizationError`, then the only
290 /// option left is to call [`close`](Self::close). Fatal errors cause the
291 /// producer to enter a defunct state in which future API calls will continue
292 /// to raise the same underlying error wrapped in a new `KafkaError`.
293 ///
294 /// It is a similar picture when idempotence is enabled, but no
295 /// `transactional.id` has been configured. In this case,
296 /// `UnsupportedVersionError` and `AuthorizationError` are considered fatal
297 /// errors. However, `ProducerFencedError` does not need to be handled.
298 /// Additionally, it is possible to continue sending after receiving an
299 /// `OutOfOrderSequenceError`, but doing so can result in out of order
300 /// delivery of pending messages. To ensure proper ordering, you should close
301 /// the producer and create a new instance.
302 ///
303 /// If the message format of the destination topic is not upgraded to
304 /// 0.11.0.0, idempotent and transactional produce requests will fail with an
305 /// `UnsupportedForMessageFormatError` error. If this is encountered during a
306 /// transaction, it is possible to abort and continue. But note that future
307 /// sends to the same topic will continue receiving the same error until the
308 /// topic is upgraded.
309 ///
310 /// Note that callbacks will generally execute in the I/O task of the
311 /// producer and so should be reasonably fast or they will delay the sending
312 /// of messages from other tasks. If you want to execute blocking or
313 /// computationally expensive callbacks it is recommended to hand the work
314 /// to another task from the callback body.
315 ///
316 /// # Arguments
317 ///
318 /// - `record` - The record to send. If the topic or the partition specified in it cannot be
319 /// found in metadata within `max.block.ms`, the returned future will time out when retrieved.
320 /// - `callback` - A user-supplied callback to execute when the record has been acknowledged by
321 /// the server (`None` indicates no callback)
322 ///
323 /// # Returns
324 ///
325 /// A future which will eventually contain the response information.
326 ///
327 /// # Errors
328 ///
329 /// - `Error::is_local_illegal_state_error` if a transactional.id has been configured and no
330 /// transaction has been started, or when send is invoked after producer has been closed.
331 /// - `Error::is_interrupt_error` If the task is interrupted while blocked
332 /// - `Error::is_serialization_error` If the key or value are not valid objects given the
333 /// configured serializers
334 /// - `Error::is_bootstrap_resolution_error` if DNS resolution of the bootstrap servers fails
335 /// within `bootstrap.resolve.timeout.ms`
336 /// - `Error::is_kafka_error` If a Kafka related error occurs that does not belong to the public
337 /// API errors.
338 ///
339 /// # Translation notes
340 ///
341 /// A Rust task is never interrupted: dropping the future cancels the wait.
342 #[doc(alias = "org.apache.kafka.clients.producer.Producer#send")]
343 fn send_with_callback(
344 &self,
345 record: ProducerRecord<K, V>,
346 callback: Option<Box<dyn Callback>>,
347 ) -> impl Future<Output = Result<KafkaFuture<RecordMetadata>, Error>> + Send;
348
349 /// Invoking this method makes all buffered records immediately available to
350 /// send (even if `linger.ms` is greater than 0) and blocks on the completion
351 /// of the requests associated with these records. The post-condition of
352 /// `flush()` is that any previously sent record will have completed (e.g.
353 /// its future is done and callbacks passed to `Self::send_with_callback`
354 /// have been called). A request is considered completed when it is
355 /// successfully acknowledged according to the `acks` configuration you have
356 /// specified or else it results in an error.
357 ///
358 /// Other tasks can continue sending records while one task is blocked
359 /// waiting for a flush call to complete, however no guarantee is made about
360 /// the completion of records sent after the flush call begins.
361 ///
362 /// This method can be useful when consuming from some input system and
363 /// producing into Kafka. The `flush()` call gives a convenient way to ensure
364 /// all previously sent messages have actually completed. Note that records
365 /// may be dropped if the produce request fails; to ensure that this does not
366 /// occur set `retries=<large_number>` in the config.
367 ///
368 /// Applications don't need to call this method for transactional producers,
369 /// since the [`commit_transaction`](Self::commit_transaction) will flush all
370 /// buffered records before performing the commit. This ensures that all the
371 /// `send` calls made since the previous
372 /// [`begin_transaction`](Self::begin_transaction) are completed before the
373 /// commit.
374 ///
375 /// **Important:** This method must not be called from within the callback
376 /// provided to `Self::send_with_callback`. Invoking `flush()` in this
377 /// context will result in a `KafkaError`, as it will cause a deadlock.
378 ///
379 /// # Errors
380 ///
381 /// - `Error::is_interrupt_error` If the task is interrupted while blocked
382 /// - `Error::is_kafka_error` If the method is invoked inside a `Self::send` callback
383 ///
384 /// # Translation notes
385 ///
386 /// A Rust task is never interrupted: dropping the future cancels the wait.
387 /// [`Callback::on_completion`] is synchronous, so it cannot await this
388 /// method.
389 #[doc(alias = "org.apache.kafka.clients.producer.Producer#flush")]
390 fn flush(&self) -> impl Future<Output = Result<(), Error>> + Send;
391
392 /// Get the partition metadata for the given topic. This can be used for
393 /// custom partitioning.
394 ///
395 /// This will attempt to refresh metadata until it finds the topic in it, or
396 /// the configured `max.block.ms` expires.
397 ///
398 /// # Arguments
399 ///
400 /// - `topic` - The topic to get partition metadata for
401 ///
402 /// # Returns
403 ///
404 /// The list of partitions.
405 ///
406 /// # Errors
407 ///
408 /// - `Error::is_authentication_error` if authentication fails.
409 /// - `Error::is_authorization_error` if not authorized to the specified topic.
410 /// - `Error::is_interrupt_error` if the task is interrupted while blocked
411 /// - `Error::is_timeout_error` if the topic cannot be found in metadata within `max.block.ms`
412 /// - `Error::is_bootstrap_resolution_error` if DNS resolution of the bootstrap servers fails
413 /// within `bootstrap.resolve.timeout.ms`
414 /// - `Error::is_kafka_error` for all Kafka-related errors, including the case where this
415 /// method is called after producer close
416 ///
417 /// # Translation notes
418 ///
419 /// A Rust task is never interrupted: dropping the future cancels the wait.
420 #[doc(alias = "org.apache.kafka.clients.producer.Producer#partitionsFor")]
421 fn partitions_for(&self, topic: &str) -> impl Future<Output = Result<Vec<PartitionInfo>, Error>> + Send;
422
423 /// Get the full set of internal metrics maintained by the producer.
424 ///
425 /// The returned map is an unmodifiable live view of the metrics. Changes to
426 /// the underlying metrics will be reflected in the returned map.
427 ///
428 /// # Returns
429 ///
430 /// An unmodifiable live view of the map of metrics currently maintained by
431 /// the producer
432 #[doc(alias = "org.apache.kafka.clients.producer.Producer#metrics")]
433 fn metrics(&self) -> MetricsView;
434
435 /// Close this producer. This method blocks until all previously sent
436 /// requests complete. This method is equivalent to
437 /// `close_with_timeout(Duration::MAX)`.
438 ///
439 /// **If close() is called from a [`Callback`], a warning message will be
440 /// logged and `close_with_timeout(Duration::ZERO)` will be called instead.
441 /// We do this because the sender task would otherwise try to join itself and
442 /// block forever.**
443 ///
444 /// # Errors
445 ///
446 /// - `Error::is_interrupt_error` If the task is interrupted while blocked.
447 /// - `Error::is_kafka_error` If an unexpected error occurs while trying to close the client,
448 /// this error should be treated as fatal and indicate the client is no longer usable.
449 ///
450 /// # Translation notes
451 ///
452 /// A Rust task is never interrupted: dropping the future cancels the wait.
453 /// [`Callback::on_completion`] is synchronous, so it cannot await this
454 /// method.
455 #[doc(alias = "org.apache.kafka.clients.producer.Producer#close")]
456 fn close(&self) -> impl Future<Output = Result<(), Error>> + Send;
457
458 /// This method waits up to `timeout` for the producer to complete the
459 /// sending of all incomplete requests.
460 ///
461 /// If the producer is unable to complete all requests before the timeout
462 /// expires, this method will fail any unsent and unacknowledged records
463 /// immediately. It will also abort the ongoing transaction if it's not
464 /// already completing.
465 ///
466 /// If invoked from within a [`Callback`] this method will not block and will
467 /// be equivalent to `close_with_timeout(Duration::ZERO)`. This is done since
468 /// no further sending will happen while blocking the I/O task of the
469 /// producer.
470 ///
471 /// # Arguments
472 ///
473 /// - `timeout` - The maximum time to wait for producer to complete any pending requests. The
474 /// value should be non-negative. Specifying a timeout of zero means do not wait for pending
475 /// send requests to complete.
476 ///
477 /// # Errors
478 ///
479 /// - `Error::is_interrupt_error` If the task is interrupted while blocked.
480 /// - `Error::is_kafka_error` If an unexpected error occurs while trying to close the client,
481 /// this error should be treated as fatal and indicate the client is no longer usable.
482 /// - `Error::is_local_illegal_argument_error` If the `timeout` is negative.
483 ///
484 /// # Translation notes
485 ///
486 /// A Rust task is never interrupted: dropping the future cancels the wait.
487 /// A `Duration` is never negative. [`Callback::on_completion`] is
488 /// synchronous, so it cannot await this method.
489 #[doc(alias = "org.apache.kafka.clients.producer.Producer#close")]
490 fn close_with_timeout(&self, timeout: Duration) -> impl Future<Output = Result<(), Error>> + Send;
491}