Skip to main content

confluent_kafka/producer/
producer.rs

1// Copyright 2025 Confluent Inc.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! The Producer trait — the interface for the KafkaProducer.
16//!
17//! Translated from `org.apache.kafka.clients.producer.Producer`.
18
19use std::collections::HashMap;
20use std::time::Duration;
21
22use crate::common::Error;
23use crate::common::KafkaFuture;
24use crate::common::PartitionInfo;
25use crate::common::TopicPartition;
26use crate::common::metrics::MetricsView;
27use crate::consumer::ConsumerGroupMetadata;
28use crate::consumer::OffsetAndMetadata;
29use crate::producer::Callback;
30use crate::producer::ProducerRecord;
31use crate::producer::RecordMetadata;
32
33/// The interface for the [`KafkaProducer`](super::KafkaProducer).
34// Every asynchronous method returns `impl Future + Send` rather than being
35// declared `async fn`, so that generic code can move the returned futures across
36// tasks and box them as `dyn Future + Send`. That guarantee is what lets
37// `DynProducer` be blanket-implemented for every `Producer`. Implementations may
38// still be written with `async fn`.
39#[doc(alias = "org.apache.kafka.clients.producer.Producer")]
40pub trait Producer<K, V>: Send + Sync {
41    /// Needs to be called before any other methods when the `transactional.id`
42    /// is set in the configuration.
43    ///
44    /// See [`KafkaProducer::init_transactions`](super::KafkaProducer::init_transactions).
45    ///
46    /// # Errors
47    ///
48    /// - `Error::is_local_illegal_state_error` if no `transactional.id` has been configured
49    /// - `Error::is_unsupported_version_error` fatal error indicating the broker does not support
50    ///   transactions (i.e. if its version is lower than 0.11.0.0)
51    /// - `Error::is_authorization_error` error indicating that the configured transactional.id is
52    ///   not authorized, or the idempotent producer id is unavailable. See the error for more
53    ///   details. User may retry this function call after fixing the permission.
54    /// - `Error::is_kafka_error` if the producer has encountered a previous fatal error or for any
55    ///   other unexpected error
56    /// - `Error::is_timeout_error` if the time taken for initialize the transaction has surpassed
57    ///   `max.block.ms`.
58    /// - `Error::is_interrupt_error` if the task is interrupted while blocked
59    ///
60    /// # Translation notes
61    ///
62    /// A Rust task is never interrupted: dropping the future cancels the wait.
63    #[doc(alias = "org.apache.kafka.clients.producer.Producer#initTransactions")]
64    fn init_transactions(&self) -> impl Future<Output = Result<(), Error>> + Send;
65
66    /// Should be called before the start of each new transaction. Note that
67    /// prior to the first invocation of this method, you must invoke
68    /// [`init_transactions`](Self::init_transactions) exactly one time.
69    ///
70    /// See [`KafkaProducer::begin_transaction`](super::KafkaProducer::begin_transaction).
71    ///
72    /// # Errors
73    ///
74    /// - `Error::is_local_illegal_state_error` if no `transactional.id` has been configured or if
75    ///   `Self::init_transactions` has not yet been invoked
76    /// - `Error::is_producer_fenced_error` if another producer with the same transactional.id is
77    ///   active
78    /// - `Error::is_invalid_producer_epoch_error` if the producer has attempted to produce with an
79    ///   old epoch to the partition leader. See the error for more details
80    /// - `Error::is_unsupported_version_error` fatal error indicating the broker does not support
81    ///   transactions (i.e. if its version is lower than 0.11.0.0)
82    /// - `Error::is_authorization_error` fatal error indicating that the configured
83    ///   `transactional.id` is not authorized. See the error for more details
84    /// - `Error::is_kafka_error` if the producer has encountered a previous fatal error or for any
85    ///   other unexpected error
86    ///
87    /// # Translation notes
88    ///
89    /// Synchronous, as Java's `beginTransaction` never blocks.
90    #[doc(alias = "org.apache.kafka.clients.producer.Producer#beginTransaction")]
91    fn begin_transaction(&self) -> Result<(), Error>;
92
93    /// Sends a list of specified offsets to the consumer group coordinator, and
94    /// also marks those offsets as part of the current transaction. These
95    /// offsets will be considered committed only if the transaction is
96    /// committed successfully.
97    ///
98    /// See [`KafkaProducer::send_offsets_to_transaction`](super::KafkaProducer::send_offsets_to_transaction).
99    ///
100    /// # Arguments
101    ///
102    /// - `offsets` - the offsets to commit, the next message your application will consume for
103    ///   each partition
104    /// - `group_metadata` - the consumer group metadata, extracted from the consumer via
105    ///   `KafkaConsumer::group_metadata`
106    ///
107    /// # Errors
108    ///
109    /// - `Error::is_local_illegal_state_error` if no transactional.id has been configured or no
110    ///   transaction has been started.
111    /// - `Error::is_producer_fenced_error` fatal error indicating another producer with the same
112    ///   transactional.id is active
113    /// - `Error::is_unsupported_version_error` fatal error indicating the broker does not support
114    ///   transactions (i.e. if its version is lower than 0.11.0.0) or the broker doesn't support
115    ///   the latest version of transactional API with all consumer group metadata (i.e. if its
116    ///   version is lower than 2.5.0).
117    /// - `Error::is_unsupported_for_message_format_error` fatal error indicating the message format
118    ///   used for the offsets topic on the broker does not support transactions
119    /// - `Error::is_authorization_error` fatal error indicating that the configured
120    ///   transactional.id is not authorized, or the consumer group id is not authorized.
121    /// - `Error::is_consumer_commit_failed_error` if the commit failed and cannot be retried (e.g.
122    ///   if the consumer has been kicked out of the group). Users should handle this by aborting
123    ///   the transaction.
124    /// - `Error::is_fenced_instance_id_error` if this producer instance gets fenced by broker due
125    ///   to a mis-configured consumer instance id within group metadata.
126    /// - `Error::is_invalid_producer_epoch_error` if the producer has attempted to produce with an
127    ///   old epoch to the partition leader. See the error for more details
128    /// - `Error::is_kafka_error` if the producer has encountered a previous fatal or abortable
129    ///   error, or for any other unexpected error
130    /// - `Error::is_timeout_error` if the combined time taken for resolving topic metadata and
131    ///   sending the offsets has surpassed `max.block.ms`.
132    /// - `Error::is_interrupt_error` if the task is interrupted while blocked
133    /// - `Error::is_bootstrap_resolution_error` if DNS resolution of the bootstrap servers fails
134    ///   within `bootstrap.resolve.timeout.ms`
135    ///
136    /// # Translation notes
137    ///
138    /// A Rust task is never interrupted: dropping the future cancels the wait.
139    #[doc(alias = "org.apache.kafka.clients.producer.Producer#sendOffsetsToTransaction")]
140    fn send_offsets_to_transaction(
141        &self,
142        offsets: HashMap<TopicPartition, OffsetAndMetadata>,
143        group_metadata: &dyn ConsumerGroupMetadata,
144    ) -> impl Future<Output = Result<(), Error>> + Send;
145
146    /// Commits the ongoing transaction. This method will flush any unsent
147    /// records before actually committing the transaction.
148    ///
149    /// See [`KafkaProducer::commit_transaction`](super::KafkaProducer::commit_transaction).
150    ///
151    /// # Errors
152    ///
153    /// - `Error::is_local_illegal_state_error` if no transactional.id has been configured or no
154    ///   transaction has been started
155    /// - `Error::is_producer_fenced_error` fatal error indicating another producer with the same
156    ///   transactional.id is active
157    /// - `Error::is_unsupported_version_error` fatal error indicating the broker does not support
158    ///   transactions (i.e. if its version is lower than 0.11.0.0)
159    /// - `Error::is_authorization_error` fatal error indicating that the configured
160    ///   transactional.id is not authorized. See the error for more details
161    /// - `Error::is_invalid_producer_epoch_error` if the producer has attempted to produce with an
162    ///   old epoch to the partition leader. See the error for more details
163    /// - `Error::is_kafka_error` if the producer has encountered a previous fatal or abortable
164    ///   error, or for any other unexpected error
165    /// - `Error::is_timeout_error` if the time taken for committing the transaction has surpassed
166    ///   `max.block.ms`.
167    /// - `Error::is_interrupt_error` if the task is interrupted while blocked
168    ///
169    /// # Translation notes
170    ///
171    /// A Rust task is never interrupted: dropping the future cancels the wait.
172    #[doc(alias = "org.apache.kafka.clients.producer.Producer#commitTransaction")]
173    fn commit_transaction(&self) -> impl Future<Output = Result<(), Error>> + Send;
174
175    /// Aborts the ongoing transaction. Any unflushed produce messages will be
176    /// aborted when this call is made.
177    ///
178    /// See [`KafkaProducer::abort_transaction`](super::KafkaProducer::abort_transaction).
179    ///
180    /// # Errors
181    ///
182    /// - `Error::is_local_illegal_state_error` if no transactional.id has been configured or no
183    ///   transaction has been started
184    /// - `Error::is_producer_fenced_error` fatal error indicating another producer with the same
185    ///   transactional.id is active
186    /// - `Error::is_invalid_producer_epoch_error` if the producer has attempted to produce with an
187    ///   old epoch to the partition leader. See the error for more details
188    /// - `Error::is_unsupported_version_error` fatal error indicating the broker does not support
189    ///   transactions (i.e. if its version is lower than 0.11.0.0)
190    /// - `Error::is_authorization_error` fatal error indicating that the configured
191    ///   transactional.id is not authorized. See the error for more details
192    /// - `Error::is_kafka_error` if the producer has encountered a previous fatal error or for any
193    ///   other unexpected error
194    /// - `Error::is_timeout_error` if the time taken for aborting the transaction has surpassed
195    ///   `max.block.ms`.
196    /// - `Error::is_interrupt_error` if the task is interrupted while blocked
197    ///
198    /// # Translation notes
199    ///
200    /// A Rust task is never interrupted: dropping the future cancels the wait.
201    #[doc(alias = "org.apache.kafka.clients.producer.Producer#abortTransaction")]
202    fn abort_transaction(&self) -> impl Future<Output = Result<(), Error>> + Send;
203
204    /// Asynchronously send a record to a topic. Equivalent to
205    /// `send_with_callback(record, None)`. See
206    /// [`send_with_callback`](Producer::send_with_callback) for details.
207    ///
208    /// # Arguments
209    ///
210    /// - `record` - The record to send. If the topic or the partition specified in it cannot be
211    ///   found in metadata within `max.block.ms`, the returned future will time out when retrieved.
212    ///
213    /// # Returns
214    ///
215    /// A future which will eventually contain the response information.
216    ///
217    /// # Errors
218    ///
219    /// - `Error::is_local_illegal_state_error` if a transactional.id has been configured and no
220    ///   transaction has been started, or when send is invoked after producer has been closed.
221    /// - `Error::is_interrupt_error` If the task is interrupted while blocked
222    /// - `Error::is_serialization_error` If the key or value are not valid objects given the
223    ///   configured serializers
224    /// - `Error::is_bootstrap_resolution_error` if DNS resolution of the bootstrap servers fails
225    ///   within `bootstrap.resolve.timeout.ms`
226    /// - `Error::is_kafka_error` If a Kafka related error occurs that does not belong to the public
227    ///   API errors.
228    ///
229    /// # Translation notes
230    ///
231    /// A Rust task is never interrupted: dropping the future cancels the wait.
232    #[doc(alias = "org.apache.kafka.clients.producer.Producer#send")]
233    fn send(
234        &self,
235        record: ProducerRecord<K, V>,
236    ) -> impl Future<Output = Result<KafkaFuture<RecordMetadata>, Error>> + Send;
237
238    /// Asynchronously send a record to a topic and invoke the provided callback
239    /// when the send has been acknowledged.
240    ///
241    /// The send is asynchronous and this method will return immediately (except
242    /// for rare cases described below) once the record has been stored in the
243    /// buffer of records waiting to be sent. This allows sending many records in
244    /// parallel without blocking to wait for the response after each one. Can
245    /// block for the following cases: 1) For the first record being sent to the
246    /// cluster by this client for the given topic. In this case it will block for
247    /// up to `max.block.ms` milliseconds while waiting for topic's metadata if
248    /// Kafka cluster is unreachable; 2) Allocating a buffer if buffer pool
249    /// doesn't have any free buffers.
250    ///
251    /// **Reducing first-send latency:** You can reduce the latency of the first
252    /// send by preloading the metadata with
253    /// [`partitions_for`](Self::partitions_for). However, be aware that metadata
254    /// cache will be cleared to free up resources after `metadata.max.idle.ms`
255    /// of inactivity, so subsequent sends after a long idle period will still
256    /// experience delays.
257    ///
258    /// The result of the send is a [`RecordMetadata`] specifying the partition
259    /// the record was sent to, the offset it was assigned and the timestamp of
260    /// the record. If the producer is configured with acks = 0, the
261    /// `RecordMetadata` will have offset = -1 because the producer does not wait
262    /// for the acknowledgement from the broker. If CreateTime is used by the
263    /// topic, the timestamp will be the user provided timestamp or the record
264    /// send time if the user did not specify a timestamp for the record. If
265    /// LogAppendTime is used for the topic, the timestamp will be the Kafka
266    /// broker local time when the message is appended.
267    ///
268    /// Since the send call is asynchronous it returns a future for the
269    /// `RecordMetadata` that will be assigned to this record. Awaiting this
270    /// future waits until the associated request completes and then returns
271    /// the metadata for the record or the error that occurred while sending the
272    /// record. Fully non-blocking usage can make use of the `callback` parameter
273    /// to provide a callback that will be invoked when the request is complete.
274    ///
275    /// Callbacks for records being sent to the same partition are guaranteed to
276    /// execute in order.
277    ///
278    /// When used as part of a transaction, it is not necessary to define a
279    /// callback or check the result of the future in order to detect errors
280    /// from `send`. If any of the send calls failed with an irrecoverable error,
281    /// the final [`commit_transaction`](Self::commit_transaction) call will fail
282    /// and return the error from the last failed send. When this happens, your
283    /// application should call [`abort_transaction`](Self::abort_transaction)
284    /// to reset the state and continue to send data.
285    ///
286    /// Some transactional send errors cannot be resolved with a call to
287    /// `abort_transaction`. In particular, if a transactional send finishes with
288    /// a `ProducerFencedError`, a `OutOfOrderSequenceError`, a
289    /// `UnsupportedVersionError`, or an `AuthorizationError`, then the only
290    /// option left is to call [`close`](Self::close). Fatal errors cause the
291    /// producer to enter a defunct state in which future API calls will continue
292    /// to raise the same underlying error wrapped in a new `KafkaError`.
293    ///
294    /// It is a similar picture when idempotence is enabled, but no
295    /// `transactional.id` has been configured. In this case,
296    /// `UnsupportedVersionError` and `AuthorizationError` are considered fatal
297    /// errors. However, `ProducerFencedError` does not need to be handled.
298    /// Additionally, it is possible to continue sending after receiving an
299    /// `OutOfOrderSequenceError`, but doing so can result in out of order
300    /// delivery of pending messages. To ensure proper ordering, you should close
301    /// the producer and create a new instance.
302    ///
303    /// If the message format of the destination topic is not upgraded to
304    /// 0.11.0.0, idempotent and transactional produce requests will fail with an
305    /// `UnsupportedForMessageFormatError` error. If this is encountered during a
306    /// transaction, it is possible to abort and continue. But note that future
307    /// sends to the same topic will continue receiving the same error until the
308    /// topic is upgraded.
309    ///
310    /// Note that callbacks will generally execute in the I/O task of the
311    /// producer and so should be reasonably fast or they will delay the sending
312    /// of messages from other tasks. If you want to execute blocking or
313    /// computationally expensive callbacks it is recommended to hand the work
314    /// to another task from the callback body.
315    ///
316    /// # Arguments
317    ///
318    /// - `record` - The record to send. If the topic or the partition specified in it cannot be
319    ///   found in metadata within `max.block.ms`, the returned future will time out when retrieved.
320    /// - `callback` - A user-supplied callback to execute when the record has been acknowledged by
321    ///   the server (`None` indicates no callback)
322    ///
323    /// # Returns
324    ///
325    /// A future which will eventually contain the response information.
326    ///
327    /// # Errors
328    ///
329    /// - `Error::is_local_illegal_state_error` if a transactional.id has been configured and no
330    ///   transaction has been started, or when send is invoked after producer has been closed.
331    /// - `Error::is_interrupt_error` If the task is interrupted while blocked
332    /// - `Error::is_serialization_error` If the key or value are not valid objects given the
333    ///   configured serializers
334    /// - `Error::is_bootstrap_resolution_error` if DNS resolution of the bootstrap servers fails
335    ///   within `bootstrap.resolve.timeout.ms`
336    /// - `Error::is_kafka_error` If a Kafka related error occurs that does not belong to the public
337    ///   API errors.
338    ///
339    /// # Translation notes
340    ///
341    /// A Rust task is never interrupted: dropping the future cancels the wait.
342    #[doc(alias = "org.apache.kafka.clients.producer.Producer#send")]
343    fn send_with_callback(
344        &self,
345        record: ProducerRecord<K, V>,
346        callback: Option<Box<dyn Callback>>,
347    ) -> impl Future<Output = Result<KafkaFuture<RecordMetadata>, Error>> + Send;
348
349    /// Invoking this method makes all buffered records immediately available to
350    /// send (even if `linger.ms` is greater than 0) and blocks on the completion
351    /// of the requests associated with these records. The post-condition of
352    /// `flush()` is that any previously sent record will have completed (e.g.
353    /// its future is done and callbacks passed to `Self::send_with_callback`
354    /// have been called). A request is considered completed when it is
355    /// successfully acknowledged according to the `acks` configuration you have
356    /// specified or else it results in an error.
357    ///
358    /// Other tasks can continue sending records while one task is blocked
359    /// waiting for a flush call to complete, however no guarantee is made about
360    /// the completion of records sent after the flush call begins.
361    ///
362    /// This method can be useful when consuming from some input system and
363    /// producing into Kafka. The `flush()` call gives a convenient way to ensure
364    /// all previously sent messages have actually completed. Note that records
365    /// may be dropped if the produce request fails; to ensure that this does not
366    /// occur set `retries=<large_number>` in the config.
367    ///
368    /// Applications don't need to call this method for transactional producers,
369    /// since the [`commit_transaction`](Self::commit_transaction) will flush all
370    /// buffered records before performing the commit. This ensures that all the
371    /// `send` calls made since the previous
372    /// [`begin_transaction`](Self::begin_transaction) are completed before the
373    /// commit.
374    ///
375    /// **Important:** This method must not be called from within the callback
376    /// provided to `Self::send_with_callback`. Invoking `flush()` in this
377    /// context will result in a `KafkaError`, as it will cause a deadlock.
378    ///
379    /// # Errors
380    ///
381    /// - `Error::is_interrupt_error` If the task is interrupted while blocked
382    /// - `Error::is_kafka_error` If the method is invoked inside a `Self::send` callback
383    ///
384    /// # Translation notes
385    ///
386    /// A Rust task is never interrupted: dropping the future cancels the wait.
387    /// [`Callback::on_completion`] is synchronous, so it cannot await this
388    /// method.
389    #[doc(alias = "org.apache.kafka.clients.producer.Producer#flush")]
390    fn flush(&self) -> impl Future<Output = Result<(), Error>> + Send;
391
392    /// Get the partition metadata for the given topic. This can be used for
393    /// custom partitioning.
394    ///
395    /// This will attempt to refresh metadata until it finds the topic in it, or
396    /// the configured `max.block.ms` expires.
397    ///
398    /// # Arguments
399    ///
400    /// - `topic` - The topic to get partition metadata for
401    ///
402    /// # Returns
403    ///
404    /// The list of partitions.
405    ///
406    /// # Errors
407    ///
408    /// - `Error::is_authentication_error` if authentication fails.
409    /// - `Error::is_authorization_error` if not authorized to the specified topic.
410    /// - `Error::is_interrupt_error` if the task is interrupted while blocked
411    /// - `Error::is_timeout_error` if the topic cannot be found in metadata within `max.block.ms`
412    /// - `Error::is_bootstrap_resolution_error` if DNS resolution of the bootstrap servers fails
413    ///   within `bootstrap.resolve.timeout.ms`
414    /// - `Error::is_kafka_error` for all Kafka-related errors, including the case where this
415    ///   method is called after producer close
416    ///
417    /// # Translation notes
418    ///
419    /// A Rust task is never interrupted: dropping the future cancels the wait.
420    #[doc(alias = "org.apache.kafka.clients.producer.Producer#partitionsFor")]
421    fn partitions_for(&self, topic: &str) -> impl Future<Output = Result<Vec<PartitionInfo>, Error>> + Send;
422
423    /// Get the full set of internal metrics maintained by the producer.
424    ///
425    /// The returned map is an unmodifiable live view of the metrics. Changes to
426    /// the underlying metrics will be reflected in the returned map.
427    ///
428    /// # Returns
429    ///
430    /// An unmodifiable live view of the map of metrics currently maintained by
431    /// the producer
432    #[doc(alias = "org.apache.kafka.clients.producer.Producer#metrics")]
433    fn metrics(&self) -> MetricsView;
434
435    /// Close this producer. This method blocks until all previously sent
436    /// requests complete. This method is equivalent to
437    /// `close_with_timeout(Duration::MAX)`.
438    ///
439    /// **If close() is called from a [`Callback`], a warning message will be
440    /// logged and `close_with_timeout(Duration::ZERO)` will be called instead.
441    /// We do this because the sender task would otherwise try to join itself and
442    /// block forever.**
443    ///
444    /// # Errors
445    ///
446    /// - `Error::is_interrupt_error` If the task is interrupted while blocked.
447    /// - `Error::is_kafka_error` If an unexpected error occurs while trying to close the client,
448    ///   this error should be treated as fatal and indicate the client is no longer usable.
449    ///
450    /// # Translation notes
451    ///
452    /// A Rust task is never interrupted: dropping the future cancels the wait.
453    /// [`Callback::on_completion`] is synchronous, so it cannot await this
454    /// method.
455    #[doc(alias = "org.apache.kafka.clients.producer.Producer#close")]
456    fn close(&self) -> impl Future<Output = Result<(), Error>> + Send;
457
458    /// This method waits up to `timeout` for the producer to complete the
459    /// sending of all incomplete requests.
460    ///
461    /// If the producer is unable to complete all requests before the timeout
462    /// expires, this method will fail any unsent and unacknowledged records
463    /// immediately. It will also abort the ongoing transaction if it's not
464    /// already completing.
465    ///
466    /// If invoked from within a [`Callback`] this method will not block and will
467    /// be equivalent to `close_with_timeout(Duration::ZERO)`. This is done since
468    /// no further sending will happen while blocking the I/O task of the
469    /// producer.
470    ///
471    /// # Arguments
472    ///
473    /// - `timeout` - The maximum time to wait for producer to complete any pending requests. The
474    ///   value should be non-negative. Specifying a timeout of zero means do not wait for pending
475    ///   send requests to complete.
476    ///
477    /// # Errors
478    ///
479    /// - `Error::is_interrupt_error` If the task is interrupted while blocked.
480    /// - `Error::is_kafka_error` If an unexpected error occurs while trying to close the client,
481    ///   this error should be treated as fatal and indicate the client is no longer usable.
482    /// - `Error::is_local_illegal_argument_error` If the `timeout` is negative.
483    ///
484    /// # Translation notes
485    ///
486    /// A Rust task is never interrupted: dropping the future cancels the wait.
487    /// A `Duration` is never negative. [`Callback::on_completion`] is
488    /// synchronous, so it cannot await this method.
489    #[doc(alias = "org.apache.kafka.clients.producer.Producer#close")]
490    fn close_with_timeout(&self, timeout: Duration) -> impl Future<Output = Result<(), Error>> + Send;
491}