Skip to main content

confluent_kafka/producer/
dyn_producer.rs

1// Copyright 2026 Confluent Inc.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! The dyn-compatible view of [`Producer`].
16//!
17//! Has no Java counterpart (`definition-of-done.md` §7). A Java interface is
18//! always usable as a runtime-polymorphic type, so a `List<Producer<K, V>>`
19//! can hold a `KafkaProducer` next to a `MockProducer`. The Rust
20//! [`Producer`] trait returns `impl Future` from its asynchronous methods,
21//! which keeps statically dispatched calls (one per record on the send path)
22//! free of a boxed future, but makes the trait itself not dyn-compatible.
23//! `DynProducer` restores the Java capability (CLAUDE.md §3, "traits must be
24//! dyn-compatible") by boxing each future, and only callers that go through
25//! `dyn` pay for it.
26//!
27//! A method added to [`Producer`] goes in three places here: the trait, the
28//! blanket impl and the impl on `dyn DynProducer`. `cargo xtask lint-custom`
29//! (`check-dyn-forwarding`) fails until all three forward it.
30
31use std::collections::HashMap;
32use std::pin::Pin;
33use std::time::Duration;
34
35use crate::common::Error;
36use crate::common::KafkaFuture;
37use crate::common::PartitionInfo;
38use crate::common::TopicPartition;
39use crate::common::metrics::MetricsView;
40use crate::consumer::ConsumerGroupMetadata;
41use crate::consumer::OffsetAndMetadata;
42use crate::producer::Callback;
43#[cfg(feature = "ffi")]
44use crate::producer::KafkaProducer;
45#[cfg(feature = "ffi")]
46use crate::producer::MockProducer;
47use crate::producer::Producer;
48use crate::producer::ProducerRecord;
49use crate::producer::RecordMetadata;
50#[cfg(feature = "ffi")]
51use crate::producer::TrySendOutcome;
52
53type BoxFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
54
55mod private {
56    // Deliberately unnameable: it seals `DynProducer` to `Producer` implementors.
57    #[expect(unnameable_types)]
58    pub trait Sealed<K, V> {}
59}
60
61impl<K, V, P: super::Producer<K, V>> private::Sealed<K, V> for P {}
62
63/// A dyn-compatible [`Producer`], for holding different producer
64/// implementations behind one type, e.g. `Vec<Box<dyn DynProducer<K, V>>>`.
65///
66/// Every [`Producer`] implements `DynProducer` automatically, and the trait is
67/// sealed, so it cannot be implemented directly: implement [`Producer`]
68/// instead. Each method forwards to the [`Producer`] method of the same name;
69/// asynchronous methods return the future boxed, which costs one allocation per
70/// call. `dyn DynProducer<K, V>` implements [`Producer`] in turn, so a boxed
71/// producer can be passed as `&*boxed` to generic code taking
72/// `P: Producer<K, V> + ?Sized`.
73///
74/// With both traits in scope a method call on a `dyn DynProducer` is ambiguous,
75/// since both define it; call it as `DynProducer::send(&*producer, record)`, or
76/// import only one of the two traits.
77///
78/// ```
79/// use confluent_kafka::producer::{DynProducer, MockProducer};
80///
81/// let producers: Vec<Box<dyn DynProducer<String, String>>> = vec![
82///     Box::new(MockProducer::<String, String>::with_auto_complete(true)),
83///     Box::new(MockProducer::<String, String>::with_auto_complete(false)),
84/// ];
85/// assert_eq!(2, producers.len());
86/// ```
87// the dyn-compatible companion of the `Producer` trait (CLAUDE.md §3)
88#[doc(alias = "rust-only")]
89pub trait DynProducer<K, V>: private::Sealed<K, V> + Send + Sync {
90    /// See [`Producer::init_transactions`].
91    ///
92    /// # Errors
93    ///
94    /// As [`Producer::init_transactions`].
95    fn init_transactions<'a>(&'a self) -> BoxFuture<'a, Result<(), Error>>
96    where
97        K: 'a,
98        V: 'a;
99
100    /// See [`Producer::begin_transaction`].
101    ///
102    /// # Errors
103    ///
104    /// As [`Producer::begin_transaction`].
105    fn begin_transaction(&self) -> Result<(), Error>;
106
107    /// See [`Producer::send_offsets_to_transaction`].
108    ///
109    /// # Arguments
110    ///
111    /// - `offsets` - as [`Producer::send_offsets_to_transaction`].
112    /// - `group_metadata` - as [`Producer::send_offsets_to_transaction`].
113    ///
114    /// # Errors
115    ///
116    /// As [`Producer::send_offsets_to_transaction`].
117    fn send_offsets_to_transaction<'a>(
118        &'a self,
119        offsets: HashMap<TopicPartition, OffsetAndMetadata>,
120        group_metadata: &'a dyn ConsumerGroupMetadata,
121    ) -> BoxFuture<'a, Result<(), Error>>
122    where
123        K: 'a,
124        V: 'a;
125
126    /// See [`Producer::commit_transaction`].
127    ///
128    /// # Errors
129    ///
130    /// As [`Producer::commit_transaction`].
131    fn commit_transaction<'a>(&'a self) -> BoxFuture<'a, Result<(), Error>>
132    where
133        K: 'a,
134        V: 'a;
135
136    /// See [`Producer::abort_transaction`].
137    ///
138    /// # Errors
139    ///
140    /// As [`Producer::abort_transaction`].
141    fn abort_transaction<'a>(&'a self) -> BoxFuture<'a, Result<(), Error>>
142    where
143        K: 'a,
144        V: 'a;
145
146    /// See [`Producer::send`].
147    ///
148    /// # Arguments
149    ///
150    /// - `record` - as [`Producer::send`].
151    ///
152    /// # Returns
153    ///
154    /// As [`Producer::send`].
155    ///
156    /// # Errors
157    ///
158    /// As [`Producer::send`].
159    fn send<'a>(&'a self, record: ProducerRecord<K, V>) -> BoxFuture<'a, Result<KafkaFuture<RecordMetadata>, Error>>
160    where
161        K: 'a,
162        V: 'a;
163
164    /// See [`Producer::send_with_callback`].
165    ///
166    /// # Arguments
167    ///
168    /// - `record` - as [`Producer::send_with_callback`].
169    /// - `callback` - as [`Producer::send_with_callback`].
170    ///
171    /// # Returns
172    ///
173    /// As [`Producer::send_with_callback`].
174    ///
175    /// # Errors
176    ///
177    /// As [`Producer::send_with_callback`].
178    fn send_with_callback<'a>(
179        &'a self,
180        record: ProducerRecord<K, V>,
181        callback: Option<Box<dyn Callback>>,
182    ) -> BoxFuture<'a, Result<KafkaFuture<RecordMetadata>, Error>>
183    where
184        K: 'a,
185        V: 'a;
186
187    /// See [`Producer::flush`].
188    ///
189    /// # Errors
190    ///
191    /// As [`Producer::flush`].
192    fn flush<'a>(&'a self) -> BoxFuture<'a, Result<(), Error>>
193    where
194        K: 'a,
195        V: 'a;
196
197    /// See [`Producer::partitions_for`].
198    ///
199    /// # Arguments
200    ///
201    /// - `topic` - as [`Producer::partitions_for`].
202    ///
203    /// # Returns
204    ///
205    /// As [`Producer::partitions_for`].
206    ///
207    /// # Errors
208    ///
209    /// As [`Producer::partitions_for`].
210    fn partitions_for<'a>(&'a self, topic: &'a str) -> BoxFuture<'a, Result<Vec<PartitionInfo>, Error>>
211    where
212        K: 'a,
213        V: 'a;
214
215    /// See [`Producer::metrics`].
216    ///
217    /// # Returns
218    ///
219    /// As [`Producer::metrics`].
220    fn metrics(&self) -> MetricsView;
221
222    /// See [`Producer::close`].
223    ///
224    /// # Errors
225    ///
226    /// As [`Producer::close`].
227    fn close<'a>(&'a self) -> BoxFuture<'a, Result<(), Error>>
228    where
229        K: 'a,
230        V: 'a;
231
232    /// See [`Producer::close_with_timeout`].
233    ///
234    /// # Arguments
235    ///
236    /// - `timeout` - as [`Producer::close_with_timeout`].
237    ///
238    /// # Errors
239    ///
240    /// As [`Producer::close_with_timeout`].
241    fn close_with_timeout<'a>(&'a self, timeout: Duration) -> BoxFuture<'a, Result<(), Error>>
242    where
243        K: 'a,
244        V: 'a;
245}
246
247impl<K, V, P: Producer<K, V>> DynProducer<K, V> for P {
248    fn init_transactions<'a>(&'a self) -> BoxFuture<'a, Result<(), Error>>
249    where
250        K: 'a,
251        V: 'a,
252    {
253        Box::pin(Producer::init_transactions(self))
254    }
255
256    fn begin_transaction(&self) -> Result<(), Error> {
257        Producer::begin_transaction(self)
258    }
259
260    fn send_offsets_to_transaction<'a>(
261        &'a self,
262        offsets: HashMap<TopicPartition, OffsetAndMetadata>,
263        group_metadata: &'a dyn ConsumerGroupMetadata,
264    ) -> BoxFuture<'a, Result<(), Error>>
265    where
266        K: 'a,
267        V: 'a,
268    {
269        Box::pin(Producer::send_offsets_to_transaction(self, offsets, group_metadata))
270    }
271
272    fn commit_transaction<'a>(&'a self) -> BoxFuture<'a, Result<(), Error>>
273    where
274        K: 'a,
275        V: 'a,
276    {
277        Box::pin(Producer::commit_transaction(self))
278    }
279
280    fn abort_transaction<'a>(&'a self) -> BoxFuture<'a, Result<(), Error>>
281    where
282        K: 'a,
283        V: 'a,
284    {
285        Box::pin(Producer::abort_transaction(self))
286    }
287
288    fn send<'a>(&'a self, record: ProducerRecord<K, V>) -> BoxFuture<'a, Result<KafkaFuture<RecordMetadata>, Error>>
289    where
290        K: 'a,
291        V: 'a,
292    {
293        Box::pin(Producer::send(self, record))
294    }
295
296    fn send_with_callback<'a>(
297        &'a self,
298        record: ProducerRecord<K, V>,
299        callback: Option<Box<dyn Callback>>,
300    ) -> BoxFuture<'a, Result<KafkaFuture<RecordMetadata>, Error>>
301    where
302        K: 'a,
303        V: 'a,
304    {
305        Box::pin(Producer::send_with_callback(self, record, callback))
306    }
307
308    fn flush<'a>(&'a self) -> BoxFuture<'a, Result<(), Error>>
309    where
310        K: 'a,
311        V: 'a,
312    {
313        Box::pin(Producer::flush(self))
314    }
315
316    fn partitions_for<'a>(&'a self, topic: &'a str) -> BoxFuture<'a, Result<Vec<PartitionInfo>, Error>>
317    where
318        K: 'a,
319        V: 'a,
320    {
321        Box::pin(Producer::partitions_for(self, topic))
322    }
323
324    fn metrics(&self) -> MetricsView {
325        Producer::metrics(self)
326    }
327
328    fn close<'a>(&'a self) -> BoxFuture<'a, Result<(), Error>>
329    where
330        K: 'a,
331        V: 'a,
332    {
333        Box::pin(Producer::close(self))
334    }
335
336    fn close_with_timeout<'a>(&'a self, timeout: Duration) -> BoxFuture<'a, Result<(), Error>>
337    where
338        K: 'a,
339        V: 'a,
340    {
341        Box::pin(Producer::close_with_timeout(self, timeout))
342    }
343}
344
345// On the trait object rather than on `Box<dyn DynProducer>`: a boxed producer
346// that implemented `Producer` would also pick up the blanket `DynProducer` impl,
347// and a method call on the box would then box every future twice.
348impl<K, V> Producer<K, V> for dyn DynProducer<K, V> + '_ {
349    fn init_transactions(&self) -> impl Future<Output = Result<(), Error>> + Send {
350        DynProducer::init_transactions(self)
351    }
352
353    fn begin_transaction(&self) -> Result<(), Error> {
354        DynProducer::begin_transaction(self)
355    }
356
357    // An `async fn` rather than returning the boxed future: the box is bounded by a
358    // single lifetime, the intersection of `self`'s and `group_metadata`'s, which
359    // the trait's opaque return type cannot name.
360    async fn send_offsets_to_transaction(
361        &self,
362        offsets: HashMap<TopicPartition, OffsetAndMetadata>,
363        group_metadata: &dyn ConsumerGroupMetadata,
364    ) -> Result<(), Error> {
365        DynProducer::send_offsets_to_transaction(self, offsets, group_metadata).await
366    }
367
368    fn commit_transaction(&self) -> impl Future<Output = Result<(), Error>> + Send {
369        DynProducer::commit_transaction(self)
370    }
371
372    fn abort_transaction(&self) -> impl Future<Output = Result<(), Error>> + Send {
373        DynProducer::abort_transaction(self)
374    }
375
376    fn send(
377        &self,
378        record: ProducerRecord<K, V>,
379    ) -> impl Future<Output = Result<KafkaFuture<RecordMetadata>, Error>> + Send {
380        DynProducer::send(self, record)
381    }
382
383    fn send_with_callback(
384        &self,
385        record: ProducerRecord<K, V>,
386        callback: Option<Box<dyn Callback>>,
387    ) -> impl Future<Output = Result<KafkaFuture<RecordMetadata>, Error>> + Send {
388        DynProducer::send_with_callback(self, record, callback)
389    }
390
391    fn flush(&self) -> impl Future<Output = Result<(), Error>> + Send {
392        DynProducer::flush(self)
393    }
394
395    // `async fn` rather than returning the boxed future directly: that future
396    // lives for the shorter of the two input lifetimes, which an opaque return
397    // type cannot name.
398    async fn partitions_for(&self, topic: &str) -> Result<Vec<PartitionInfo>, Error> {
399        DynProducer::partitions_for(self, topic).await
400    }
401
402    fn metrics(&self) -> MetricsView {
403        DynProducer::metrics(self)
404    }
405
406    fn close(&self) -> impl Future<Output = Result<(), Error>> + Send {
407        DynProducer::close(self)
408    }
409
410    fn close_with_timeout(&self, timeout: Duration) -> impl Future<Output = Result<(), Error>> + Send {
411        DynProducer::close_with_timeout(self, timeout)
412    }
413}
414
415/// The non-blocking send of either producer behind one `dyn` handle: a
416/// [`DynProducer`] with [`KafkaProducer::try_send`] beside its methods.
417///
418/// There is no Java counterpart (DoD #7). [`KafkaProducer::try_send`] has none
419/// either — Java's `send` always blocks for up to `max.block.ms` — and it is an
420/// inherent method rather than a [`Producer`] one, because only `KafkaProducer`
421/// has pre-append wait points to skip. The C FFI, though, reaches the producer
422/// it wraps only through a `dyn` handle (see [`DynProducer`]), and its
423/// non-blocking send entry point needs the fast path of the real producer and
424/// the `send` of the mock through that same handle. This trait is that
425/// handle's type: [`try_send`](Self::try_send) is the real producer's
426/// `try_send`, and the mock's `send_with_callback`, which has no metadata or
427/// buffer wait to skip, reported as [`TrySendOutcome::Accepted`]. It cannot be a method of `DynProducer`: that
428/// trait's blanket impl covers every [`Producer`], and a `Producer` has no
429/// `try_send` to forward to. Crate-internal, and only the `ffi` feature names
430/// it.
431#[cfg(feature = "ffi")]
432pub(crate) trait DynTrySend<K, V>: DynProducer<K, V> {
433    /// See [`KafkaProducer::try_send`].
434    fn try_send<'a>(
435        &'a self,
436        record: ProducerRecord<K, V>,
437        callback: Option<Box<dyn Callback>>,
438    ) -> BoxFuture<'a, Result<TrySendOutcome, Error>>
439    where
440        K: 'a,
441        V: 'a;
442
443    /// Whether this producer has a sender task that no close has stopped: a
444    /// `KafkaProducer` not closed yet. The C binding's `destroy` closes such a
445    /// producer before dropping it. The mock has no sender.
446    fn has_unclosed_sender(&self) -> bool;
447}
448
449#[cfg(feature = "ffi")]
450impl<K, V> DynTrySend<K, V> for KafkaProducer<K, V>
451where
452    K: Send + Sync,
453    V: Send + Sync,
454{
455    fn try_send<'a>(
456        &'a self,
457        record: ProducerRecord<K, V>,
458        callback: Option<Box<dyn Callback>>,
459    ) -> BoxFuture<'a, Result<TrySendOutcome, Error>>
460    where
461        K: 'a,
462        V: 'a,
463    {
464        Box::pin(KafkaProducer::try_send(self, record, callback))
465    }
466
467    fn has_unclosed_sender(&self) -> bool {
468        self.is_running()
469    }
470}
471
472/// The mock has no metadata or buffer wait — its `send_with_callback` awaits only
473/// the serializers, then records the send under its own lock — so its
474/// non-blocking send is its send, always `Accepted`.
475#[cfg(feature = "ffi")]
476impl<K, V> DynTrySend<K, V> for MockProducer<K, V>
477where
478    K: Send + Sync,
479    V: Send + Sync,
480{
481    fn try_send<'a>(
482        &'a self,
483        record: ProducerRecord<K, V>,
484        callback: Option<Box<dyn Callback>>,
485    ) -> BoxFuture<'a, Result<TrySendOutcome, Error>>
486    where
487        K: 'a,
488        V: 'a,
489    {
490        Box::pin(async move {
491            Producer::send_with_callback(self, record, callback)
492                .await
493                .map(TrySendOutcome::Accepted)
494        })
495    }
496
497    fn has_unclosed_sender(&self) -> bool {
498        false
499    }
500}
501
502#[cfg(test)]
503mod tests {
504    use super::*;
505    use crate::producer::MockProducer;
506    use std::sync::Arc;
507
508    fn record(value: &str) -> ProducerRecord<String, String> {
509        ProducerRecord::with_key("topic".to_string(), Some("key".to_string()), Some(value.to_string()))
510    }
511
512    /// The mock's non-blocking send is its send: it has no wait to skip, so
513    /// through the `dyn` handle every record is `Accepted` and lands in its
514    /// history.
515    #[cfg(feature = "ffi")]
516    #[tokio::test]
517    async fn test_mock_try_send_is_always_accepted() {
518        let mock = MockProducer::<String, String>::with_auto_complete(true);
519        let producer: &dyn DynTrySend<String, String> = &mock;
520
521        let outcome = producer.try_send(record("value"), None).await.expect("try_send");
522
523        let future = match outcome {
524            TrySendOutcome::Accepted(future) => future,
525            other => panic!("expected Accepted, got {other:?}"),
526        };
527        assert_eq!(future.get().await.expect("metadata").offset(), 0);
528        assert_eq!(mock.history().len(), 1);
529    }
530
531    async fn send_through_generic<P: Producer<String, String> + ?Sized>(producer: &P, value: &str) -> i64 {
532        let future = Producer::send(producer, record(value)).await.expect("send");
533        future.get().await.expect("metadata").offset()
534    }
535
536    #[tokio::test]
537    async fn test_different_producers_in_one_vec() {
538        let auto = MockProducer::<String, String>::with_auto_complete(true);
539        let manual = MockProducer::<String, String>::with_auto_complete(false);
540        let producers: Vec<&dyn DynProducer<String, String>> = vec![&auto, &manual];
541
542        for (index, producer) in producers.iter().enumerate() {
543            DynProducer::send(*producer, record(&format!("value{index}")))
544                .await
545                .expect("send");
546        }
547
548        assert_eq!(vec![record("value0")], auto.history());
549        assert_eq!(vec![record("value1")], manual.history());
550        assert!(manual.complete_next(), "the manual mock still holds the uncompleted send");
551    }
552
553    #[tokio::test]
554    async fn test_boxed_producers_in_one_vec() {
555        let producers: Vec<Box<dyn DynProducer<String, String>>> = vec![
556            Box::new(MockProducer::<String, String>::with_auto_complete(true)),
557            Box::new(MockProducer::<String, String>::with_auto_complete(true)),
558        ];
559
560        for producer in &producers {
561            let future = DynProducer::send(&**producer, record("value0")).await.expect("send");
562            assert!(future.is_done(), "an auto-complete mock completes the send immediately");
563            DynProducer::flush(&**producer).await.expect("flush");
564        }
565    }
566
567    #[tokio::test]
568    async fn test_dyn_producer_is_a_producer() {
569        let boxed: Box<dyn DynProducer<String, String>> =
570            Box::new(MockProducer::<String, String>::with_auto_complete(true));
571
572        assert_eq!(0, send_through_generic(&*boxed, "value0").await);
573        assert_eq!(1, send_through_generic(&*boxed, "value1").await);
574    }
575
576    #[test]
577    fn test_errors_pass_through_unchanged() {
578        let mock = MockProducer::<String, String>::with_auto_complete(true);
579        let as_dyn: &dyn DynProducer<String, String> = &mock;
580
581        let error = DynProducer::begin_transaction(as_dyn).expect_err("begin_transaction before init_transactions");
582        let expected = Producer::begin_transaction(&mock).expect_err("same call on the mock itself");
583        assert_eq!(expected.to_string(), error.to_string());
584    }
585
586    #[tokio::test]
587    async fn test_dyn_producer_futures_can_be_spawned() {
588        let shared: Arc<dyn DynProducer<String, String>> =
589            Arc::new(MockProducer::<String, String>::with_auto_complete(true));
590        let task = tokio::spawn({
591            let shared = shared.clone();
592            async move { DynProducer::send(&*shared, record("value0")).await.map(|_| ()) }
593        });
594        task.await.expect("join").expect("send");
595    }
596}