Skip to main content

confluent_kafka/consumer/
consumer_records.rs

1// Copyright 2025 Confluent Inc.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! A container that holds per-partition lists of [`ConsumerRecord`].
16//!
17//! Translated from `org.apache.kafka.clients.consumer.ConsumerRecords`.
18
19use indexmap::IndexMap;
20use std::collections::HashMap;
21
22use crate::common::TopicPartition;
23use crate::consumer::ConsumerRecord;
24use crate::consumer::OffsetAndMetadata;
25
26/// A container that holds the list [`ConsumerRecord`] per partition for a
27/// particular topic. There is one [`ConsumerRecord`] list for every topic
28/// partition returned by a [`Consumer::poll`](crate::consumer::Consumer::poll)
29/// operation.
30// Uses an `IndexMap` internally so iteration order matches insertion order:
31// Java iterates in the underlying map's order, and its tests assume insertion
32// order when a `LinkedHashMap` is supplied.
33#[derive(Debug)]
34#[doc(alias = "org.apache.kafka.clients.consumer.ConsumerRecords")]
35pub struct ConsumerRecords<K, V> {
36    records: IndexMap<TopicPartition, Vec<ConsumerRecord<K, V>>>,
37    next_offsets: HashMap<TopicPartition, OffsetAndMetadata>,
38    // Whether the consumed position advanced for at least one partition
39    // during the `collect_fetch` that produced this batch, even if no
40    // records were returned (e.g. all records in a batch were aborted under
41    // READ_COMMITTED).
42    //
43    // This field carries Java's internal `Fetch.positionAdvanced` flag —
44    // the Rust port collapses Java's internal `Fetch<K, V>` into
45    // `ConsumerRecords<K, V>` (no separate `Fetch` type). The public
46    // [`Self::is_empty`] still mirrors Java's *public*
47    // `ConsumerRecords.isEmpty()` (`records.isEmpty()`), but the poll loop
48    // needs Java's internal `Fetch.isEmpty()` (`numRecords == 0 &&
49    // !positionAdvanced`) so it returns promptly when only the position
50    // advanced — see [`Self::is_fetch_empty`].
51    position_advanced: bool,
52}
53
54impl<K, V> ConsumerRecords<K, V> {
55    /// Constructs a new `ConsumerRecords` with the given records and next offsets.
56    ///
57    /// # Arguments
58    ///
59    /// - `records` - The records for each partition
60    /// - `next_offsets` - The next offset and metadata for each partition whose position was
61    ///   advanced during the poll call. These represent the offsets that the consumer will start
62    ///   reading from on the next poll.
63    ///
64    /// # Returns
65    ///
66    /// The new `ConsumerRecords`.
67    // Java's deprecated `ConsumerRecords(Map)`, which supplies no next offsets,
68    // is not translated (CLAUDE.md §3).
69    #[doc(alias = "org.apache.kafka.clients.consumer.ConsumerRecords#ConsumerRecords(Map,Map)")]
70    pub fn with_next_offsets(
71        records: IndexMap<TopicPartition, Vec<ConsumerRecord<K, V>>>,
72        next_offsets: HashMap<TopicPartition, OffsetAndMetadata>,
73    ) -> Self {
74        Self { records, next_offsets, position_advanced: false }
75    }
76
77    /// Like [`Self::with_next_offsets`] but also records whether the consumed
78    /// position advanced (Java's internal `Fetch.positionAdvanced`). Used by
79    /// `FetchCollector::collect_fetch` so the poll loop can mirror Java's
80    /// `Fetch.isEmpty()` semantics.
81    #[doc(alias = "org.apache.kafka.clients.consumer.ConsumerRecords#ConsumerRecords")]
82    pub(crate) fn with_position_advanced(
83        records: IndexMap<TopicPartition, Vec<ConsumerRecord<K, V>>>,
84        next_offsets: HashMap<TopicPartition, OffsetAndMetadata>,
85        position_advanced: bool,
86    ) -> Self {
87        Self { records, next_offsets, position_advanced }
88    }
89
90    /// Returns an empty `ConsumerRecords` instance.
91    ///
92    /// # Returns
93    ///
94    /// An empty `ConsumerRecords`.
95    // Java's static `ConsumerRecords.empty()` / `ConsumerRecords.EMPTY`.
96    #[doc(alias = "org.apache.kafka.clients.consumer.ConsumerRecords#empty")]
97    pub fn empty() -> Self {
98        Self { records: IndexMap::new(), next_offsets: HashMap::new(), position_advanced: false }
99    }
100
101    /// Java's internal `Fetch.isEmpty()`: `numRecords == 0 &&
102    /// !positionAdvanced`. The poll loop uses this (NOT [`Self::is_empty`])
103    /// to decide whether to return early — so an all-aborted batch that
104    /// advances the position with zero records returns promptly instead of
105    /// blocking until the poll timeout, matching `AsyncKafkaConsumer.poll`.
106    pub(crate) fn is_fetch_empty(&self) -> bool {
107        Self::fetch_is_empty(&self.records, self.position_advanced)
108    }
109
110    /// Java's internal `Fetch.isEmpty()` (`org.apache.kafka.clients.consumer.internals.Fetch#isEmpty`) evaluated over
111    /// the two accumulators that build a `Fetch`, for callers that do not have
112    /// a `ConsumerRecords` yet.
113    ///
114    /// `FetchCollector::collect_fetch` mirrors Java's
115    /// `final Fetch<K, V> fetch = Fetch.empty()` with a plain
116    /// records-map + `position_advanced` pair (it only materialises the
117    /// `ConsumerRecords` on the way out), yet Java tests `fetch.isEmpty()`
118    /// three times *during* the loop to decide whether to swallow an error and
119    /// whether to leave the offending entry queued. This associated function
120    /// exists so both callers answer the question from one place: the
121    /// records-only spelling was wrong at all three collector sites, and a
122    /// second inline copy of the predicate is exactly how that drifted.
123    ///
124    /// Note `records.is_empty()` is the faithful reading of Java's
125    /// `numRecords == 0`: the collector inserts a partition entry only when it
126    /// decoded at least one record for it, so an empty map and a zero record
127    /// count are the same condition.
128    pub(crate) fn fetch_is_empty(
129        records: &IndexMap<TopicPartition, Vec<ConsumerRecord<K, V>>>,
130        position_advanced: bool,
131    ) -> bool {
132        records.is_empty() && !position_advanced
133    }
134
135    /// Get just the records for the given partition.
136    ///
137    /// # Arguments
138    ///
139    /// - `partition` - The partition to get records for
140    ///
141    /// # Returns
142    ///
143    /// The records, an empty slice if there are none.
144    #[doc(alias = "org.apache.kafka.clients.consumer.ConsumerRecords#records")]
145    pub fn records_with_partition(&self, partition: &TopicPartition) -> &[ConsumerRecord<K, V>] {
146        self.records.get(partition).map(Vec::as_slice).unwrap_or(&[])
147    }
148
149    /// Get just the records for the given topic.
150    ///
151    /// # Arguments
152    ///
153    /// - `topic` - The topic to get records for
154    ///
155    /// # Returns
156    ///
157    /// An iterator over the records, in partition-insertion order.
158    #[doc(alias = "org.apache.kafka.clients.consumer.ConsumerRecords#records")]
159    pub fn records_with_topic<'a>(&'a self, topic: &'a str) -> impl Iterator<Item = &'a ConsumerRecord<K, V>> + 'a {
160        self.records
161            .iter()
162            .filter(move |(tp, _)| tp.topic() == topic)
163            .flat_map(|(_, records)| records.iter())
164    }
165
166    /// Get the partitions which have records contained in this record set.
167    ///
168    /// # Returns
169    ///
170    /// The set of partitions with data in this record set (may be empty if no data was returned),
171    /// as an iterator in insertion order.
172    #[doc(alias = "org.apache.kafka.clients.consumer.ConsumerRecords#partitions")]
173    pub fn partitions(&self) -> impl Iterator<Item = &TopicPartition> {
174        self.records.keys()
175    }
176
177    /// Returns the number of records for all topics.
178    #[doc(alias = "org.apache.kafka.clients.consumer.ConsumerRecords#count")]
179    pub fn count(&self) -> usize {
180        self.records.values().map(Vec::len).sum()
181    }
182
183    /// Returns whether this container has any records.
184    ///
185    /// # Returns
186    ///
187    /// True if there are no records, false otherwise
188    // Java returns `records.isEmpty()`, not `count() == 0`.
189    #[doc(alias = "org.apache.kafka.clients.consumer.ConsumerRecords#isEmpty")]
190    pub fn is_empty(&self) -> bool {
191        self.records.is_empty()
192    }
193
194    /// Get the next offsets and metadata corresponding to all topic partitions
195    /// for which the position have been advanced in this poll call.
196    ///
197    /// # Returns
198    ///
199    /// The next offsets that the consumer will consume
200    #[doc(alias = "org.apache.kafka.clients.consumer.ConsumerRecords#nextOffsets")]
201    pub fn next_offsets(&self) -> &HashMap<TopicPartition, OffsetAndMetadata> {
202        &self.next_offsets
203    }
204}
205
206impl<K, V> Default for ConsumerRecords<K, V> {
207    fn default() -> Self {
208        Self::empty()
209    }
210}
211
212/// Borrowed iteration over every record across all partitions (in
213/// partition-insertion order).
214///
215/// Equivalent to Java's `ConsumerRecords.iterator()` (which iterates the
216/// concatenated values of the underlying map).
217impl<'a, K, V> IntoIterator for &'a ConsumerRecords<K, V> {
218    type Item = &'a ConsumerRecord<K, V>;
219    type IntoIter = std::iter::FlatMap<
220        indexmap::map::Values<'a, TopicPartition, Vec<ConsumerRecord<K, V>>>,
221        std::slice::Iter<'a, ConsumerRecord<K, V>>,
222        fn(&'a Vec<ConsumerRecord<K, V>>) -> std::slice::Iter<'a, ConsumerRecord<K, V>>,
223    >;
224
225    fn into_iter(self) -> Self::IntoIter {
226        // The function pointer type must reference `&Vec<T>` exactly to match
227        // the `IntoIter` associated type alias above; using `&[T]` here would
228        // produce a slice iterator whose lifetime cannot be unified with the
229        // map values iterator.
230        #[expect(clippy::ptr_arg)]
231        fn iter_vec<T>(v: &Vec<T>) -> std::slice::Iter<'_, T> {
232            v.iter()
233        }
234        self.records.values().flat_map(iter_vec)
235    }
236}
237
238/// Owned iteration that consumes the [`ConsumerRecords`].
239impl<K, V> IntoIterator for ConsumerRecords<K, V> {
240    type Item = ConsumerRecord<K, V>;
241    type IntoIter = std::iter::FlatMap<
242        indexmap::map::IntoValues<TopicPartition, Vec<ConsumerRecord<K, V>>>,
243        std::vec::IntoIter<ConsumerRecord<K, V>>,
244        fn(Vec<ConsumerRecord<K, V>>) -> std::vec::IntoIter<ConsumerRecord<K, V>>,
245    >;
246
247    fn into_iter(self) -> Self::IntoIter {
248        fn iter_vec<T>(v: Vec<T>) -> std::vec::IntoIter<T> {
249            v.into_iter()
250        }
251        self.records.into_values().flat_map(iter_vec)
252    }
253}
254
255#[cfg(test)]
256mod tests {
257    use super::*;
258    use crate::common::record::TimestampType;
259    use crate::consumer::ConsumerRecordOptionsBuilder;
260
261    fn one_record() -> IndexMap<TopicPartition, Vec<ConsumerRecord<i32, String>>> {
262        let tp = TopicPartition::new("topic".to_string(), 0);
263        let record = ConsumerRecord::with_options(
264            ConsumerRecordOptionsBuilder::new()
265                .set_topic("topic")
266                .set_partition(0)
267                .set_offset(0)
268                .set_key(Some(0))
269                .set_value(Some("value".to_string()))
270                .set_timestamp(0)
271                .set_timestamp_type(TimestampType::CreateTime)
272                .set_serialized_key_size(0)
273                .set_serialized_value_size(0)
274                .build()
275                .unwrap(),
276        );
277        let mut records = IndexMap::new();
278        records.insert(tp, vec![record]);
279        records
280    }
281
282    /// Translated from
283    /// `ConsumerRecordsTest.testNextOffsetsDoesNotLogErrorWhenConstructedWithNextOffsets`.
284    ///
285    /// Java also asserts no `ERROR` line is logged. That log belongs to the
286    /// deprecated `ConsumerRecords(Map)` constructor, which is not translated
287    /// (CLAUDE.md §3), so only the returned offsets are asserted here.
288    /// `testNextOffsetsLogsErrorPeriodicallyWhenConstructedWithDeprecatedConstructor`
289    /// exercises that constructor alone and is not translated.
290    #[test]
291    #[doc(
292        alias = "org.apache.kafka.clients.consumer.ConsumerRecordsTest#testNextOffsetsDoesNotLogErrorWhenConstructedWithNextOffsets"
293    )]
294    fn test_next_offsets_does_not_log_error_when_constructed_with_next_offsets() {
295        let records = one_record();
296        let tp = TopicPartition::new("topic".to_string(), 0);
297        let mut next_offsets = HashMap::new();
298        next_offsets.insert(tp, OffsetAndMetadata::new(1).unwrap());
299
300        let consumer_records = ConsumerRecords::with_next_offsets(records, next_offsets.clone());
301
302        assert!(!consumer_records.next_offsets().is_empty());
303        assert_eq!(&next_offsets, consumer_records.next_offsets());
304    }
305
306    /// Translated from
307    /// `ConsumerRecordsTest.testNextOffsetsDoesNotLogErrorForEmptyRecords`.
308    ///
309    /// As above, the no-`ERROR`-log assertion has no counterpart here.
310    #[test]
311    #[doc(
312        alias = "org.apache.kafka.clients.consumer.ConsumerRecordsTest#testNextOffsetsDoesNotLogErrorForEmptyRecords"
313    )]
314    fn test_next_offsets_does_not_log_error_for_empty_records() {
315        let empty: ConsumerRecords<i32, String> = ConsumerRecords::empty();
316        assert!(empty.next_offsets().is_empty());
317    }
318}