confluent_kafka/consumer/consumer_records.rs
1// Copyright 2025 Confluent Inc.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! A container that holds per-partition lists of [`ConsumerRecord`].
16//!
17//! Translated from `org.apache.kafka.clients.consumer.ConsumerRecords`.
18
19use indexmap::IndexMap;
20use std::collections::HashMap;
21
22use crate::common::TopicPartition;
23use crate::consumer::ConsumerRecord;
24use crate::consumer::OffsetAndMetadata;
25
26/// A container that holds the list [`ConsumerRecord`] per partition for a
27/// particular topic. There is one [`ConsumerRecord`] list for every topic
28/// partition returned by a [`Consumer::poll`](crate::consumer::Consumer::poll)
29/// operation.
30// Uses an `IndexMap` internally so iteration order matches insertion order:
31// Java iterates in the underlying map's order, and its tests assume insertion
32// order when a `LinkedHashMap` is supplied.
33#[derive(Debug)]
34#[doc(alias = "org.apache.kafka.clients.consumer.ConsumerRecords")]
35pub struct ConsumerRecords<K, V> {
36 records: IndexMap<TopicPartition, Vec<ConsumerRecord<K, V>>>,
37 next_offsets: HashMap<TopicPartition, OffsetAndMetadata>,
38 // Whether the consumed position advanced for at least one partition
39 // during the `collect_fetch` that produced this batch, even if no
40 // records were returned (e.g. all records in a batch were aborted under
41 // READ_COMMITTED).
42 //
43 // This field carries Java's internal `Fetch.positionAdvanced` flag —
44 // the Rust port collapses Java's internal `Fetch<K, V>` into
45 // `ConsumerRecords<K, V>` (no separate `Fetch` type). The public
46 // [`Self::is_empty`] still mirrors Java's *public*
47 // `ConsumerRecords.isEmpty()` (`records.isEmpty()`), but the poll loop
48 // needs Java's internal `Fetch.isEmpty()` (`numRecords == 0 &&
49 // !positionAdvanced`) so it returns promptly when only the position
50 // advanced — see [`Self::is_fetch_empty`].
51 position_advanced: bool,
52}
53
54impl<K, V> ConsumerRecords<K, V> {
55 /// Constructs a new `ConsumerRecords` with the given records and next offsets.
56 ///
57 /// # Arguments
58 ///
59 /// - `records` - The records for each partition
60 /// - `next_offsets` - The next offset and metadata for each partition whose position was
61 /// advanced during the poll call. These represent the offsets that the consumer will start
62 /// reading from on the next poll.
63 ///
64 /// # Returns
65 ///
66 /// The new `ConsumerRecords`.
67 // Java's deprecated `ConsumerRecords(Map)`, which supplies no next offsets,
68 // is not translated (CLAUDE.md §3).
69 #[doc(alias = "org.apache.kafka.clients.consumer.ConsumerRecords#ConsumerRecords(Map,Map)")]
70 pub fn with_next_offsets(
71 records: IndexMap<TopicPartition, Vec<ConsumerRecord<K, V>>>,
72 next_offsets: HashMap<TopicPartition, OffsetAndMetadata>,
73 ) -> Self {
74 Self { records, next_offsets, position_advanced: false }
75 }
76
77 /// Like [`Self::with_next_offsets`] but also records whether the consumed
78 /// position advanced (Java's internal `Fetch.positionAdvanced`). Used by
79 /// `FetchCollector::collect_fetch` so the poll loop can mirror Java's
80 /// `Fetch.isEmpty()` semantics.
81 #[doc(alias = "org.apache.kafka.clients.consumer.ConsumerRecords#ConsumerRecords")]
82 pub(crate) fn with_position_advanced(
83 records: IndexMap<TopicPartition, Vec<ConsumerRecord<K, V>>>,
84 next_offsets: HashMap<TopicPartition, OffsetAndMetadata>,
85 position_advanced: bool,
86 ) -> Self {
87 Self { records, next_offsets, position_advanced }
88 }
89
90 /// Returns an empty `ConsumerRecords` instance.
91 ///
92 /// # Returns
93 ///
94 /// An empty `ConsumerRecords`.
95 // Java's static `ConsumerRecords.empty()` / `ConsumerRecords.EMPTY`.
96 #[doc(alias = "org.apache.kafka.clients.consumer.ConsumerRecords#empty")]
97 pub fn empty() -> Self {
98 Self { records: IndexMap::new(), next_offsets: HashMap::new(), position_advanced: false }
99 }
100
101 /// Java's internal `Fetch.isEmpty()`: `numRecords == 0 &&
102 /// !positionAdvanced`. The poll loop uses this (NOT [`Self::is_empty`])
103 /// to decide whether to return early — so an all-aborted batch that
104 /// advances the position with zero records returns promptly instead of
105 /// blocking until the poll timeout, matching `AsyncKafkaConsumer.poll`.
106 pub(crate) fn is_fetch_empty(&self) -> bool {
107 Self::fetch_is_empty(&self.records, self.position_advanced)
108 }
109
110 /// Java's internal `Fetch.isEmpty()` (`org.apache.kafka.clients.consumer.internals.Fetch#isEmpty`) evaluated over
111 /// the two accumulators that build a `Fetch`, for callers that do not have
112 /// a `ConsumerRecords` yet.
113 ///
114 /// `FetchCollector::collect_fetch` mirrors Java's
115 /// `final Fetch<K, V> fetch = Fetch.empty()` with a plain
116 /// records-map + `position_advanced` pair (it only materialises the
117 /// `ConsumerRecords` on the way out), yet Java tests `fetch.isEmpty()`
118 /// three times *during* the loop to decide whether to swallow an error and
119 /// whether to leave the offending entry queued. This associated function
120 /// exists so both callers answer the question from one place: the
121 /// records-only spelling was wrong at all three collector sites, and a
122 /// second inline copy of the predicate is exactly how that drifted.
123 ///
124 /// Note `records.is_empty()` is the faithful reading of Java's
125 /// `numRecords == 0`: the collector inserts a partition entry only when it
126 /// decoded at least one record for it, so an empty map and a zero record
127 /// count are the same condition.
128 pub(crate) fn fetch_is_empty(
129 records: &IndexMap<TopicPartition, Vec<ConsumerRecord<K, V>>>,
130 position_advanced: bool,
131 ) -> bool {
132 records.is_empty() && !position_advanced
133 }
134
135 /// Get just the records for the given partition.
136 ///
137 /// # Arguments
138 ///
139 /// - `partition` - The partition to get records for
140 ///
141 /// # Returns
142 ///
143 /// The records, an empty slice if there are none.
144 #[doc(alias = "org.apache.kafka.clients.consumer.ConsumerRecords#records")]
145 pub fn records_with_partition(&self, partition: &TopicPartition) -> &[ConsumerRecord<K, V>] {
146 self.records.get(partition).map(Vec::as_slice).unwrap_or(&[])
147 }
148
149 /// Get just the records for the given topic.
150 ///
151 /// # Arguments
152 ///
153 /// - `topic` - The topic to get records for
154 ///
155 /// # Returns
156 ///
157 /// An iterator over the records, in partition-insertion order.
158 #[doc(alias = "org.apache.kafka.clients.consumer.ConsumerRecords#records")]
159 pub fn records_with_topic<'a>(&'a self, topic: &'a str) -> impl Iterator<Item = &'a ConsumerRecord<K, V>> + 'a {
160 self.records
161 .iter()
162 .filter(move |(tp, _)| tp.topic() == topic)
163 .flat_map(|(_, records)| records.iter())
164 }
165
166 /// Get the partitions which have records contained in this record set.
167 ///
168 /// # Returns
169 ///
170 /// The set of partitions with data in this record set (may be empty if no data was returned),
171 /// as an iterator in insertion order.
172 #[doc(alias = "org.apache.kafka.clients.consumer.ConsumerRecords#partitions")]
173 pub fn partitions(&self) -> impl Iterator<Item = &TopicPartition> {
174 self.records.keys()
175 }
176
177 /// Returns the number of records for all topics.
178 #[doc(alias = "org.apache.kafka.clients.consumer.ConsumerRecords#count")]
179 pub fn count(&self) -> usize {
180 self.records.values().map(Vec::len).sum()
181 }
182
183 /// Returns whether this container has any records.
184 ///
185 /// # Returns
186 ///
187 /// True if there are no records, false otherwise
188 // Java returns `records.isEmpty()`, not `count() == 0`.
189 #[doc(alias = "org.apache.kafka.clients.consumer.ConsumerRecords#isEmpty")]
190 pub fn is_empty(&self) -> bool {
191 self.records.is_empty()
192 }
193
194 /// Get the next offsets and metadata corresponding to all topic partitions
195 /// for which the position have been advanced in this poll call.
196 ///
197 /// # Returns
198 ///
199 /// The next offsets that the consumer will consume
200 #[doc(alias = "org.apache.kafka.clients.consumer.ConsumerRecords#nextOffsets")]
201 pub fn next_offsets(&self) -> &HashMap<TopicPartition, OffsetAndMetadata> {
202 &self.next_offsets
203 }
204}
205
206impl<K, V> Default for ConsumerRecords<K, V> {
207 fn default() -> Self {
208 Self::empty()
209 }
210}
211
212/// Borrowed iteration over every record across all partitions (in
213/// partition-insertion order).
214///
215/// Equivalent to Java's `ConsumerRecords.iterator()` (which iterates the
216/// concatenated values of the underlying map).
217impl<'a, K, V> IntoIterator for &'a ConsumerRecords<K, V> {
218 type Item = &'a ConsumerRecord<K, V>;
219 type IntoIter = std::iter::FlatMap<
220 indexmap::map::Values<'a, TopicPartition, Vec<ConsumerRecord<K, V>>>,
221 std::slice::Iter<'a, ConsumerRecord<K, V>>,
222 fn(&'a Vec<ConsumerRecord<K, V>>) -> std::slice::Iter<'a, ConsumerRecord<K, V>>,
223 >;
224
225 fn into_iter(self) -> Self::IntoIter {
226 // The function pointer type must reference `&Vec<T>` exactly to match
227 // the `IntoIter` associated type alias above; using `&[T]` here would
228 // produce a slice iterator whose lifetime cannot be unified with the
229 // map values iterator.
230 #[expect(clippy::ptr_arg)]
231 fn iter_vec<T>(v: &Vec<T>) -> std::slice::Iter<'_, T> {
232 v.iter()
233 }
234 self.records.values().flat_map(iter_vec)
235 }
236}
237
238/// Owned iteration that consumes the [`ConsumerRecords`].
239impl<K, V> IntoIterator for ConsumerRecords<K, V> {
240 type Item = ConsumerRecord<K, V>;
241 type IntoIter = std::iter::FlatMap<
242 indexmap::map::IntoValues<TopicPartition, Vec<ConsumerRecord<K, V>>>,
243 std::vec::IntoIter<ConsumerRecord<K, V>>,
244 fn(Vec<ConsumerRecord<K, V>>) -> std::vec::IntoIter<ConsumerRecord<K, V>>,
245 >;
246
247 fn into_iter(self) -> Self::IntoIter {
248 fn iter_vec<T>(v: Vec<T>) -> std::vec::IntoIter<T> {
249 v.into_iter()
250 }
251 self.records.into_values().flat_map(iter_vec)
252 }
253}
254
255#[cfg(test)]
256mod tests {
257 use super::*;
258 use crate::common::record::TimestampType;
259 use crate::consumer::ConsumerRecordOptionsBuilder;
260
261 fn one_record() -> IndexMap<TopicPartition, Vec<ConsumerRecord<i32, String>>> {
262 let tp = TopicPartition::new("topic".to_string(), 0);
263 let record = ConsumerRecord::with_options(
264 ConsumerRecordOptionsBuilder::new()
265 .set_topic("topic")
266 .set_partition(0)
267 .set_offset(0)
268 .set_key(Some(0))
269 .set_value(Some("value".to_string()))
270 .set_timestamp(0)
271 .set_timestamp_type(TimestampType::CreateTime)
272 .set_serialized_key_size(0)
273 .set_serialized_value_size(0)
274 .build()
275 .unwrap(),
276 );
277 let mut records = IndexMap::new();
278 records.insert(tp, vec![record]);
279 records
280 }
281
282 /// Translated from
283 /// `ConsumerRecordsTest.testNextOffsetsDoesNotLogErrorWhenConstructedWithNextOffsets`.
284 ///
285 /// Java also asserts no `ERROR` line is logged. That log belongs to the
286 /// deprecated `ConsumerRecords(Map)` constructor, which is not translated
287 /// (CLAUDE.md §3), so only the returned offsets are asserted here.
288 /// `testNextOffsetsLogsErrorPeriodicallyWhenConstructedWithDeprecatedConstructor`
289 /// exercises that constructor alone and is not translated.
290 #[test]
291 #[doc(
292 alias = "org.apache.kafka.clients.consumer.ConsumerRecordsTest#testNextOffsetsDoesNotLogErrorWhenConstructedWithNextOffsets"
293 )]
294 fn test_next_offsets_does_not_log_error_when_constructed_with_next_offsets() {
295 let records = one_record();
296 let tp = TopicPartition::new("topic".to_string(), 0);
297 let mut next_offsets = HashMap::new();
298 next_offsets.insert(tp, OffsetAndMetadata::new(1).unwrap());
299
300 let consumer_records = ConsumerRecords::with_next_offsets(records, next_offsets.clone());
301
302 assert!(!consumer_records.next_offsets().is_empty());
303 assert_eq!(&next_offsets, consumer_records.next_offsets());
304 }
305
306 /// Translated from
307 /// `ConsumerRecordsTest.testNextOffsetsDoesNotLogErrorForEmptyRecords`.
308 ///
309 /// As above, the no-`ERROR`-log assertion has no counterpart here.
310 #[test]
311 #[doc(
312 alias = "org.apache.kafka.clients.consumer.ConsumerRecordsTest#testNextOffsetsDoesNotLogErrorForEmptyRecords"
313 )]
314 fn test_next_offsets_does_not_log_error_for_empty_records() {
315 let empty: ConsumerRecords<i32, String> = ConsumerRecords::empty();
316 assert!(empty.next_offsets().is_empty());
317 }
318}