Merge remote-tracking branch 'public/main' into support-parse-list

Author: dqkqd

.github/workflows/docs.yml

@@ -71,7 +71,7 @@ jobs:
     steps:
       - uses: actions/checkout@v5
       - name: Download crate docs
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: crate-docs
           path: website/build

CHANGELOG-old.md

@@ -68,7 +68,7 @@ exclude = [
 ]
 
 [workspace.package]
-version = "56.2.0"
+version = "57.0.0"
 homepage = "https://github.com/apache/arrow-rs"
 repository = "https://github.com/apache/arrow-rs"
 authors = ["Apache Arrow <dev@arrow.apache.org>"]
@@ -85,22 +85,22 @@ edition = "2024"
 rust-version = "1.85"
 
 [workspace.dependencies]
-arrow = { version = "56.2.0", path = "./arrow", default-features = false }
-arrow-arith = { version = "56.2.0", path = "./arrow-arith" }
-arrow-array = { version = "56.2.0", path = "./arrow-array" }
-arrow-buffer = { version = "56.2.0", path = "./arrow-buffer" }
-arrow-cast = { version = "56.2.0", path = "./arrow-cast" }
-arrow-csv = { version = "56.2.0", path = "./arrow-csv" }
-arrow-data = { version = "56.2.0", path = "./arrow-data" }
-arrow-ipc = { version = "56.2.0", path = "./arrow-ipc" }
-arrow-json = { version = "56.2.0", path = "./arrow-json" }
-arrow-ord = { version = "56.2.0", path = "./arrow-ord" }
-arrow-pyarrow = { version = "56.2.0", path = "./arrow-pyarrow" }
-arrow-row = { version = "56.2.0", path = "./arrow-row" }
-arrow-schema = { version = "56.2.0", path = "./arrow-schema" }
-arrow-select = { version = "56.2.0", path = "./arrow-select" }
-arrow-string = { version = "56.2.0", path = "./arrow-string" }
-parquet = { version = "56.2.0", path = "./parquet", default-features = false }
+arrow = { version = "57.0.0", path = "./arrow", default-features = false }
+arrow-arith = { version = "57.0.0", path = "./arrow-arith" }
+arrow-array = { version = "57.0.0", path = "./arrow-array" }
+arrow-buffer = { version = "57.0.0", path = "./arrow-buffer" }
+arrow-cast = { version = "57.0.0", path = "./arrow-cast" }
+arrow-csv = { version = "57.0.0", path = "./arrow-csv" }
+arrow-data = { version = "57.0.0", path = "./arrow-data" }
+arrow-ipc = { version = "57.0.0", path = "./arrow-ipc" }
+arrow-json = { version = "57.0.0", path = "./arrow-json" }
+arrow-ord = { version = "57.0.0", path = "./arrow-ord" }
+arrow-pyarrow = { version = "57.0.0", path = "./arrow-pyarrow" }
+arrow-row = { version = "57.0.0", path = "./arrow-row" }
+arrow-schema = { version = "57.0.0", path = "./arrow-schema" }
+arrow-select = { version = "57.0.0", path = "./arrow-select" }
+arrow-string = { version = "57.0.0", path = "./arrow-string" }
+parquet = { version = "57.0.0", path = "./parquet", default-features = false }
 
 # These crates have not yet been released and thus do not use the workspace version
 parquet-geospatial = { version = "0.1.0", path = "./parquet-geospatial" }

CHANGELOG.md

@@ -466,7 +466,7 @@ impl<OffsetSize: OffsetSizeTrait> From<FixedSizeListArray> for GenericListArray<
             _ => unreachable!(),
         };
 
-        let offsets = OffsetBuffer::from_lengths(std::iter::repeat_n(size, value.len()));
+        let offsets = OffsetBuffer::from_repeated_length(size, value.len());
 
         Self {
             data_type: Self::DATA_TYPE_CONSTRUCTOR(field.clone()),

Cargo.toml

@@ -306,15 +306,30 @@ impl<T: ByteViewType + ?Sized> GenericByteViewBuilder<T> {
     /// - String length exceeds `u32::MAX`
     #[inline]
     pub fn append_value(&mut self, value: impl AsRef<T::Native>) {
+        self.try_append_value(value).unwrap()
+    }
+
+    /// Appends a value into the builder
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if:
+    /// - String buffer count exceeds `u32::MAX`
+    /// - String length exceeds `u32::MAX`
+    #[inline]
+    pub fn try_append_value(&mut self, value: impl AsRef<T::Native>) -> Result<(), ArrowError> {
         let v: &[u8] = value.as_ref().as_ref();
-        let length: u32 = v.len().try_into().unwrap();
+        let length: u32 = v.len().try_into().map_err(|_| {
+            ArrowError::InvalidArgumentError(format!("String length {} exceeds u32::MAX", v.len()))
+        })?;
+
         if length <= MAX_INLINE_VIEW_LEN {
             let mut view_buffer = [0; 16];
             view_buffer[0..4].copy_from_slice(&length.to_le_bytes());
             view_buffer[4..4 + v.len()].copy_from_slice(v);
             self.views_buffer.push(u128::from_le_bytes(view_buffer));
             self.null_buffer_builder.append_non_null();
-            return;
+            return Ok(());
         }
 
         // Deduplication if:
@@ -339,7 +354,7 @@ impl<T: ByteViewType + ?Sized> GenericByteViewBuilder<T> {
                     self.views_buffer.push(self.views_buffer[*idx]);
                     self.null_buffer_builder.append_non_null();
                     self.string_tracker = Some((ht, hasher));
-                    return;
+                    return Ok(());
                 }
                 Entry::Vacant(vacant) => {
                     // o.w. we insert the (string hash -> view index)
@@ -356,17 +371,28 @@ impl<T: ByteViewType + ?Sized> GenericByteViewBuilder<T> {
             let to_reserve = v.len().max(self.block_size.next_size() as usize);
             self.in_progress.reserve(to_reserve);
         };
+
         let offset = self.in_progress.len() as u32;
         self.in_progress.extend_from_slice(v);
 
+        let buffer_index: u32 = self.completed.len().try_into().map_err(|_| {
+            ArrowError::InvalidArgumentError(format!(
+                "Buffer count {} exceeds u32::MAX",
+                self.completed.len()
+            ))
+        })?;
+
         let view = ByteView {
             length,
+            // This won't panic as we checked the length of prefix earlier.
             prefix: u32::from_le_bytes(v[0..4].try_into().unwrap()),
-            buffer_index: self.completed.len() as u32,
+            buffer_index,
             offset,
         };
         self.views_buffer.push(view.into());
         self.null_buffer_builder.append_non_null();
+
+        Ok(())
     }
 
     /// Append an `Option` value into the builder
@@ -581,7 +607,6 @@ mod tests {
     use core::str;
 
     use super::*;
-    use crate::Array;
 
     #[test]
     fn test_string_view_deduplicate() {

arrow-array/src/array/list_array.rs

@@ -1324,7 +1324,7 @@ pub trait DecimalType:
     /// Maximum no of digits after the decimal point (note the scale can be negative)
     const MAX_SCALE: i8;
     /// The maximum value for each precision in `0..=MAX_PRECISION`: [0, 9, 99, ...]
-    const MAX_FOR_EACH_PRECISION: &[Self::Native];
+    const MAX_FOR_EACH_PRECISION: &'static [Self::Native];
     /// fn to create its [`DataType`]
     const TYPE_CONSTRUCTOR: fn(u8, i8) -> DataType;
     /// Default values for [`DataType`]
@@ -1395,7 +1395,8 @@ impl DecimalType for Decimal32Type {
     const BYTE_LENGTH: usize = 4;
     const MAX_PRECISION: u8 = DECIMAL32_MAX_PRECISION;
     const MAX_SCALE: i8 = DECIMAL32_MAX_SCALE;
-    const MAX_FOR_EACH_PRECISION: &[i32] = &arrow_data::decimal::MAX_DECIMAL32_FOR_EACH_PRECISION;
+    const MAX_FOR_EACH_PRECISION: &'static [i32] =
+        &arrow_data::decimal::MAX_DECIMAL32_FOR_EACH_PRECISION;
     const TYPE_CONSTRUCTOR: fn(u8, i8) -> DataType = DataType::Decimal32;
     const DEFAULT_TYPE: DataType =
         DataType::Decimal32(DECIMAL32_MAX_PRECISION, DECIMAL32_DEFAULT_SCALE);
@@ -1430,7 +1431,8 @@ impl DecimalType for Decimal64Type {
     const BYTE_LENGTH: usize = 8;
     const MAX_PRECISION: u8 = DECIMAL64_MAX_PRECISION;
     const MAX_SCALE: i8 = DECIMAL64_MAX_SCALE;
-    const MAX_FOR_EACH_PRECISION: &[i64] = &arrow_data::decimal::MAX_DECIMAL64_FOR_EACH_PRECISION;
+    const MAX_FOR_EACH_PRECISION: &'static [i64] =
+        &arrow_data::decimal::MAX_DECIMAL64_FOR_EACH_PRECISION;
     const TYPE_CONSTRUCTOR: fn(u8, i8) -> DataType = DataType::Decimal64;
     const DEFAULT_TYPE: DataType =
         DataType::Decimal64(DECIMAL64_MAX_PRECISION, DECIMAL64_DEFAULT_SCALE);
@@ -1465,7 +1467,8 @@ impl DecimalType for Decimal128Type {
     const BYTE_LENGTH: usize = 16;
     const MAX_PRECISION: u8 = DECIMAL128_MAX_PRECISION;
     const MAX_SCALE: i8 = DECIMAL128_MAX_SCALE;
-    const MAX_FOR_EACH_PRECISION: &[i128] = &arrow_data::decimal::MAX_DECIMAL128_FOR_EACH_PRECISION;
+    const MAX_FOR_EACH_PRECISION: &'static [i128] =
+        &arrow_data::decimal::MAX_DECIMAL128_FOR_EACH_PRECISION;
     const TYPE_CONSTRUCTOR: fn(u8, i8) -> DataType = DataType::Decimal128;
     const DEFAULT_TYPE: DataType =
         DataType::Decimal128(DECIMAL128_MAX_PRECISION, DECIMAL_DEFAULT_SCALE);
@@ -1500,7 +1503,8 @@ impl DecimalType for Decimal256Type {
     const BYTE_LENGTH: usize = 32;
     const MAX_PRECISION: u8 = DECIMAL256_MAX_PRECISION;
     const MAX_SCALE: i8 = DECIMAL256_MAX_SCALE;
-    const MAX_FOR_EACH_PRECISION: &[i256] = &arrow_data::decimal::MAX_DECIMAL256_FOR_EACH_PRECISION;
+    const MAX_FOR_EACH_PRECISION: &'static [i256] =
+        &arrow_data::decimal::MAX_DECIMAL256_FOR_EACH_PRECISION;
     const TYPE_CONSTRUCTOR: fn(u8, i8) -> DataType = DataType::Decimal256;
     const DEFAULT_TYPE: DataType =
         DataType::Decimal256(DECIMAL256_MAX_PRECISION, DECIMAL_DEFAULT_SCALE);

arrow-array/src/builder/generic_bytes_view_builder.rs

@@ -44,14 +44,14 @@ This crate provides:
 
 ```toml
 [dependencies]
-arrow-avro = "56"
+arrow-avro = "57.0.0"
 ````
 
 Disable defaults and pick only what you need (see **Feature Flags**):
 
 ```toml
 [dependencies]
-arrow-avro = { version = "56", default-features = false, features = ["deflate", "snappy"] }
+arrow-avro = { version = "57.0.0", default-features = false, features = ["deflate", "snappy"] }
 ```
 
 ---

arrow-array/src/types.rs

@@ -59,3 +59,8 @@ harness = false
 [[bench]]
 name = "offset"
 harness = false
+
+[[bench]]
+name = "mutable_buffer_repeat_slice"
+harness = false
+

arrow-avro/README.md

@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use arrow_buffer::Buffer;
+use criterion::*;
+use rand::distr::Alphanumeric;
+use rand::rngs::StdRng;
+use rand::{Rng, SeedableRng};
+use std::hint;
+
+fn criterion_benchmark(c: &mut Criterion) {
+    let mut group = c.benchmark_group("MutableBuffer repeat slice");
+    let mut rng = StdRng::seed_from_u64(42);
+
+    for slice_length in [3, 20, 100] {
+        let slice_to_repeat: Vec<u8> = hint::black_box(
+            (&mut rng)
+                .sample_iter(&Alphanumeric)
+                .take(slice_length)
+                .collect(),
+        );
+        let slice_to_repeat: &[u8] = slice_to_repeat.as_ref();
+
+        for repeat_count in [3, 64, 1024, 8192] {
+            let parameter_string = format!("slice_len={slice_length} n={repeat_count}");
+
+            group.bench_with_input(
+                BenchmarkId::new("repeat_slice_n_times", &parameter_string),
+                &(repeat_count),
+                |b, &repeat_count| {
+                    b.iter(|| {
+                        let mut mutable_buffer = arrow_buffer::MutableBuffer::with_capacity(0);
+
+                        mutable_buffer.repeat_slice_n_times(slice_to_repeat, repeat_count);
+
+                        let buffer: Buffer = mutable_buffer.into();
+
+                        buffer
+                    })
+                },
+            );
+            group.bench_with_input(
+                BenchmarkId::new("extend_from_slice loop", &parameter_string),
+                &(repeat_count),
+                |b, &repeat_count| {
+                    b.iter(|| {
+                        let mut mutable_buffer = arrow_buffer::MutableBuffer::with_capacity(
+                            size_of_val(slice_to_repeat) * repeat_count,
+                        );
+
+                        for _ in 0..repeat_count {
+                            mutable_buffer.extend_from_slice(slice_to_repeat);
+                        }
+
+                        let buffer: Buffer = mutable_buffer.into();
+
+                        buffer
+                    })
+                },
+            );
+        }
+    }
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);