From f89c2638ef1f1da12f0cbf093492b72677e355ab Mon Sep 17 00:00:00 2001 From: Shar-jeel-Sajid Date: Mon, 17 Mar 2025 06:03:10 +0500 Subject: [PATCH 1/4] procedural macros in into_impls file --- libraries/arrow-convert/src/into_impls.rs | 173 +++++++--------------- 1 file changed, 53 insertions(+), 120 deletions(-) diff --git a/libraries/arrow-convert/src/into_impls.rs b/libraries/arrow-convert/src/into_impls.rs index 055d0bcb..8d86e444 100644 --- a/libraries/arrow-convert/src/into_impls.rs +++ b/libraries/arrow-convert/src/into_impls.rs @@ -1,6 +1,9 @@ use crate::IntoArrow; use arrow::array::{PrimitiveArray, StringArray, TimestampNanosecondArray}; -use arrow::datatypes::{ArrowPrimitiveType, ArrowTimestampType}; +use arrow::datatypes::{ + ArrowPrimitiveType, ArrowTimestampType, Float32Type, Float64Type, Int16Type, Int32Type, + Int64Type, Int8Type, UInt16Type, UInt32Type, UInt64Type, UInt8Type, +}; use chrono::{NaiveDate, NaiveDateTime, NaiveTime}; impl IntoArrow for bool { @@ -10,67 +13,58 @@ impl IntoArrow for bool { } } -impl IntoArrow for u8 { - type A = PrimitiveArray; - fn into_arrow(self) -> Self::A { - std::iter::once(self).collect() - } -} -impl IntoArrow for u16 { - type A = PrimitiveArray; - fn into_arrow(self) -> Self::A { - std::iter::once(self).collect() - } -} -impl IntoArrow for u32 { - type A = PrimitiveArray; - fn into_arrow(self) -> Self::A { - std::iter::once(self).collect() - } +macro_rules! impl_into_arrow_for_uint { + ($($t:ty => $arrow_type:ty),*) => { + $( + impl IntoArrow for $t { + type A = PrimitiveArray<$arrow_type>; + fn into_arrow(self) -> Self::A { + std::iter::once(self).collect() + } + } + )* + }; } -impl IntoArrow for u64 { - type A = PrimitiveArray; - fn into_arrow(self) -> Self::A { - std::iter::once(self).collect() - } -} -impl IntoArrow for i8 { - type A = PrimitiveArray; - fn into_arrow(self) -> Self::A { - std::iter::once(self).collect() - } -} -impl IntoArrow for i16 { - type A = PrimitiveArray; - fn into_arrow(self) -> Self::A { - std::iter::once(self).collect() - } -} -impl IntoArrow for i32 { - type A = PrimitiveArray; - fn into_arrow(self) -> Self::A { - std::iter::once(self).collect() - } -} -impl IntoArrow for i64 { - type A = PrimitiveArray; - fn into_arrow(self) -> Self::A { - std::iter::once(self).collect() - } -} -impl IntoArrow for f32 { - type A = PrimitiveArray; - fn into_arrow(self) -> Self::A { - std::iter::once(self).collect() - } -} -impl IntoArrow for f64 { - type A = PrimitiveArray; - fn into_arrow(self) -> Self::A { - std::iter::once(self).collect() - } + +macro_rules! impl_into_arrow_for_vec { + ($($t:ty => $arrow_type:ty),*) => { + $( + impl IntoArrow for Vec<$t> { + type A = PrimitiveArray<$arrow_type>; + fn into_arrow(self) -> Self::A { + self.into() + } + } + )* + }; } +impl_into_arrow_for_uint!( + u8 => UInt8Type, + u16 => UInt16Type, + u32 => UInt32Type, + u64 => UInt64Type, + i8 => Int8Type, + i16 => Int16Type, + i32 => Int32Type, + i64 => Int64Type, + f32 => Float32Type, + f64 => Float64Type +); + +impl_into_arrow_for_vec!( + u8 => UInt8Type, + u16 => UInt16Type, + u32 => UInt32Type, + u64 => UInt64Type, + i8 => Int8Type, + i16 => Int16Type, + i32 => Int32Type, + i64 => Int64Type, + f32 => Float32Type, + f64 => Float64Type +); + impl IntoArrow for &str { type A = StringArray; fn into_arrow(self) -> Self::A { @@ -78,67 +72,6 @@ impl IntoArrow for &str { } } -impl IntoArrow for Vec { - type A = PrimitiveArray; - fn into_arrow(self) -> Self::A { - self.into() - } -} -impl IntoArrow for Vec { - type A = PrimitiveArray; - fn into_arrow(self) -> Self::A { - self.into() - } -} -impl IntoArrow for Vec { - type A = PrimitiveArray; - fn into_arrow(self) -> Self::A { - self.into() - } -} -impl IntoArrow for Vec { - type A = PrimitiveArray; - fn into_arrow(self) -> Self::A { - self.into() - } -} -impl IntoArrow for Vec { - type A = PrimitiveArray; - fn into_arrow(self) -> Self::A { - self.into() - } -} -impl IntoArrow for Vec { - type A = PrimitiveArray; - fn into_arrow(self) -> Self::A { - self.into() - } -} -impl IntoArrow for Vec { - type A = PrimitiveArray; - fn into_arrow(self) -> Self::A { - self.into() - } -} -impl IntoArrow for Vec { - type A = PrimitiveArray; - fn into_arrow(self) -> Self::A { - self.into() - } -} -impl IntoArrow for Vec { - type A = PrimitiveArray; - fn into_arrow(self) -> Self::A { - self.into() - } -} -impl IntoArrow for Vec { - type A = PrimitiveArray; - fn into_arrow(self) -> Self::A { - self.into() - } -} - impl IntoArrow for () { type A = arrow::array::NullArray; From 77532746ff99df8246e591abf881d465ad496671 Mon Sep 17 00:00:00 2001 From: Shar-jeel-Sajid Date: Mon, 17 Mar 2025 08:16:08 +0500 Subject: [PATCH 2/4] procedural macros in from_impls file --- libraries/arrow-convert/src/from_impls.rs | 368 +++++----------------- libraries/arrow-convert/src/into_impls.rs | 26 +- 2 files changed, 92 insertions(+), 302 deletions(-) diff --git a/libraries/arrow-convert/src/from_impls.rs b/libraries/arrow-convert/src/from_impls.rs index 465dfa66..08c77fad 100644 --- a/libraries/arrow-convert/src/from_impls.rs +++ b/libraries/arrow-convert/src/from_impls.rs @@ -36,98 +36,88 @@ impl TryFrom<&ArrowData> for bool { Ok(bool_array.value(0)) } } -impl TryFrom<&ArrowData> for u8 { - type Error = eyre::Report; - fn try_from(value: &ArrowData) -> Result { - let array = value - .as_primitive_opt::() - .context("not a primitive UInt8Type array")?; - extract_single_primitive(array) - } -} -impl TryFrom<&ArrowData> for u16 { - type Error = eyre::Report; - fn try_from(value: &ArrowData) -> Result { - let array = value - .as_primitive_opt::() - .context("not a primitive UInt16Type array")?; - extract_single_primitive(array) - } -} -impl TryFrom<&ArrowData> for u32 { - type Error = eyre::Report; - fn try_from(value: &ArrowData) -> Result { - let array = value - .as_primitive_opt::() - .context("not a primitive UInt32Type array")?; - extract_single_primitive(array) - } -} -impl TryFrom<&ArrowData> for u64 { - type Error = eyre::Report; - fn try_from(value: &ArrowData) -> Result { - let array = value - .as_primitive_opt::() - .context("not a primitive UInt64Type array")?; - extract_single_primitive(array) - } -} -impl TryFrom<&ArrowData> for i8 { - type Error = eyre::Report; - fn try_from(value: &ArrowData) -> Result { - let array = value - .as_primitive_opt::() - .context("not a primitive Int8Type array")?; - extract_single_primitive(array) - } -} -impl TryFrom<&ArrowData> for i16 { - type Error = eyre::Report; - fn try_from(value: &ArrowData) -> Result { - let array = value - .as_primitive_opt::() - .context("not a primitive Int16Type array")?; - extract_single_primitive(array) - } -} -impl TryFrom<&ArrowData> for i32 { - type Error = eyre::Report; - fn try_from(value: &ArrowData) -> Result { - let array = value - .as_primitive_opt::() - .context("not a primitive Int32Type array")?; - extract_single_primitive(array) - } -} -impl TryFrom<&ArrowData> for i64 { - type Error = eyre::Report; - fn try_from(value: &ArrowData) -> Result { - let array = value - .as_primitive_opt::() - .context("not a primitive Int64Type array")?; - extract_single_primitive(array) - } -} -impl TryFrom<&ArrowData> for f32 { - type Error = eyre::Report; - fn try_from(value: &ArrowData) -> Result { - let array = value - .as_primitive_opt::() - .context("not a primitive Float32Type array")?; - extract_single_primitive(array) - } -} -impl TryFrom<&ArrowData> for f64 { - type Error = eyre::Report; - fn try_from(value: &ArrowData) -> Result { - let array = value - .as_primitive_opt::() - .context("not a primitive Float64Type array")?; - extract_single_primitive(array) - } +macro_rules! impl_try_from_arrow_data { + ($($t:ty => $arrow_type:ident),*) => { + $( + impl TryFrom<&ArrowData> for $t { + type Error = eyre::Report; + + fn try_from(value: &ArrowData) -> Result { + let array = value + .as_primitive_opt::() + .context(concat!("not a primitive ", stringify!($arrow_type), " array"))?; + extract_single_primitive(array) + } + } + )* + }; +} + +impl_try_from_arrow_data!( + u8 => UInt8Type, + u16 => UInt16Type, + u32 => UInt32Type, + u64 => UInt64Type, + i8 => Int8Type, + i16 =>Int16Type, + i32 => Int32Type, + i64 => Int64Type, + f32 => Float32Type, + f64 => Float64Type +); + +macro_rules! impl_try_from_arrow_data_for_slice { + ($($t:ty => $arrow_type:ident),*) => { + $( + impl<'a> TryFrom<&'a ArrowData> for &'a [$t] { + type Error = eyre::Report; + + fn try_from(value: &'a ArrowData) -> Result { + let array: &PrimitiveArray = value + .as_primitive_opt() + .wrap_err(concat!("not a primitive ", stringify!($arrow_type), " array"))?; + if array.null_count() != 0 { + eyre::bail!("array has nulls"); + } + Ok(array.values()) + } + } + )* + }; +} + +impl_try_from_arrow_data_for_slice!( + u8 => UInt8Type, + u16 => UInt16Type, + u32 => UInt32Type, + u64 => UInt64Type, + i8 => Int8Type, + i16 =>Int16Type, + i32 => Int32Type, + i64 => Int64Type, + f32 => Float32Type, + f64 => Float64Type +); + +macro_rules! impl_try_from_arrow_data_for_vec { + ($($t:ty),*) => { + $( + impl<'a> TryFrom<&'a ArrowData> for Vec<$t> { + type Error = eyre::Report; + + fn try_from(value: &'a ArrowData) -> Result { + value + .try_into() + .map(|slice: &'a [$t]| slice.to_vec()) + } + } + )* + }; } +impl_try_from_arrow_data_for_vec!(u8, u16, u32, u64, i8, i16, i32, i64, f32, f64); + impl<'a> TryFrom<&'a ArrowData> for &'a str { type Error = eyre::Report; fn try_from(value: &'a ArrowData) -> Result { @@ -145,206 +135,6 @@ impl<'a> TryFrom<&'a ArrowData> for &'a str { } } -impl<'a> TryFrom<&'a ArrowData> for &'a [u8] { - type Error = eyre::Report; - fn try_from(value: &'a ArrowData) -> Result { - let array: &PrimitiveArray = value - .as_primitive_opt() - .wrap_err("not a primitive UInt8Type array")?; - if array.null_count() != 0 { - eyre::bail!("array has nulls"); - } - Ok(array.values()) - } -} - -impl<'a> TryFrom<&'a ArrowData> for Vec { - type Error = eyre::Report; - fn try_from(value: &'a ArrowData) -> Result { - value.try_into().map(|slice: &'a [u8]| slice.to_vec()) - } -} - -impl<'a> TryFrom<&'a ArrowData> for &'a [u16] { - type Error = eyre::Report; - fn try_from(value: &'a ArrowData) -> Result { - let array: &PrimitiveArray = value - .as_primitive_opt() - .wrap_err("not a primitive UInt16Type array")?; - if array.null_count() != 0 { - eyre::bail!("array has nulls"); - } - Ok(array.values()) - } -} - -impl<'a> TryFrom<&'a ArrowData> for Vec { - type Error = eyre::Report; - fn try_from(value: &'a ArrowData) -> Result { - value.try_into().map(|slice: &'a [u16]| slice.to_vec()) - } -} - -impl<'a> TryFrom<&'a ArrowData> for &'a [u32] { - type Error = eyre::Report; - fn try_from(value: &'a ArrowData) -> Result { - let array: &PrimitiveArray = value - .as_primitive_opt() - .wrap_err("not a primitive UInt32Type array")?; - if array.null_count() != 0 { - eyre::bail!("array has nulls"); - } - Ok(array.values()) - } -} - -impl<'a> TryFrom<&'a ArrowData> for Vec { - type Error = eyre::Report; - fn try_from(value: &'a ArrowData) -> Result { - value.try_into().map(|slice: &'a [u32]| slice.to_vec()) - } -} - -impl<'a> TryFrom<&'a ArrowData> for &'a [u64] { - type Error = eyre::Report; - fn try_from(value: &'a ArrowData) -> Result { - let array: &PrimitiveArray = value - .as_primitive_opt() - .wrap_err("not a primitive UInt64Type array")?; - if array.null_count() != 0 { - eyre::bail!("array has nulls"); - } - Ok(array.values()) - } -} - -impl<'a> TryFrom<&'a ArrowData> for Vec { - type Error = eyre::Report; - fn try_from(value: &'a ArrowData) -> Result { - value.try_into().map(|slice: &'a [u64]| slice.to_vec()) - } -} - -impl<'a> TryFrom<&'a ArrowData> for &'a [i8] { - type Error = eyre::Report; - fn try_from(value: &'a ArrowData) -> Result { - let array: &PrimitiveArray = value - .as_primitive_opt() - .wrap_err("not a primitive Int8Type array")?; - if array.null_count() != 0 { - eyre::bail!("array has nulls"); - } - Ok(array.values()) - } -} - -impl<'a> TryFrom<&'a ArrowData> for Vec { - type Error = eyre::Report; - fn try_from(value: &'a ArrowData) -> Result { - value.try_into().map(|slice: &'a [i8]| slice.to_vec()) - } -} - -impl<'a> TryFrom<&'a ArrowData> for &'a [i16] { - type Error = eyre::Report; - fn try_from(value: &'a ArrowData) -> Result { - let array: &PrimitiveArray = value - .as_primitive_opt() - .wrap_err("not a primitive Int16Type array")?; - if array.null_count() != 0 { - eyre::bail!("array has nulls"); - } - Ok(array.values()) - } -} - -impl<'a> TryFrom<&'a ArrowData> for Vec { - type Error = eyre::Report; - fn try_from(value: &'a ArrowData) -> Result { - value.try_into().map(|slice: &'a [i16]| slice.to_vec()) - } -} - -impl<'a> TryFrom<&'a ArrowData> for &'a [i32] { - type Error = eyre::Report; - fn try_from(value: &'a ArrowData) -> Result { - let array: &PrimitiveArray = value - .as_primitive_opt() - .wrap_err("not a primitive Int32Type array")?; - if array.null_count() != 0 { - eyre::bail!("array has nulls"); - } - Ok(array.values()) - } -} - -impl<'a> TryFrom<&'a ArrowData> for Vec { - type Error = eyre::Report; - fn try_from(value: &'a ArrowData) -> Result { - value.try_into().map(|slice: &'a [i32]| slice.to_vec()) - } -} - -impl<'a> TryFrom<&'a ArrowData> for &'a [i64] { - type Error = eyre::Report; - fn try_from(value: &'a ArrowData) -> Result { - let array: &PrimitiveArray = value - .as_primitive_opt() - .wrap_err("not a primitive Int64Type array")?; - if array.null_count() != 0 { - eyre::bail!("array has nulls"); - } - Ok(array.values()) - } -} - -impl<'a> TryFrom<&'a ArrowData> for Vec { - type Error = eyre::Report; - fn try_from(value: &'a ArrowData) -> Result { - value.try_into().map(|slice: &'a [i64]| slice.to_vec()) - } -} - -impl<'a> TryFrom<&'a ArrowData> for &'a [f32] { - type Error = eyre::Report; - fn try_from(value: &'a ArrowData) -> Result { - let array: &PrimitiveArray = value - .as_primitive_opt() - .wrap_err("not a primitive Float32Type array")?; - if array.null_count() != 0 { - eyre::bail!("array has nulls"); - } - Ok(array.values()) - } -} - -impl<'a> TryFrom<&'a ArrowData> for Vec { - type Error = eyre::Report; - fn try_from(value: &'a ArrowData) -> Result { - value.try_into().map(|slice: &'a [f32]| slice.to_vec()) - } -} - -impl<'a> TryFrom<&'a ArrowData> for &'a [f64] { - type Error = eyre::Report; - fn try_from(value: &'a ArrowData) -> Result { - let array: &PrimitiveArray = value - .as_primitive_opt() - .wrap_err("not a primitive Float64Type array")?; - if array.null_count() != 0 { - eyre::bail!("array has nulls"); - } - Ok(array.values()) - } -} - -impl<'a> TryFrom<&'a ArrowData> for Vec { - type Error = eyre::Report; - fn try_from(value: &'a ArrowData) -> Result { - value.try_into().map(|slice: &'a [f64]| slice.to_vec()) - } -} - impl<'a> TryFrom<&'a ArrowData> for String { type Error = eyre::Report; fn try_from(value: &'a ArrowData) -> Result { diff --git a/libraries/arrow-convert/src/into_impls.rs b/libraries/arrow-convert/src/into_impls.rs index 8d86e444..e3f7473a 100644 --- a/libraries/arrow-convert/src/into_impls.rs +++ b/libraries/arrow-convert/src/into_impls.rs @@ -26,19 +26,6 @@ macro_rules! impl_into_arrow_for_uint { }; } -macro_rules! impl_into_arrow_for_vec { - ($($t:ty => $arrow_type:ty),*) => { - $( - impl IntoArrow for Vec<$t> { - type A = PrimitiveArray<$arrow_type>; - fn into_arrow(self) -> Self::A { - self.into() - } - } - )* - }; -} - impl_into_arrow_for_uint!( u8 => UInt8Type, u16 => UInt16Type, @@ -52,6 +39,19 @@ impl_into_arrow_for_uint!( f64 => Float64Type ); +macro_rules! impl_into_arrow_for_vec { + ($($t:ty => $arrow_type:ty),*) => { + $( + impl IntoArrow for Vec<$t> { + type A = PrimitiveArray<$arrow_type>; + fn into_arrow(self) -> Self::A { + self.into() + } + } + )* + }; +} + impl_into_arrow_for_vec!( u8 => UInt8Type, u16 => UInt16Type, From ee5426565b7693051035f67e904fd0e98397d4b8 Mon Sep 17 00:00:00 2001 From: Shar-jeel-Sajid Date: Mon, 17 Mar 2025 16:46:48 +0500 Subject: [PATCH 3/4] Combining macros --- libraries/arrow-convert/src/from_impls.rs | 47 ++++++----------------- libraries/arrow-convert/src/into_impls.rs | 21 +--------- 2 files changed, 14 insertions(+), 54 deletions(-) diff --git a/libraries/arrow-convert/src/from_impls.rs b/libraries/arrow-convert/src/from_impls.rs index 08c77fad..dbc743fa 100644 --- a/libraries/arrow-convert/src/from_impls.rs +++ b/libraries/arrow-convert/src/from_impls.rs @@ -51,24 +51,7 @@ macro_rules! impl_try_from_arrow_data { } } )* - }; -} -impl_try_from_arrow_data!( - u8 => UInt8Type, - u16 => UInt16Type, - u32 => UInt32Type, - u64 => UInt64Type, - i8 => Int8Type, - i16 =>Int16Type, - i32 => Int32Type, - i64 => Int64Type, - f32 => Float32Type, - f64 => Float64Type -); - -macro_rules! impl_try_from_arrow_data_for_slice { - ($($t:ty => $arrow_type:ident),*) => { $( impl<'a> TryFrom<&'a ArrowData> for &'a [$t] { type Error = eyre::Report; @@ -84,24 +67,7 @@ macro_rules! impl_try_from_arrow_data_for_slice { } } )* - }; -} - -impl_try_from_arrow_data_for_slice!( - u8 => UInt8Type, - u16 => UInt16Type, - u32 => UInt32Type, - u64 => UInt64Type, - i8 => Int8Type, - i16 =>Int16Type, - i32 => Int32Type, - i64 => Int64Type, - f32 => Float32Type, - f64 => Float64Type -); -macro_rules! impl_try_from_arrow_data_for_vec { - ($($t:ty),*) => { $( impl<'a> TryFrom<&'a ArrowData> for Vec<$t> { type Error = eyre::Report; @@ -116,7 +82,18 @@ macro_rules! impl_try_from_arrow_data_for_vec { }; } -impl_try_from_arrow_data_for_vec!(u8, u16, u32, u64, i8, i16, i32, i64, f32, f64); +impl_try_from_arrow_data!( + u8 => UInt8Type, + u16 => UInt16Type, + u32 => UInt32Type, + u64 => UInt64Type, + i8 => Int8Type, + i16 =>Int16Type, + i32 => Int32Type, + i64 => Int64Type, + f32 => Float32Type, + f64 => Float64Type +); impl<'a> TryFrom<&'a ArrowData> for &'a str { type Error = eyre::Report; diff --git a/libraries/arrow-convert/src/into_impls.rs b/libraries/arrow-convert/src/into_impls.rs index e3f7473a..f2a05fae 100644 --- a/libraries/arrow-convert/src/into_impls.rs +++ b/libraries/arrow-convert/src/into_impls.rs @@ -13,7 +13,7 @@ impl IntoArrow for bool { } } -macro_rules! impl_into_arrow_for_uint { +macro_rules! impl_into_arrow { ($($t:ty => $arrow_type:ty),*) => { $( impl IntoArrow for $t { @@ -23,24 +23,7 @@ macro_rules! impl_into_arrow_for_uint { } } )* - }; -} -impl_into_arrow_for_uint!( - u8 => UInt8Type, - u16 => UInt16Type, - u32 => UInt32Type, - u64 => UInt64Type, - i8 => Int8Type, - i16 => Int16Type, - i32 => Int32Type, - i64 => Int64Type, - f32 => Float32Type, - f64 => Float64Type -); - -macro_rules! impl_into_arrow_for_vec { - ($($t:ty => $arrow_type:ty),*) => { $( impl IntoArrow for Vec<$t> { type A = PrimitiveArray<$arrow_type>; @@ -52,7 +35,7 @@ macro_rules! impl_into_arrow_for_vec { }; } -impl_into_arrow_for_vec!( +impl_into_arrow!( u8 => UInt8Type, u16 => UInt16Type, u32 => UInt32Type, From 524eaa0dec1c719cc6e55d0287e0203c0353252d Mon Sep 17 00:00:00 2001 From: Shar-jeel-Sajid <135236134+Shar-jeel-Sajid@users.noreply.github.com> Date: Mon, 17 Mar 2025 18:54:08 +0500 Subject: [PATCH 4/4] Very small typo Co-authored-by: Haixuan Xavier Tao --- libraries/arrow-convert/src/from_impls.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/arrow-convert/src/from_impls.rs b/libraries/arrow-convert/src/from_impls.rs index dbc743fa..f6ccb7e2 100644 --- a/libraries/arrow-convert/src/from_impls.rs +++ b/libraries/arrow-convert/src/from_impls.rs @@ -88,7 +88,7 @@ impl_try_from_arrow_data!( u32 => UInt32Type, u64 => UInt64Type, i8 => Int8Type, - i16 =>Int16Type, + i16 => Int16Type, i32 => Int32Type, i64 => Int64Type, f32 => Float32Type,