Browse Source

replace const cache line with 128 to match arrow

tags/v0.3.0-rc4
haixuanTao 2 years ago
parent
commit
fe065eec6f
7 changed files with 33 additions and 18 deletions
  1. +5
    -2
      apis/python/operator/src/lib.rs
  2. +2
    -2
      apis/rust/node/src/event_stream/event.rs
  3. +9
    -5
      apis/rust/node/src/node/mod.rs
  4. +2
    -2
      binaries/daemon/src/lib.rs
  5. +6
    -2
      binaries/runtime/src/operator/python.rs
  6. +6
    -2
      binaries/runtime/src/operator/shared_lib.rs
  7. +3
    -3
      libraries/core/src/daemon_messages.rs

+ 5
- 2
apis/python/operator/src/lib.rs View File

@@ -151,7 +151,7 @@ pub fn metadata_to_pydict<'a>(metadata: &'a Metadata, py: Python<'a>) -> &'a PyD
mod tests {
use std::sync::Arc;

use aligned_vec::avec;
use aligned_vec::{avec, AVec, ConstAlign};
use arrow::{
array::{
ArrayData, ArrayRef, BooleanArray, Float64Array, Int32Array, Int64Array, Int8Array,
@@ -169,7 +169,10 @@ mod tests {

fn assert_roundtrip(arrow_array: &ArrayData) -> Result<()> {
let size = required_data_size(arrow_array);
let mut sample = avec![0; size];
let mut sample: AVec<u8, ConstAlign<128>> = AVec::new(128);
for _ in 0..size {
sample.push(0);
}

let info = copy_array_into_sample(&mut sample, arrow_array)?;



+ 2
- 2
apis/rust/node/src/event_stream/event.rs View File

@@ -1,6 +1,6 @@
use std::{ptr::NonNull, sync::Arc};

use aligned_vec::{AVec, ConstAlign, CACHELINE_ALIGN};
use aligned_vec::{AVec, ConstAlign};
use dora_arrow_convert::{ArrowData, IntoArrow};
use dora_core::{
config::{DataId, OperatorId},
@@ -29,7 +29,7 @@ pub enum Event {

pub enum RawData {
Empty,
Vec(AVec<u8, ConstAlign<CACHELINE_ALIGN>>),
Vec(AVec<u8, ConstAlign<128>>),
SharedMemory(SharedMemoryData),
}



+ 9
- 5
apis/rust/node/src/node/mod.rs View File

@@ -5,7 +5,7 @@ use self::{
control_channel::ControlChannel,
drop_stream::DropStream,
};
use aligned_vec::{avec, AVec, ConstAlign, CACHELINE_ALIGN};
use aligned_vec::{AVec, ConstAlign};
use arrow::array::Array;
use dora_core::{
config::{DataId, NodeId, NodeRunConfig},
@@ -257,7 +257,11 @@ impl DoraNode {
len: data_len,
}
} else {
avec![0u8; data_len].into()
let mut avec: AVec<u8, ConstAlign<128>> = AVec::new(128);
for _ in 0..data_len {
avec.push(0);
}
avec.into()
};

Ok(data)
@@ -420,8 +424,8 @@ impl DerefMut for DataSample {
}
}

impl From<AVec<u8, ConstAlign<CACHELINE_ALIGN>>> for DataSample {
fn from(value: AVec<u8, ConstAlign<CACHELINE_ALIGN>>) -> Self {
impl From<AVec<u8, ConstAlign<128>>> for DataSample {
fn from(value: AVec<u8, ConstAlign<128>>) -> Self {
Self {
len: value.len(),
inner: DataSampleInner::Vec(value),
@@ -444,7 +448,7 @@ impl std::fmt::Debug for DataSample {

enum DataSampleInner {
Shmem(ShmemHandle),
Vec(AVec<u8, ConstAlign<CACHELINE_ALIGN>>),
Vec(AVec<u8, ConstAlign<128>>),
}

struct ShmemHandle(Box<Shmem>);


+ 2
- 2
binaries/daemon/src/lib.rs View File

@@ -1,4 +1,4 @@
use aligned_vec::{AVec, ConstAlign, CACHELINE_ALIGN};
use aligned_vec::{AVec, ConstAlign};
use coordinator::CoordinatorEvent;
use dora_core::config::{Input, OperatorId};
use dora_core::coordinator_messages::CoordinatorRequest;
@@ -1073,7 +1073,7 @@ async fn send_output_to_local_receivers(
metadata: &dora_core::message::Metadata,
data: Option<DataMessage>,
clock: &HLC,
) -> Result<Option<AVec<u8, ConstAlign<CACHELINE_ALIGN>>>, eyre::ErrReport> {
) -> Result<Option<AVec<u8, ConstAlign<128>>>, eyre::ErrReport> {
let timestamp = metadata.timestamp();
let empty_set = BTreeSet::new();
let output_id = OutputId(node_id, output_id);


+ 6
- 2
binaries/runtime/src/operator/python.rs View File

@@ -269,7 +269,7 @@ mod callback_impl {
use crate::operator::OperatorEvent;

use super::SendOutputCallback;
use aligned_vec::avec;
use aligned_vec::{AVec, ConstAlign};
use arrow::{array::ArrayData, pyarrow::FromPyArrow};
use dora_core::message::ArrowTypeInfo;
use dora_node_api::{
@@ -312,7 +312,11 @@ mod callback_impl {
.wrap_err("failed to request output sample")?
.wrap_err("failed to allocate output sample")
} else {
Ok(avec![0; data_len].into())
let mut avec: AVec<u8, ConstAlign<128>> = AVec::new(128);
for _ in 0..data_len {
avec.push(0);
}
Ok(avec.into())
}
};



+ 6
- 2
binaries/runtime/src/operator/shared_lib.rs View File

@@ -1,5 +1,5 @@
use super::{OperatorEvent, StopReason};
use aligned_vec::avec;
use aligned_vec::{AVec, ConstAlign};
use dora_core::{
adjust_shared_library_path,
config::{DataId, NodeId, OperatorId},
@@ -134,7 +134,11 @@ impl<'lib> SharedLibraryOperator<'lib> {
};

let total_len = required_data_size(&arrow_array);
let mut sample = avec![0; total_len];
let mut sample: AVec<u8, ConstAlign<128>> = AVec::new(128);
for _ in 0..total_len {
sample.push(0);
}

let type_info = match copy_array_into_sample(&mut sample, &arrow_array) {
Ok(t) => t,
Err(err) => {


+ 3
- 3
libraries/core/src/daemon_messages.rs View File

@@ -9,7 +9,7 @@ use crate::{
config::{DataId, NodeId, NodeRunConfig, OperatorId},
descriptor::{Descriptor, OperatorDefinition, ResolvedNode},
};
use aligned_vec::{AVec, ConstAlign, CACHELINE_ALIGN};
use aligned_vec::{AVec, ConstAlign};
use dora_message::{uhlc, Metadata};
use uuid::Uuid;

@@ -88,7 +88,7 @@ impl DaemonRequest {

#[derive(serde::Serialize, serde::Deserialize, Clone)]
pub enum DataMessage {
Vec(AVec<u8, ConstAlign<CACHELINE_ALIGN>>),
Vec(AVec<u8, ConstAlign<128>>),
SharedMemory {
shared_memory_id: String,
len: usize,
@@ -234,7 +234,7 @@ pub enum InterDaemonEvent {
node_id: NodeId,
output_id: DataId,
metadata: Metadata,
data: Option<AVec<u8, ConstAlign<CACHELINE_ALIGN>>>,
data: Option<AVec<u8, ConstAlign<128>>>,
},
InputsClosed {
dataflow_id: DataflowId,


Loading…
Cancel
Save