From c36a36304c563812a73fe9ccb37bca90b246a6d6 Mon Sep 17 00:00:00 2001 From: YangLuo Date: Tue, 29 Sep 2020 14:29:33 +0800 Subject: [PATCH] [MD] sampler log update --- .../source/sampler/distributed_sampler.cc | 5 ++++- .../source/sampler/python_sampler.cc | 6 +++-- .../source/sampler/subset_random_sampler.cc | 5 ++--- .../source/sampler/weighted_random_sampler.cc | 22 ++++++++++++++----- 4 files changed, 26 insertions(+), 12 deletions(-) diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/distributed_sampler.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/distributed_sampler.cc index 4e63ce987d..979da9e8b6 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/distributed_sampler.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/distributed_sampler.cc @@ -76,7 +76,10 @@ Status DistributedSampler::InitSampler() { Status DistributedSampler::GetNextSample(std::unique_ptr *out_buffer) { if (cnt_ > samples_per_buffer_) { - RETURN_STATUS_UNEXPECTED("Distributed Sampler Error"); + RETURN_STATUS_UNEXPECTED( + "Number of samples(cnt) that have already been filled in to buffer should be less than or " + "equal to samples_per_buffer, but got cnt: " + + std::to_string(cnt_) + ", samples_per_buffer: " + std::to_string(samples_per_buffer_)); } else if (cnt_ == samples_per_buffer_ && (non_empty_ || !even_dist_)) { (*out_buffer) = std::make_unique(0, DataBuffer::kDeBFlagEOE); if (!samples_per_buffer_) { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/python_sampler.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/python_sampler.cc index a501a2dcb0..8f920e60fd 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/python_sampler.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/python_sampler.cc @@ -53,7 +53,8 @@ Status PythonSampler::GetNextSample(std::unique_ptr *out_buffer) { } catch (const py::error_already_set &e) { return Status(StatusCode::kPyFuncException, e.what()); } catch (const py::cast_error &e) { - return Status(StatusCode::kPyFuncException, "Python Sampler iterator should return integer index"); + return Status(StatusCode::kPyFuncException, + "Invalid data, python sampler iterator should return an integer index."); } } TensorRow row(1, sample_ids); @@ -64,7 +65,8 @@ Status PythonSampler::GetNextSample(std::unique_ptr *out_buffer) { } Status PythonSampler::InitSampler() { - CHECK_FAIL_RETURN_UNEXPECTED(num_rows_ > 0, "ERROR num_rows_ should be greater than 0"); + CHECK_FAIL_RETURN_UNEXPECTED( + num_rows_ > 0, "Invalid parameter, num_rows must be greater than 0, but got " + std::to_string(num_rows_)); // Special value of 0 for num_samples means that the user wants to sample the entire set of data. // If the user asked to sample more rows than exists in the dataset, adjust the num_samples accordingly. if (num_samples_ == 0 || num_samples_ > num_rows_) { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/subset_random_sampler.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/subset_random_sampler.cc index 3a9a0b418e..0d58f23d5e 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/subset_random_sampler.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/subset_random_sampler.cc @@ -98,9 +98,8 @@ Status SubsetRandomSampler::GetNextSample(std::unique_ptr *out_buffe auto id_ptr = outputIds->begin(); while (sample_id_ < last_id) { if (indices_[sample_id_] >= num_rows_) { - std::string err_msg = - "Generated id is bigger than numRows (out of bound). indices_: " + std::to_string(indices_[sample_id_]) + - " num_rows_: " + std::to_string(num_rows_); + std::string err_msg = "Generated indice is out of bound, expect range [0, num_data-1], got indice: " + + std::to_string(indices_[sample_id_]) + ", num_data: " + std::to_string(num_rows_ - 1); RETURN_STATUS_UNEXPECTED(err_msg); } diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/weighted_random_sampler.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/weighted_random_sampler.cc index 98aec562a9..0a2a3f9925 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/weighted_random_sampler.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/weighted_random_sampler.cc @@ -50,11 +50,15 @@ Status WeightedRandomSampler::InitSampler() { std::to_string(samples_per_buffer_) + ".\n"); if (weights_.size() > static_cast(num_rows_)) { return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, - "Invalid parameter, number of samples weights is more than num of rows. " - "Might generate id out of bound OR other errors"); + "Invalid parameter, size of sample weights must be less than or equal to num of data, " + "otherwise might cause generated id out of bound or other errors, but got weight size: " + + std::to_string(weights_.size()) + ", num of data: " + std::to_string(num_rows_)); } if (!replacement_ && (weights_.size() < static_cast(num_samples_))) { - RETURN_STATUS_UNEXPECTED("Invalid parameter, without replacement, weights size must be greater than num_samples."); + RETURN_STATUS_UNEXPECTED( + "Invalid parameter, without replacement, weights size must be greater than or equal to num_samples, " + "but got weight size: " + + std::to_string(weights_.size()) + ", num_samples: " + std::to_string(num_samples_)); } // Initialize random generator with seed from config manager @@ -110,11 +114,16 @@ Status WeightedRandomSampler::ResetSampler() { Status WeightedRandomSampler::GetNextSample(std::unique_ptr *out_buffer) { if (weights_.size() > static_cast(num_rows_)) { return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, - "number of samples weights is more than num of rows. Might generate id out of bound OR other errors"); + "Invalid parameter, size of sample weights must be less than or equal to num of data, " + "otherwise might cause generated id out of bound or other errors, but got weight size: " + + std::to_string(weights_.size()) + ", num of data: " + std::to_string(num_rows_)); } if (!replacement_ && (weights_.size() < static_cast(num_samples_))) { - RETURN_STATUS_UNEXPECTED("Without replacement, sample weights less than numSamples"); + RETURN_STATUS_UNEXPECTED( + "Invalid parameter, without replacement, weights size must be greater than or equal to num_samples, " + "but got weight size: " + + std::to_string(weights_.size()) + ", num_samples: " + std::to_string(num_samples_)); } if (sample_id_ == num_samples_) { @@ -150,7 +159,8 @@ Status WeightedRandomSampler::GetNextSample(std::unique_ptr *out_buf } if (genId >= num_rows_) { - RETURN_STATUS_UNEXPECTED("generated id is bigger than numRows (out of bound)."); + RETURN_STATUS_UNEXPECTED("Generated indice is out of bound, expect range [0, num_data-1], got indice: " + + std::to_string(genId) + ", num_data: " + std::to_string(num_rows_ - 1)); } if (HasChildSampler()) {