| @@ -33,8 +33,8 @@ CelebAOp::Builder::Builder() : builder_decode_(false), builder_sampler_(nullptr) | |||||
| } | } | ||||
| Status CelebAOp::Builder::Build(std::shared_ptr<CelebAOp> *op) { | Status CelebAOp::Builder::Build(std::shared_ptr<CelebAOp> *op) { | ||||
| MS_LOG(INFO) << "Celeba dataset directory is " << builder_dir_.c_str() << "."; | |||||
| MS_LOG(INFO) << "Celeba dataset type is " << builder_dataset_type_.c_str() << "."; | |||||
| MS_LOG(DEBUG) << "Celeba dataset directory is " << builder_dir_.c_str() << "."; | |||||
| MS_LOG(DEBUG) << "Celeba dataset type is " << builder_dataset_type_.c_str() << "."; | |||||
| RETURN_IF_NOT_OK(SanityCheck()); | RETURN_IF_NOT_OK(SanityCheck()); | ||||
| if (builder_sampler_ == nullptr) { | if (builder_sampler_ == nullptr) { | ||||
| builder_sampler_ = std::make_shared<SequentialSampler>(); | builder_sampler_ = std::make_shared<SequentialSampler>(); | ||||
| @@ -240,9 +240,11 @@ Status CelebAOp::ParseImageAttrInfo() { | |||||
| num_rows_exact_ = image_labels_vec_.size(); | num_rows_exact_ = image_labels_vec_.size(); | ||||
| num_samples_ = (num_samples_ == 0 || num_samples_ > num_rows_exact_) ? num_rows_exact_ : num_samples_; | num_samples_ = (num_samples_ == 0 || num_samples_ > num_rows_exact_) ? num_rows_exact_ : num_samples_; | ||||
| if (num_rows_exact_ == 0) { | if (num_rows_exact_ == 0) { | ||||
| RETURN_STATUS_UNEXPECTED("Number of rows in celeba dataset is zero"); | |||||
| RETURN_STATUS_UNEXPECTED( | |||||
| "There is no valid data matching the dataset API CelebADataset.Please check file path or dataset API " | |||||
| "validation first."); | |||||
| } | } | ||||
| MS_LOG(INFO) << "Celeba dataset rows number is " << num_rows_exact_ << "."; | |||||
| MS_LOG(DEBUG) << "Celeba dataset rows number is " << num_rows_exact_ << "."; | |||||
| return Status::OK(); | return Status::OK(); | ||||
| } | } | ||||
| @@ -267,7 +269,9 @@ std::vector<std::string> CelebAOp::Split(const std::string &line) { | |||||
| // Derived from RandomAccessOp | // Derived from RandomAccessOp | ||||
| Status CelebAOp::GetNumSamples(int64_t *num) const { | Status CelebAOp::GetNumSamples(int64_t *num) const { | ||||
| if (num == nullptr || num_samples_ == 0) { | if (num == nullptr || num_samples_ == 0) { | ||||
| RETURN_STATUS_UNEXPECTED("NumSample not set"); | |||||
| RETURN_STATUS_UNEXPECTED( | |||||
| "There is no valid data matching the dataset API CelebADataset.Please check file path or dataset API " | |||||
| "validation first."); | |||||
| } | } | ||||
| (*num) = num_samples_; | (*num) = num_samples_; | ||||
| return Status::OK(); | return Status::OK(); | ||||
| @@ -275,7 +279,9 @@ Status CelebAOp::GetNumSamples(int64_t *num) const { | |||||
| Status CelebAOp::GetNumRowsInDataset(int64_t *num) const { | Status CelebAOp::GetNumRowsInDataset(int64_t *num) const { | ||||
| if (num == nullptr || num_rows_exact_ == 0) { | if (num == nullptr || num_rows_exact_ == 0) { | ||||
| return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, "NumRow not set"); | |||||
| RETURN_STATUS_UNEXPECTED( | |||||
| "There is no valid data matching the dataset API CelebADataset.Please check file path or dataset API " | |||||
| "validation first."); | |||||
| } | } | ||||
| *num = num_rows_exact_; | *num = num_rows_exact_; | ||||
| @@ -247,7 +247,10 @@ Status CifarOp::InitSampler() { | |||||
| // Derived from RandomAccessOp | // Derived from RandomAccessOp | ||||
| Status CifarOp::GetNumSamples(int64_t *num) const { | Status CifarOp::GetNumSamples(int64_t *num) const { | ||||
| if (num == nullptr || num_rows_ == 0) { | if (num == nullptr || num_rows_ == 0) { | ||||
| RETURN_STATUS_UNEXPECTED("NumRow not set"); | |||||
| std::string api = cifar_type_ == kCifar10 ? "Cifar10Dataset" : "Cifar100Dataset"; | |||||
| std::string err_msg = "There is no valid data matching the dataset API " + api + | |||||
| ".Please check file path or dataset API validation first."; | |||||
| RETURN_STATUS_UNEXPECTED(err_msg); | |||||
| } | } | ||||
| (*num) = num_samples_; | (*num) = num_samples_; | ||||
| return Status::OK(); | return Status::OK(); | ||||
| @@ -256,7 +259,10 @@ Status CifarOp::GetNumSamples(int64_t *num) const { | |||||
| // Derived from RandomAccessOp | // Derived from RandomAccessOp | ||||
| Status CifarOp::GetNumRowsInDataset(int64_t *num) const { | Status CifarOp::GetNumRowsInDataset(int64_t *num) const { | ||||
| if (num == nullptr || num_rows_ == 0) { | if (num == nullptr || num_rows_ == 0) { | ||||
| RETURN_STATUS_UNEXPECTED("NumRow not set"); | |||||
| std::string api = cifar_type_ == kCifar10 ? "Cifar10Dataset" : "Cifar100Dataset"; | |||||
| std::string err_msg = "There is no valid data matching the dataset API " + api + | |||||
| ".Please check file path or dataset API validation first."; | |||||
| RETURN_STATUS_UNEXPECTED(err_msg); | |||||
| } | } | ||||
| (*num) = num_rows_; | (*num) = num_rows_; | ||||
| return Status::OK(); | return Status::OK(); | ||||
| @@ -389,7 +395,10 @@ Status CifarOp::ParseCifarData() { | |||||
| num_rows_ = cifar_image_label_pairs_.size(); | num_rows_ = cifar_image_label_pairs_.size(); | ||||
| num_samples_ = (num_samples_ == 0 || num_samples_ > num_rows_) ? num_rows_ : num_samples_; | num_samples_ = (num_samples_ == 0 || num_samples_ > num_rows_) ? num_rows_ : num_samples_; | ||||
| if (num_rows_ == 0) { | if (num_rows_ == 0) { | ||||
| RETURN_STATUS_UNEXPECTED("Init Cifar failed, not a single row read from dataset!"); | |||||
| std::string api = cifar_type_ == kCifar10 ? "Cifar10Dataset" : "Cifar100Dataset"; | |||||
| std::string err_msg = "There is no valid data matching the dataset API " + api + | |||||
| ".Please check file path or dataset API validation first."; | |||||
| RETURN_STATUS_UNEXPECTED(err_msg); | |||||
| } | } | ||||
| cifar_raw_data_block_->Reset(); | cifar_raw_data_block_->Reset(); | ||||
| return Status::OK(); | return Status::OK(); | ||||
| @@ -265,7 +265,9 @@ Status ImageFolderOp::InitSampler() { | |||||
| // Derived from RandomAccessOp | // Derived from RandomAccessOp | ||||
| Status ImageFolderOp::GetNumSamples(int64_t *num) const { | Status ImageFolderOp::GetNumSamples(int64_t *num) const { | ||||
| if (num == nullptr || num_samples_ == 0) { | if (num == nullptr || num_samples_ == 0) { | ||||
| RETURN_STATUS_UNEXPECTED("NumRow not set"); | |||||
| RETURN_STATUS_UNEXPECTED( | |||||
| "There is no valid data matching the dataset API ImageFolderDatasetV2.Please check file path or dataset API " | |||||
| "validation first."); | |||||
| } | } | ||||
| (*num) = num_samples_; | (*num) = num_samples_; | ||||
| return Status::OK(); | return Status::OK(); | ||||
| @@ -274,7 +276,9 @@ Status ImageFolderOp::GetNumSamples(int64_t *num) const { | |||||
| // Derived from RandomAccessOp | // Derived from RandomAccessOp | ||||
| Status ImageFolderOp::GetNumRowsInDataset(int64_t *num) const { | Status ImageFolderOp::GetNumRowsInDataset(int64_t *num) const { | ||||
| if (num == nullptr || num_rows_ == 0) { | if (num == nullptr || num_rows_ == 0) { | ||||
| RETURN_STATUS_UNEXPECTED("NumRow not set"); | |||||
| RETURN_STATUS_UNEXPECTED( | |||||
| "There is no valid data matching the dataset API ImageFolderDatasetV2.Please check file path or dataset API " | |||||
| "validation first."); | |||||
| } | } | ||||
| (*num) = num_rows_; | (*num) = num_rows_; | ||||
| return Status::OK(); | return Status::OK(); | ||||
| @@ -261,7 +261,9 @@ Status ManifestOp::InitSampler() { | |||||
| // Derived from RandomAccessOp | // Derived from RandomAccessOp | ||||
| Status ManifestOp::GetNumSamples(int64_t *num) const { | Status ManifestOp::GetNumSamples(int64_t *num) const { | ||||
| if (num == nullptr || num_rows_ == 0) { | if (num == nullptr || num_rows_ == 0) { | ||||
| RETURN_STATUS_UNEXPECTED("NumRow not set"); | |||||
| RETURN_STATUS_UNEXPECTED( | |||||
| "There is no valid data matching the dataset API ManifestDataset.Please check file path or dataset API " | |||||
| "validation first."); | |||||
| } | } | ||||
| (*num) = num_samples_; | (*num) = num_samples_; | ||||
| return Status::OK(); | return Status::OK(); | ||||
| @@ -270,7 +272,9 @@ Status ManifestOp::GetNumSamples(int64_t *num) const { | |||||
| // Derived from RandomAccessOp | // Derived from RandomAccessOp | ||||
| Status ManifestOp::GetNumRowsInDataset(int64_t *num) const { | Status ManifestOp::GetNumRowsInDataset(int64_t *num) const { | ||||
| if (num == nullptr || num_rows_ == 0) { | if (num == nullptr || num_rows_ == 0) { | ||||
| RETURN_STATUS_UNEXPECTED("NumRow not set"); | |||||
| RETURN_STATUS_UNEXPECTED( | |||||
| "There is no valid data matching the dataset API ManifestDataset.Please check file path or dataset API " | |||||
| "validation first."); | |||||
| } | } | ||||
| (*num) = num_rows_; | (*num) = num_rows_; | ||||
| return Status::OK(); | return Status::OK(); | ||||
| @@ -279,7 +283,7 @@ Status ManifestOp::GetNumRowsInDataset(int64_t *num) const { | |||||
| // Derived from RandomAccessOp | // Derived from RandomAccessOp | ||||
| Status ManifestOp::GetClassIds(std::map<int32_t, std::vector<int64_t>> *cls_ids) const { | Status ManifestOp::GetClassIds(std::map<int32_t, std::vector<int64_t>> *cls_ids) const { | ||||
| if (cls_ids == nullptr || !cls_ids->empty() || image_labelname_.empty()) { | if (cls_ids == nullptr || !cls_ids->empty() || image_labelname_.empty()) { | ||||
| RETURN_STATUS_UNEXPECTED("Number rows is 0"); | |||||
| RETURN_STATUS_UNEXPECTED("Class indexing is invalid."); | |||||
| } | } | ||||
| for (size_t i = 0; i < image_labelname_.size(); i++) { | for (size_t i = 0; i < image_labelname_.size(); i++) { | ||||
| @@ -395,7 +399,9 @@ Status ManifestOp::CountDatasetInfo() { | |||||
| num_rows_ = static_cast<int64_t>(image_labelname_.size()); | num_rows_ = static_cast<int64_t>(image_labelname_.size()); | ||||
| num_samples_ = (num_samples_ == 0 || num_samples_ > num_rows_) ? num_rows_ : num_samples_; | num_samples_ = (num_samples_ == 0 || num_samples_ > num_rows_) ? num_rows_ : num_samples_; | ||||
| if (num_rows_ == 0) { | if (num_rows_ == 0) { | ||||
| RETURN_STATUS_UNEXPECTED("Number of rows is 0"); | |||||
| RETURN_STATUS_UNEXPECTED( | |||||
| "There is no valid data matching the dataset API ManifestDataset.Please check file path or dataset API " | |||||
| "validation first."); | |||||
| } | } | ||||
| return Status::OK(); | return Status::OK(); | ||||
| } | } | ||||
| @@ -212,7 +212,9 @@ Status MnistOp::InitSampler() { | |||||
| // Derived from RandomAccessOp | // Derived from RandomAccessOp | ||||
| Status MnistOp::GetNumSamples(int64_t *num) const { | Status MnistOp::GetNumSamples(int64_t *num) const { | ||||
| if (num == nullptr || num_rows_ == 0) { | if (num == nullptr || num_rows_ == 0) { | ||||
| RETURN_STATUS_UNEXPECTED("NumRow not set"); | |||||
| RETURN_STATUS_UNEXPECTED( | |||||
| "There is no valid data matching the dataset API MnistDataset.Please check file path or dataset API " | |||||
| "validation first."); | |||||
| } | } | ||||
| (*num) = num_samples_; | (*num) = num_samples_; | ||||
| return Status::OK(); | return Status::OK(); | ||||
| @@ -436,7 +438,9 @@ Status MnistOp::CountTotalRows(const std::string &dir, int64_t numSamples, int64 | |||||
| // Derived from RandomAccessOp | // Derived from RandomAccessOp | ||||
| Status MnistOp::GetNumRowsInDataset(int64_t *num) const { | Status MnistOp::GetNumRowsInDataset(int64_t *num) const { | ||||
| if (num == nullptr || num_rows_ == 0) { | if (num == nullptr || num_rows_ == 0) { | ||||
| RETURN_STATUS_UNEXPECTED("NumRow not set"); | |||||
| RETURN_STATUS_UNEXPECTED( | |||||
| "There is no valid data matching the dataset API MnistDataset.Please check file path or dataset API " | |||||
| "validation first."); | |||||
| } | } | ||||
| (*num) = num_rows_; | (*num) = num_rows_; | ||||
| return Status::OK(); | return Status::OK(); | ||||
| @@ -438,7 +438,9 @@ Status TextFileOp::CalculateNumRowsPerShard() { | |||||
| all_num_rows_ += count; | all_num_rows_ += count; | ||||
| } | } | ||||
| if (all_num_rows_ == 0) { | if (all_num_rows_ == 0) { | ||||
| RETURN_STATUS_UNEXPECTED("Number of rows can not be zero"); | |||||
| RETURN_STATUS_UNEXPECTED( | |||||
| "There is no valid data matching the dataset API TextFileDataset.Please check file path or dataset API " | |||||
| "validation first."); | |||||
| } | } | ||||
| num_rows_per_shard_ = static_cast<int64_t>(std::ceil(all_num_rows_ * 1.0 / num_devices_)); | num_rows_per_shard_ = static_cast<int64_t>(std::ceil(all_num_rows_ * 1.0 / num_devices_)); | ||||
| @@ -198,7 +198,9 @@ Status TFReaderOp::CalculateNumRowsPerShard() { | |||||
| } | } | ||||
| num_rows_per_shard_ = static_cast<int64_t>(std::ceil(num_rows_ * 1.0 / num_devices_)); | num_rows_per_shard_ = static_cast<int64_t>(std::ceil(num_rows_ * 1.0 / num_devices_)); | ||||
| if (num_rows_per_shard_ == 0) { | if (num_rows_per_shard_ == 0) { | ||||
| RETURN_STATUS_UNEXPECTED("Number of rows can not be zero"); | |||||
| RETURN_STATUS_UNEXPECTED( | |||||
| "There is no valid data matching the dataset API TFRecordDataset.Please check file path or dataset API " | |||||
| "validation first."); | |||||
| } | } | ||||
| return Status::OK(); | return Status::OK(); | ||||
| } | } | ||||
| @@ -147,7 +147,9 @@ Status VOCOp::Reset() { | |||||
| Status VOCOp::GetNumSamples(int64_t *num) const { | Status VOCOp::GetNumSamples(int64_t *num) const { | ||||
| if (num == nullptr || num_rows_ == 0) { | if (num == nullptr || num_rows_ == 0) { | ||||
| RETURN_STATUS_UNEXPECTED("NumRow not set"); | |||||
| RETURN_STATUS_UNEXPECTED( | |||||
| "There is no valid data matching the dataset API VOCDataset.Please check file path or dataset API " | |||||
| "validation first."); | |||||
| } | } | ||||
| (*num) = num_samples_; | (*num) = num_samples_; | ||||
| return Status::OK(); | return Status::OK(); | ||||
| @@ -261,7 +263,9 @@ Status VOCOp::ReadImageToTensor(const std::string &path, const ColDescriptor &co | |||||
| // Derived from RandomAccessOp | // Derived from RandomAccessOp | ||||
| Status VOCOp::GetNumRowsInDataset(int64_t *num) const { | Status VOCOp::GetNumRowsInDataset(int64_t *num) const { | ||||
| if (num == nullptr || num_rows_ == 0) { | if (num == nullptr || num_rows_ == 0) { | ||||
| RETURN_STATUS_UNEXPECTED("NumRow not set"); | |||||
| RETURN_STATUS_UNEXPECTED( | |||||
| "There is no valid data matching the dataset API VOCDataset.Please check file path or dataset API " | |||||
| "validation first."); | |||||
| } | } | ||||
| (*num) = num_rows_; | (*num) = num_rows_; | ||||
| return Status::OK(); | return Status::OK(); | ||||