!174 enhance: the error info is not detail when the column list is invalid by MindDataset

Merge pull request !174 from guozhijian/fix_false_error_info

mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.cc

@@ -103,7 +103,8 @@ Status MindRecordOp::Init() {
   shard_reader_ = mindspore::make_unique<ShardReader>();
   auto rc = shard_reader_->Open(dataset_file_, num_mind_record_workers_, columns_to_load_, operators_, block_reader_);
 
-  CHECK_FAIL_RETURN_UNEXPECTED(rc != MSRStatus::FAILED, "MindRecordOp init failed.");
+  CHECK_FAIL_RETURN_UNEXPECTED(rc != MSRStatus::FAILED,
+                               "MindRecordOp init failed. Error message: " + ErrnoToMessage(rc));
 
   data_schema_ = mindspore::make_unique<DataSchema>();
 

mindspore/ccsrc/dataset/engine/datasetops/source/mindrecord_op.h

@@ -32,6 +32,7 @@
 #include "dataset/engine/datasetops/source/io_block.h"
 #include "dataset/util/queue.h"
 #include "dataset/util/status.h"
+#include "mindrecord/include/shard_error.h"
 #include "mindrecord/include/shard_reader.h"
 #include "mindrecord/include/common/shard_utils.h"
 #include "dataset/util/wait_post.h"

mindspore/ccsrc/mindrecord/common/shard_error.cc

@@ -0,0 +1,178 @@
+/**
+ * Copyright 2019 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "mindrecord/include/shard_error.h"
+
+namespace mindspore {
+namespace mindrecord {
+std::string ErrnoToMessage(MSRStatus status) {
+  switch (status) {
+    case FAILED:
+      return "operator failed";
+      break;
+    case SUCCESS:
+      return "operator success";
+      break;
+    case OPEN_FILE_FAILED:
+      return "open file failed";
+      break;
+    case CLOSE_FILE_FAILED:
+      return "close file failed";
+      break;
+    case WRITE_METADATA_FAILED:
+      return "write metadata failed";
+      break;
+    case WRITE_RAWDATA_FAILED:
+      return "write rawdata failed";
+      break;
+    case GET_SCHEMA_FAILED:
+      return "get schema failed";
+      break;
+    case ILLEGAL_RAWDATA:
+      return "illegal raw data";
+      break;
+    case PYTHON_TO_JSON_FAILED:
+      return "pybind: python object to json failed";
+      break;
+    case DIR_CREATE_FAILED:
+      return "directory create failed";
+      break;
+    case OPEN_DIR_FAILED:
+      return "open directory failed";
+      break;
+    case INVALID_STATISTICS:
+      return "invalid statistics object";
+      break;
+    case OPEN_DATABASE_FAILED:
+      return "open database failed";
+      break;
+    case CLOSE_DATABASE_FAILED:
+      return "close database failed";
+      break;
+    case DATABASE_OPERATE_FAILED:
+      return "database operate failed";
+      break;
+    case BUILD_SCHEMA_FAILED:
+      return "build schema failed";
+      break;
+    case DIVISOR_IS_ILLEGAL:
+      return "divisor is illegal";
+      break;
+    case INVALID_FILE_PATH:
+      return "file path is invalid";
+      break;
+    case SECURE_FUNC_FAILED:
+      return "secure function failed";
+      break;
+    case ALLOCATE_MEM_FAILED:
+      return "allocate memory failed";
+      break;
+    case ILLEGAL_FIELD_NAME:
+      return "illegal field name";
+      break;
+    case ILLEGAL_FIELD_TYPE:
+      return "illegal field type";
+      break;
+    case SET_METADATA_FAILED:
+      return "set metadata failed";
+      break;
+    case ILLEGAL_SCHEMA_DEFINITION:
+      return "illegal schema definition";
+      break;
+    case ILLEGAL_COLUMN_LIST:
+      return "illegal column list";
+      break;
+    case SQL_ERROR:
+      return "sql error";
+      break;
+    case ILLEGAL_SHARD_COUNT:
+      return "illegal shard count";
+      break;
+    case ILLEGAL_SCHEMA_COUNT:
+      return "illegal schema count";
+      break;
+    case VERSION_ERROR:
+      return "data version is not matched";
+      break;
+    case ADD_SCHEMA_FAILED:
+      return "add schema failed";
+      break;
+    case ILLEGAL_Header_SIZE:
+      return "illegal header size";
+      break;
+    case ILLEGAL_Page_SIZE:
+      return "illegal page size";
+      break;
+    case ILLEGAL_SIZE_VALUE:
+      return "illegal size value";
+      break;
+    case INDEX_FIELD_ERROR:
+      return "add index fields failed";
+      break;
+    case GET_CANDIDATE_CATEGORYFIELDS_FAILED:
+      return "get candidate category fields failed";
+      break;
+    case GET_CATEGORY_INFO_FAILED:
+      return "get category information failed";
+      break;
+    case ILLEGAL_CATEGORY_ID:
+      return "illegal category id";
+      break;
+    case ILLEGAL_ROWNUMBER_OF_PAGE:
+      return "illegal row number of page";
+      break;
+    case ILLEGAL_SCHEMA_ID:
+      return "illegal schema id";
+      break;
+    case DESERIALIZE_SCHEMA_FAILED:
+      return "deserialize schema failed";
+      break;
+    case DESERIALIZE_STATISTICS_FAILED:
+      return "deserialize statistics failed";
+      break;
+    case ILLEGAL_DB_FILE:
+      return "illegal db file";
+      break;
+    case OVERWRITE_DB_FILE:
+      return "overwrite db file";
+      break;
+    case OVERWRITE_MINDRECORD_FILE:
+      return "overwrite mindrecord file";
+      break;
+    case ILLEGAL_MINDRECORD_FILE:
+      return "illegal mindrecord file";
+      break;
+    case PARSE_JSON_FAILED:
+      return "parse json failed";
+      break;
+    case ILLEGAL_PARAMETERS:
+      return "illegal parameters";
+      break;
+    case GET_PAGE_BY_GROUP_ID_FAILED:
+      return "get page by group id failed";
+      break;
+    case GET_SYSTEM_STATE_FAILED:
+      return "get system state failed";
+      break;
+    case IO_FAILED:
+      return "io operate failed";
+      break;
+    default:
+      return "invalid error no";
+  }
+}
+}  // namespace mindrecord
+}  // namespace mindspore

mindspore/ccsrc/mindrecord/include/shard_error.h

@@ -18,63 +18,65 @@
 #define MINDRECORD_INCLUDE_SHARD_ERROR_H_
 
 #include <map>
-#include "utils/error_code.h"
+#include <string>
 
 namespace mindspore {
 namespace mindrecord {
-DE_ERRORNO_MINDRECORD(OPEN_FILE_FAILED, 0, "open file failed");
-DE_ERRORNO_MINDRECORD(CLOSE_FILE_FAILED, 1, "close file failed");
-DE_ERRORNO_MINDRECORD(WRITE_METADATA_FAILED, 2, "write metadata failed");
-DE_ERRORNO_MINDRECORD(WRITE_RAWDATA_FAILED, 3, "write rawdata failed");
-DE_ERRORNO_MINDRECORD(GET_SCHEMA_FAILED, 4, "get schema failed");
-DE_ERRORNO_MINDRECORD(ILLEGAL_RAWDATA, 5, "illegal raw data");
-DE_ERRORNO_MINDRECORD(PYTHON_TO_JSON_FAILED, 6, "pybind: python object to json failed");
-DE_ERRORNO_MINDRECORD(DIR_CREATE_FAILED, 7, "directory create failed");
-DE_ERRORNO_MINDRECORD(OPEN_DIR_FAILED, 8, "open directory failed");
-DE_ERRORNO_MINDRECORD(INVALID_STATISTICS, 9, "invalid statistics object");
-DE_ERRORNO_MINDRECORD(OPEN_DATABASE_FAILED, 10, "open database failed");
-DE_ERRORNO_MINDRECORD(CLOSE_DATABASE_FAILED, 11, "close database failed");
-DE_ERRORNO_MINDRECORD(DATABASE_OPERATE_FAILED, 12, "database operate failed");
-DE_ERRORNO_MINDRECORD(BUILD_SCHEMA_FAILED, 13, "build schema failed");
-DE_ERRORNO_MINDRECORD(DIVISOR_IS_ILLEGAL, 14, "divisor is illegal");
-DE_ERRORNO_MINDRECORD(INVALID_FILE_PATH, 15, "file path is invalid");
-DE_ERRORNO_MINDRECORD(SECURE_FUNC_FAILED, 16, "secure function failed");
-DE_ERRORNO_MINDRECORD(ALLOCATE_MEM_FAILED, 17, "allocate memory failed");
-DE_ERRORNO_MINDRECORD(ILLEGAL_FIELD_NAME, 18, "illegal field name");
-DE_ERRORNO_MINDRECORD(ILLEGAL_FIELD_TYPE, 19, "illegal field type");
-DE_ERRORNO_MINDRECORD(SET_METADATA_FAILED, 20, "set metadata failed");
-DE_ERRORNO_MINDRECORD(ILLEGAL_SCHEMA_DEFINITION, 21, "illegal schema definition");
-DE_ERRORNO_MINDRECORD(ILLEGAL_COLUMN_LIST, 22, "illegal column list");
-DE_ERRORNO_MINDRECORD(SQL_ERROR, 23, "sql error");
-DE_ERRORNO_MINDRECORD(ILLEGAL_SHARD_COUNT, 24, "illegal shard count");
-DE_ERRORNO_MINDRECORD(ILLEGAL_SCHEMA_COUNT, 25, "illegal schema count");
-DE_ERRORNO_MINDRECORD(VERSION_ERROR, 26, "data version is not matched");
-DE_ERRORNO_MINDRECORD(ADD_SCHEMA_FAILED, 27, "add schema failed");
-DE_ERRORNO_MINDRECORD(ILLEGAL_Header_SIZE, 28, "illegal header size");
-DE_ERRORNO_MINDRECORD(ILLEGAL_Page_SIZE, 29, "illegal page size");
-DE_ERRORNO_MINDRECORD(ILLEGAL_SIZE_VALUE, 30, "illegal size value");
-DE_ERRORNO_MINDRECORD(INDEX_FIELD_FAILED, 31, "add index fields failed");
-DE_ERRORNO_MINDRECORD(GET_CANDIDATE_CATEGORYFIELDS_FAILED, 32, "get candidate categoryFields failed");
-DE_ERRORNO_MINDRECORD(GET_CATEGORY_INFO, 33, "get category information failed");
-DE_ERRORNO_MINDRECORD(ILLEGAL_CATEGORY_ID, 34, "illegal category id");
-DE_ERRORNO_MINDRECORD(ILLEGAL_ROWNUMBER_OF_PAGE, 35, "illegal row number of page");
-DE_ERRORNO_MINDRECORD(ILLEGAL_SCHEMA_ID, 36, "illegal schema id");
-DE_ERRORNO_MINDRECORD(DESERIALIZE_SCHEMA_FAILED, 37, "deserialize schema failed");
-DE_ERRORNO_MINDRECORD(DESERIALIZE_STATISTICS_FAILED, 38, "deserialize statistics failed");
-DE_ERRORNO_MINDRECORD(ILLEGAL_DB_FILE, 39, "illegal db file.");
-DE_ERRORNO_MINDRECORD(OVERWRITE_DB_FILE, 40, "overwrite db file.");
-DE_ERRORNO_MINDRECORD(OVERWRITE_MINDRECORD_FILE, 41, "overwrite mindrecord file.");
-DE_ERRORNO_MINDRECORD(ILLEGAL_MINDRECORD_FILE, 42, "illegal mindrecord file.");
-DE_ERRORNO_MINDRECORD(PARSE_JSON_FAILED, 43, "parse json failed.");
-DE_ERRORNO_MINDRECORD(ILLEGAL_PARAMETERS, 44, "illegal parameters.");
-DE_ERRORNO_MINDRECORD(GET_PAGE_BY_GROUP_ID_FAILED, 46, "get page by group id failed.");
-DE_ERRORNO_MINDRECORD(GET_SYSTEM_STATE_FAILED, 47, "get system state failed.");
-DE_ERRORNO_MINDRECORD(IO_FAILED, 48, "io operate failed.");
-
 enum MSRStatus {
   SUCCESS = 0,
   FAILED = 1,
+  OPEN_FILE_FAILED,
+  CLOSE_FILE_FAILED,
+  WRITE_METADATA_FAILED,
+  WRITE_RAWDATA_FAILED,
+  GET_SCHEMA_FAILED,
+  ILLEGAL_RAWDATA,
+  PYTHON_TO_JSON_FAILED,
+  DIR_CREATE_FAILED,
+  OPEN_DIR_FAILED,
+  INVALID_STATISTICS,
+  OPEN_DATABASE_FAILED,
+  CLOSE_DATABASE_FAILED,
+  DATABASE_OPERATE_FAILED,
+  BUILD_SCHEMA_FAILED,
+  DIVISOR_IS_ILLEGAL,
+  INVALID_FILE_PATH,
+  SECURE_FUNC_FAILED,
+  ALLOCATE_MEM_FAILED,
+  ILLEGAL_FIELD_NAME,
+  ILLEGAL_FIELD_TYPE,
+  SET_METADATA_FAILED,
+  ILLEGAL_SCHEMA_DEFINITION,
+  ILLEGAL_COLUMN_LIST,
+  SQL_ERROR,
+  ILLEGAL_SHARD_COUNT,
+  ILLEGAL_SCHEMA_COUNT,
+  VERSION_ERROR,
+  ADD_SCHEMA_FAILED,
+  ILLEGAL_Header_SIZE,
+  ILLEGAL_Page_SIZE,
+  ILLEGAL_SIZE_VALUE,
+  INDEX_FIELD_ERROR,
+  GET_CANDIDATE_CATEGORYFIELDS_FAILED,
+  GET_CATEGORY_INFO_FAILED,
+  ILLEGAL_CATEGORY_ID,
+  ILLEGAL_ROWNUMBER_OF_PAGE,
+  ILLEGAL_SCHEMA_ID,
+  DESERIALIZE_SCHEMA_FAILED,
+  DESERIALIZE_STATISTICS_FAILED,
+  ILLEGAL_DB_FILE,
+  OVERWRITE_DB_FILE,
+  OVERWRITE_MINDRECORD_FILE,
+  ILLEGAL_MINDRECORD_FILE,
+  PARSE_JSON_FAILED,
+  ILLEGAL_PARAMETERS,
+  GET_PAGE_BY_GROUP_ID_FAILED,
+  GET_SYSTEM_STATE_FAILED,
+  IO_FAILED
 };
+
+// convert error no to string message
+std::string ErrnoToMessage(MSRStatus status);
 }  // namespace mindrecord
 }  // namespace mindspore
 

mindspore/ccsrc/mindrecord/io/shard_reader.cc

@@ -676,7 +676,7 @@ MSRStatus ShardReader::Open(const std::string &file_path, int n_consumer,
 
   if (CheckColumnList(selected_columns_) == FAILED) {
     MS_LOG(ERROR) << "Illegal column list";
-    return FAILED;
+    return ILLEGAL_COLUMN_LIST;
   }
 
   // Initialize argument

tests/ut/cpp/dataset/mind_record_op_test.cc

@@ -21,6 +21,7 @@
 #include "common/utils.h"
 #include "gtest/gtest.h"
 #include "mindrecord/include/shard_category.h"
+#include "mindrecord/include/shard_error.h"
 #include "mindrecord/include/shard_sample.h"
 #include "mindrecord/include/shard_shuffle.h"
 #include "utils/log_adapter.h"
@@ -479,3 +480,38 @@ TEST_F(MindDataTestMindRecordOp, TestMindRecordBlockReaderRepeat) {
     row_count++;
   }
 }
+
+TEST_F(MindDataTestMindRecordOp, TestMindRecordInvalidColumnList) {
+  // single MindRecord op and nothing else
+  //
+  //    MindRecordOp
+
+  MS_LOG(INFO) << "UT test TestMindRecordInvalidColumnList";
+
+  Status rc;
+
+  // Start with an empty execution tree
+  auto my_tree = std::make_shared<ExecutionTree>();
+
+  // Test info:
+  // Dataset from testDataset1 has 10 rows, 2 columns.
+  // RowsPerBuffer buffer setting of 3 yields 4 buffers with the last buffer having single row
+  // only.  2 workers.
+  // Test a column selection instead of all columns as well.
+
+  std::vector<std::string> column_list;
+  std::string label_col_name("file_name_2");
+  column_list.push_back(label_col_name);
+  label_col_name = "label";
+  column_list.push_back(label_col_name);
+
+  std::shared_ptr<MindRecordOp> my_mindrecord_op;
+  MindRecordOp::Builder builder;
+  builder.SetDatasetFile(mindrecord_root_path_ + "/testMindDataSet/testImageNetData/imagenet.mindrecord0")
+      .SetRowsPerBuffer(3)
+      .SetNumMindRecordWorkers(4)
+      .SetColumnsToLoad(column_list);
+  rc = builder.Build(&my_mindrecord_op);
+  ASSERT_TRUE(rc.IsError());
+  ASSERT_TRUE(rc.ToString().find_first_of("illegal column list") != std::string::npos);
+}

tests/ut/cpp/mindrecord/ut_shard_reader_test.cc

@@ -155,7 +155,7 @@ TEST_F(TestShardReader, TestShardReaderColumnNotInSchema) {
   auto column_list = std::vector<std::string>{"file_namex"};
   ShardReader dataset;
   MSRStatus ret = dataset.Open(file_name, 4, column_list);
-  ASSERT_EQ(ret, FAILED);
+  ASSERT_EQ(ret, ILLEGAL_COLUMN_LIST);
 }
 
 TEST_F(TestShardReader, TestShardVersion) {