Browse Source

move GetValue out of try-catch

tags/v1.0.0
jiangzhiwen 5 years ago
parent
commit
767f97cdb3
2 changed files with 12 additions and 30 deletions
  1. +12
    -23
      mindspore/ccsrc/minddata/dataset/engine/datasetops/source/clue_op.cc
  2. +0
    -7
      mindspore/ccsrc/minddata/dataset/engine/datasetops/source/clue_op.h

+ 12
- 23
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/clue_op.cc View File

@@ -122,16 +122,6 @@ Status ClueOp::Reset() {
return Status::OK();
}

Status ClueOp::LoadTensor(const std::string &line, std::unique_ptr<TensorQTable> *tensor_table, int64_t row) {
TensorRow tRow(1, nullptr);
(*tensor_table)->push_back(std::move(tRow));

std::shared_ptr<Tensor> tensor;
RETURN_IF_NOT_OK(Tensor::CreateScalar(line, &tensor));
(**tensor_table)[row][0] = std::move(tensor);
return Status::OK();
}

Status ClueOp::GetValue(const nlohmann::json &js, std::vector<std::string> key_chain, std::shared_ptr<Tensor> *t) {
nlohmann::json cursor = js;
for (int i = 0; i < key_chain.size(); i++) {
@@ -191,25 +181,24 @@ Status ClueOp::LoadFile(const std::string &file, const int64_t start_offset, con
continue;
}

nlohmann::json js;
try {
nlohmann::json js = nlohmann::json::parse(line);
int cols_count = cols_to_keyword_.size();
TensorRow tRow(cols_count, nullptr);
tensor_table->push_back(std::move(tRow));

int cout = 0;
for (auto &p : cols_to_keyword_) {
std::shared_ptr<Tensor> tensor;
RETURN_IF_NOT_OK(GetValue(js, p.second, &tensor));
(*tensor_table)[rows_each_buffer][cout] = std::move(tensor);
cout++;
}
js = nlohmann::json::parse(line);
} catch (const std::exception &err) {
// Catch any exception and convert to Status return code
RETURN_STATUS_UNEXPECTED("Failed to load json file");
}
int cols_count = cols_to_keyword_.size();
TensorRow tRow(cols_count, nullptr);
tensor_table->push_back(std::move(tRow));
int cout = 0;
for (auto &p : cols_to_keyword_) {
std::shared_ptr<Tensor> tensor;
RETURN_IF_NOT_OK(GetValue(js, p.second, &tensor));
(*tensor_table)[rows_each_buffer][cout] = std::move(tensor);
cout++;
}

// RETURN_IF_NOT_OK(LoadTensor(line, &tensor_table, rows_each_buffer));
rows_each_buffer++;
rows_total++;
if (rows_each_buffer == rows_per_buffer_) {


+ 0
- 7
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/clue_op.h View File

@@ -179,13 +179,6 @@ class ClueOp : public ParallelOp {
// @return Status - the error code returned.
Status WorkerEntry(int32_t worker_id) override;

// Parses a single row and puts the data into a tensor table.
// @param line - the content of the row.
// @param tensor_table - the tensor table to put the parsed data in.
// @param row - the id of the row filled in the tensor table.
// @return Status - the error code returned.
Status LoadTensor(const std::string &line, std::unique_ptr<TensorQTable> *tensor_table, int64_t row);

// Reads a clue file and loads the data into multiple buffers.
// @param file - the file to read.
// @param start_offset - the start offset of file.


Loading…
Cancel
Save