|
|
|
@@ -122,16 +122,6 @@ Status ClueOp::Reset() { |
|
|
|
return Status::OK(); |
|
|
|
} |
|
|
|
|
|
|
|
Status ClueOp::LoadTensor(const std::string &line, std::unique_ptr<TensorQTable> *tensor_table, int64_t row) { |
|
|
|
TensorRow tRow(1, nullptr); |
|
|
|
(*tensor_table)->push_back(std::move(tRow)); |
|
|
|
|
|
|
|
std::shared_ptr<Tensor> tensor; |
|
|
|
RETURN_IF_NOT_OK(Tensor::CreateScalar(line, &tensor)); |
|
|
|
(**tensor_table)[row][0] = std::move(tensor); |
|
|
|
return Status::OK(); |
|
|
|
} |
|
|
|
|
|
|
|
Status ClueOp::GetValue(const nlohmann::json &js, std::vector<std::string> key_chain, std::shared_ptr<Tensor> *t) { |
|
|
|
nlohmann::json cursor = js; |
|
|
|
for (int i = 0; i < key_chain.size(); i++) { |
|
|
|
@@ -191,25 +181,24 @@ Status ClueOp::LoadFile(const std::string &file, const int64_t start_offset, con |
|
|
|
continue; |
|
|
|
} |
|
|
|
|
|
|
|
nlohmann::json js; |
|
|
|
try { |
|
|
|
nlohmann::json js = nlohmann::json::parse(line); |
|
|
|
int cols_count = cols_to_keyword_.size(); |
|
|
|
TensorRow tRow(cols_count, nullptr); |
|
|
|
tensor_table->push_back(std::move(tRow)); |
|
|
|
|
|
|
|
int cout = 0; |
|
|
|
for (auto &p : cols_to_keyword_) { |
|
|
|
std::shared_ptr<Tensor> tensor; |
|
|
|
RETURN_IF_NOT_OK(GetValue(js, p.second, &tensor)); |
|
|
|
(*tensor_table)[rows_each_buffer][cout] = std::move(tensor); |
|
|
|
cout++; |
|
|
|
} |
|
|
|
js = nlohmann::json::parse(line); |
|
|
|
} catch (const std::exception &err) { |
|
|
|
// Catch any exception and convert to Status return code |
|
|
|
RETURN_STATUS_UNEXPECTED("Failed to load json file"); |
|
|
|
} |
|
|
|
int cols_count = cols_to_keyword_.size(); |
|
|
|
TensorRow tRow(cols_count, nullptr); |
|
|
|
tensor_table->push_back(std::move(tRow)); |
|
|
|
int cout = 0; |
|
|
|
for (auto &p : cols_to_keyword_) { |
|
|
|
std::shared_ptr<Tensor> tensor; |
|
|
|
RETURN_IF_NOT_OK(GetValue(js, p.second, &tensor)); |
|
|
|
(*tensor_table)[rows_each_buffer][cout] = std::move(tensor); |
|
|
|
cout++; |
|
|
|
} |
|
|
|
|
|
|
|
// RETURN_IF_NOT_OK(LoadTensor(line, &tensor_table, rows_each_buffer)); |
|
|
|
rows_each_buffer++; |
|
|
|
rows_total++; |
|
|
|
if (rows_each_buffer == rows_per_buffer_) { |
|
|
|
|