!13220 Add data helper to C++ API

From: @ezphlow
Reviewed-by: 
Signed-off-by:

mindspore/ccsrc/minddata/dataset/api/CMakeLists.txt

@@ -29,6 +29,7 @@ endif()
 
 add_library(cpp-API OBJECT
         config.cc
+        data_helper.cc
         datasets.cc
         execute.cc
         iterator.cc

mindspore/ccsrc/minddata/dataset/api/data_helper.cc

@@ -0,0 +1,193 @@
+/**
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "minddata/dataset/include/data_helper.h"
+
+#include <algorithm>
+#include <fstream>
+#include <iostream>
+#include <map>
+#include <memory>
+#include <sstream>
+
+#include "minddata/dataset/util/json_helper.h"
+#include "minddata/dataset/util/log_adapter.h"
+#include "minddata/dataset/util/path.h"
+#include "include/api/status.h"
+
+namespace mindspore {
+namespace dataset {
+// Create a numbered json file from image folder
+Status DataHelper::CreateAlbumIF(const std::vector<char> &in_dir, const std::vector<char> &out_dir) {
+  auto jh = JsonHelper();
+  return jh.CreateAlbum(CharToString(in_dir), CharToString(out_dir));
+}
+
+// A print method typically used for debugging
+void DataHelper::Print(std::ostream &out) const {
+  out << "  Data Helper"
+      << "\n";
+}
+
+Status DataHelper::UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
+                                 const std::vector<std::vector<char>> &value, const std::vector<char> &out_file) {
+  auto jh = JsonHelper();
+  return jh.UpdateArray(CharToString(in_file), CharToString(key), VectorCharToString(value), CharToString(out_file));
+}
+
+Status DataHelper::UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
+                                 const std::vector<bool> &value, const std::vector<char> &out_file) {
+  auto jh = JsonHelper();
+  return jh.UpdateArray(CharToString(in_file), CharToString(key), value, CharToString(out_file));
+}
+
+Status DataHelper::UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
+                                 const std::vector<int8_t> &value, const std::vector<char> &out_file) {
+  auto jh = JsonHelper();
+  return jh.UpdateArray(CharToString(in_file), CharToString(key), value, CharToString(out_file));
+}
+
+Status DataHelper::UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
+                                 const std::vector<uint8_t> &value, const std::vector<char> &out_file) {
+  auto jh = JsonHelper();
+  return jh.UpdateArray(CharToString(in_file), CharToString(key), value, CharToString(out_file));
+}
+
+Status DataHelper::UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
+                                 const std::vector<int16_t> &value, const std::vector<char> &out_file) {
+  auto jh = JsonHelper();
+  return jh.UpdateArray(CharToString(in_file), CharToString(key), value, CharToString(out_file));
+}
+
+Status DataHelper::UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
+                                 const std::vector<uint16_t> &value, const std::vector<char> &out_file) {
+  auto jh = JsonHelper();
+  return jh.UpdateArray(CharToString(in_file), CharToString(key), value, CharToString(out_file));
+}
+
+Status DataHelper::UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
+                                 const std::vector<int32_t> &value, const std::vector<char> &out_file) {
+  auto jh = JsonHelper();
+  return jh.UpdateArray(CharToString(in_file), CharToString(key), value, CharToString(out_file));
+}
+
+Status DataHelper::UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
+                                 const std::vector<uint32_t> &value, const std::vector<char> &out_file) {
+  auto jh = JsonHelper();
+  return jh.UpdateArray(CharToString(in_file), CharToString(key), value, CharToString(out_file));
+}
+
+Status DataHelper::UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
+                                 const std::vector<int64_t> &value, const std::vector<char> &out_file) {
+  auto jh = JsonHelper();
+  return jh.UpdateArray(CharToString(in_file), CharToString(key), value, CharToString(out_file));
+}
+
+Status DataHelper::UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
+                                 const std::vector<uint64_t> &value, const std::vector<char> &out_file) {
+  auto jh = JsonHelper();
+  return jh.UpdateArray(CharToString(in_file), CharToString(key), value, CharToString(out_file));
+}
+
+Status DataHelper::UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
+                                 const std::vector<float> &value, const std::vector<char> &out_file) {
+  auto jh = JsonHelper();
+  return jh.UpdateArray(CharToString(in_file), CharToString(key), value, CharToString(out_file));
+}
+
+Status DataHelper::UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
+                                 const std::vector<double> &value, const std::vector<char> &out_file) {
+  auto jh = JsonHelper();
+  return jh.UpdateArray(CharToString(in_file), CharToString(key), value, CharToString(out_file));
+}
+
+Status DataHelper::UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key,
+                                 const std::vector<char> &value, const std::vector<char> &out_file) {
+  auto jh = JsonHelper();
+  return jh.UpdateValue(CharToString(in_file), CharToString(key), CharToString(value), CharToString(out_file));
+}
+
+Status DataHelper::UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const bool &value,
+                                 const std::vector<char> &out_file) {
+  auto jh = JsonHelper();
+  return jh.UpdateValue(CharToString(in_file), CharToString(key), value, CharToString(out_file));
+}
+
+Status DataHelper::UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const int8_t &value,
+                                 const std::vector<char> &out_file) {
+  auto jh = JsonHelper();
+  return jh.UpdateValue(CharToString(in_file), CharToString(key), value, CharToString(out_file));
+}
+Status DataHelper::UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const uint8_t &value,
+                                 const std::vector<char> &out_file) {
+  auto jh = JsonHelper();
+  return jh.UpdateValue(CharToString(in_file), CharToString(key), value, CharToString(out_file));
+}
+Status DataHelper::UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const int16_t &value,
+                                 const std::vector<char> &out_file) {
+  auto jh = JsonHelper();
+  return jh.UpdateValue(CharToString(in_file), CharToString(key), value, CharToString(out_file));
+}
+Status DataHelper::UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const uint16_t &value,
+                                 const std::vector<char> &out_file) {
+  auto jh = JsonHelper();
+  return jh.UpdateValue(CharToString(in_file), CharToString(key), value, CharToString(out_file));
+}
+Status DataHelper::UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const int32_t &value,
+                                 const std::vector<char> &out_file) {
+  auto jh = JsonHelper();
+  return jh.UpdateValue(CharToString(in_file), CharToString(key), value, CharToString(out_file));
+}
+Status DataHelper::UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const uint32_t &value,
+                                 const std::vector<char> &out_file) {
+  auto jh = JsonHelper();
+  return jh.UpdateValue(CharToString(in_file), CharToString(key), value, CharToString(out_file));
+}
+Status DataHelper::UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const int64_t &value,
+                                 const std::vector<char> &out_file) {
+  auto jh = JsonHelper();
+  return jh.UpdateValue(CharToString(in_file), CharToString(key), value, CharToString(out_file));
+}
+Status DataHelper::UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const uint64_t &value,
+                                 const std::vector<char> &out_file) {
+  auto jh = JsonHelper();
+  return jh.UpdateValue(CharToString(in_file), CharToString(key), value, CharToString(out_file));
+}
+Status DataHelper::UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const float &value,
+                                 const std::vector<char> &out_file) {
+  auto jh = JsonHelper();
+  return jh.UpdateValue(CharToString(in_file), CharToString(key), value, CharToString(out_file));
+}
+
+Status DataHelper::UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const double &value,
+                                 const std::vector<char> &out_file) {
+  auto jh = JsonHelper();
+  return jh.UpdateValue(CharToString(in_file), CharToString(key), value, CharToString(out_file));
+}
+
+Status DataHelper::RemoveKeyIF(const std::vector<char> &in_file, const std::vector<char> &key,
+                               const std::vector<char> &out_file) {
+  auto jh = JsonHelper();
+  return jh.RemoveKey(CharToString(in_file), CharToString(key), CharToString(out_file));
+}
+
+size_t DataHelper::DumpData(const unsigned char *tensor_addr, const size_t &tensor_size, void *addr,
+                            const size_t &buffer_size) {
+  auto jh = JsonHelper();
+  return jh.DumpData(tensor_addr, tensor_size, addr, buffer_size);
+}
+
+}  // namespace dataset
+}  // namespace mindspore

mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/core/bindings.cc

@@ -22,7 +22,7 @@
 #include "minddata/dataset/core/client.h"  // DE client
 #include "minddata/dataset/util/status.h"
 #include "pybind11/numpy.h"
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 
 namespace mindspore {
 namespace dataset {

mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/datasetops/bindings.cc

@@ -20,7 +20,7 @@
 #include "minddata/dataset/api/python/pybind_conversion.h"
 #include "minddata/dataset/api/python/pybind_register.h"
 #include "minddata/dataset/callback/py_ds_callback.h"
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/core/global_context.h"
 #include "minddata/dataset/engine/serdes.h"
 #include "minddata/dataset/include/datasets.h"

mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/datasetops/source/bindings.cc

@@ -20,7 +20,7 @@
 #include "minddata/dataset/api/python/pybind_conversion.h"
 #include "minddata/dataset/api/python/pybind_register.h"
 #include "minddata/dataset/callback/py_ds_callback.h"
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/core/global_context.h"
 #include "minddata/dataset/engine/serdes.h"
 #include "minddata/dataset/include/datasets.h"

mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/datasetops/source/samplers/bindings.cc

@@ -21,7 +21,7 @@
 #include "minddata/dataset/api/python/pybind_conversion.h"
 #include "minddata/dataset/api/python/pybind_register.h"
 #include "minddata/dataset/callback/py_ds_callback.h"
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/core/global_context.h"
 #include "minddata/dataset/engine/ir/datasetops/source/samplers/samplers_ir.h"
 

mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/schema/bindings.cc

@@ -19,7 +19,7 @@
 
 #include "minddata/dataset/api/python/pybind_register.h"
 #include "minddata/dataset/core/global_context.h"
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/api/python/pybind_conversion.h"
 #include "minddata/dataset/include/datasets.h"
 

mindspore/ccsrc/minddata/dataset/core/client.h

@@ -19,7 +19,7 @@
 // client.h
 // Include file for DE client functions
 
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/core/data_type.h"
 #include "minddata/dataset/core/tensor.h"
 #include "minddata/dataset/core/tensor_shape.h"

mindspore/ccsrc/minddata/dataset/core/config_manager.h

@@ -22,7 +22,7 @@
 
 #include <nlohmann/json.hpp>
 
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/util/path.h"
 #include "minddata/dataset/util/status.h"
 
@@ -148,7 +148,7 @@ class ConfigManager {
   void set_numa_enable(bool numa_enable);
 
   /// getter function
-  /// Now we want to seperate the numa link to _c_dataengine in the CMakeLists,
+  /// Now we want to separate the numa link to _c_dataengine in the CMakeLists,
   /// so we want user to choose whether to open numa switch.
   /// @return Get the current numa switch state.
   bool numa_enable() const { return numa_enable_; }

mindspore/ccsrc/minddata/dataset/core/constants.h

@@ -1,112 +0,0 @@
-/**
- * Copyright 2019 Huawei Technologies Co., Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_CORE_CONSTANTS_H_
-#define MINDSPORE_CCSRC_MINDDATA_DATASET_CORE_CONSTANTS_H_
-
-#include <cstdint>
-#include <limits>
-#include <random>
-
-namespace mindspore {
-namespace dataset {
-// Various type defines for convenience
-using uchar = unsigned char;
-using dsize_t = int64_t;
-
-// Target devices to perform map operation
-enum class MapTargetDevice { kCpu, kGpu, kDvpp };
-
-// Possible dataset types for holding the data and client type
-enum class DatasetType { kUnknown, kArrow, kTf };
-
-// Possible flavours of Tensor implementations
-enum class TensorImpl { kNone, kFlexible, kCv, kNP };
-
-// Possible values for shuffle
-enum class ShuffleMode { kFalse = 0, kFiles = 1, kGlobal = 2 };
-
-// Possible values for Border types
-enum class BorderType { kConstant = 0, kEdge = 1, kReflect = 2, kSymmetric = 3 };
-
-// Possible values for Image format types in a batch
-enum class ImageBatchFormat { kNHWC = 0, kNCHW = 1 };
-
-// Possible values for Image format types
-enum class ImageFormat { HWC = 0, CHW = 1, HW = 2 };
-
-// Possible interpolation modes
-enum class InterpolationMode { kLinear = 0, kNearestNeighbour = 1, kCubic = 2, kArea = 3 };
-
-// Possible JiebaMode modes
-enum class JiebaMode { kMix = 0, kMp = 1, kHmm = 2 };
-
-// Possible values for SPieceTokenizerOutType
-enum class SPieceTokenizerOutType { kString = 0, kInt = 1 };
-
-// Possible values for SPieceTokenizerLoadType
-enum class SPieceTokenizerLoadType { kFile = 0, kModel = 1 };
-
-// Possible values for SentencePieceModel
-enum class SentencePieceModel { kUnigram = 0, kBpe = 1, kChar = 2, kWord = 3 };
-
-// Possible values for NormalizeForm
-enum class NormalizeForm {
-  kNone = 0,
-  kNfc,
-  kNfkc,
-  kNfd,
-  kNfkd,
-};
-
-// Possible values for SamplingStrategy
-enum class SamplingStrategy { kRandom = 0, kEdgeWeight = 1 };
-
-// convenience functions for 32bit int bitmask
-inline bool BitTest(uint32_t bits, uint32_t bitMask) { return (bits & bitMask) == bitMask; }
-
-inline void BitSet(uint32_t *bits, uint32_t bitMask) { *bits |= bitMask; }
-
-inline void BitClear(uint32_t *bits, uint32_t bitMask) { *bits &= (~bitMask); }
-
-constexpr int32_t kDeMaxDim = std::numeric_limits<int32_t>::max();  // 2147483647 or 2^32 -1
-constexpr int32_t kDeMaxRank = std::numeric_limits<int32_t>::max();
-constexpr int64_t kDeMaxFreq = std::numeric_limits<int64_t>::max();  // 9223372036854775807 or 2^(64-1)
-constexpr int64_t kDeMaxTopk = std::numeric_limits<int64_t>::max();
-
-constexpr uint32_t kCfgRowsPerBuffer = 1;
-constexpr uint32_t kCfgParallelWorkers = 8;
-constexpr uint32_t kCfgWorkerConnectorSize = 16;
-constexpr uint32_t kCfgOpConnectorSize = 16;
-constexpr int32_t kCfgDefaultRankId = -1;
-constexpr uint32_t kCfgDefaultSeed = std::mt19937::default_seed;
-constexpr uint32_t kCfgMonitorSamplingInterval = 10;
-constexpr uint32_t kCfgCallbackTimeout = 60;  // timeout value for callback in seconds
-constexpr int32_t kCfgDefaultCachePort = 50052;
-constexpr char kCfgDefaultCacheHost[] = "127.0.0.1";
-constexpr int32_t kDftPrefetchSize = 20;
-constexpr int32_t kDftNumConnections = 12;
-constexpr int32_t kDftAutoNumWorkers = false;
-
-// Invalid OpenCV type should not be from 0 to 7 (opencv4/opencv2/core/hal/interface.h)
-constexpr uint8_t kCVInvalidType = 255;
-
-using connection_id_type = uint64_t;
-using session_id_type = uint32_t;
-using row_id_type = int64_t;
-}  // namespace dataset
-}  // namespace mindspore
-
-#endif  // MINDSPORE_CCSRC_MINDDATA_DATASET_CORE_CONSTANTS_H_

mindspore/ccsrc/minddata/dataset/core/cv_tensor.cc

@@ -18,7 +18,7 @@
 #include <memory>
 #include <vector>
 
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/core/tensor.h"
 
 namespace mindspore {

mindspore/ccsrc/minddata/dataset/core/cv_tensor.h

@@ -24,7 +24,7 @@
 
 #include "./securec.h"
 
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/core/data_type.h"
 #include "minddata/dataset/core/tensor.h"
 

mindspore/ccsrc/minddata/dataset/core/data_type.h

@@ -29,7 +29,7 @@ namespace py = pybind11;
 #else
 #include "base/float16.h"
 #endif
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 namespace mindspore {
 namespace dataset {
 

mindspore/ccsrc/minddata/dataset/core/de_tensor.cc

@@ -16,7 +16,7 @@
 
 #include "minddata/dataset/core/de_tensor.h"
 #include "minddata/dataset/core/device_tensor.h"
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/core/data_type.h"
 #include "minddata/dataset/include/type_id.h"
 #include "mindspore/core/ir/dtype/type_id.h"

mindspore/ccsrc/minddata/dataset/core/device_tensor.h

@@ -20,7 +20,7 @@
 #include <utility>
 #include <vector>
 #include "include/api/status.h"
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/core/data_type.h"
 #include "minddata/dataset/core/tensor.h"
 #include "minddata/dataset/util/status.h"

mindspore/ccsrc/minddata/dataset/core/global_context.h

@@ -21,7 +21,7 @@
 
 #include "include/api/status.h"
 #include "minddata/dataset/core/config_manager.h"
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/util/allocator.h"
 
 namespace mindspore {

mindspore/ccsrc/minddata/dataset/core/tensor.cc

@@ -27,7 +27,7 @@
 #include "minddata/dataset/include/type_id.h"
 
 #include "utils/ms_utils.h"
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 
 #ifndef ENABLE_ANDROID
 #include "minddata/dataset/core/cv_tensor.h"

mindspore/ccsrc/minddata/dataset/core/tensor.h

@@ -37,7 +37,7 @@
 #include "pybind11/stl.h"
 #endif
 
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/core/data_type.h"
 #include "minddata/dataset/core/tensor_helpers.h"
 #include "minddata/dataset/core/tensor_shape.h"

mindspore/ccsrc/minddata/dataset/core/tensor_helpers.h

@@ -19,7 +19,7 @@
 #include <memory>
 #include <vector>
 
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 
 namespace mindspore {
 namespace dataset {

mindspore/ccsrc/minddata/dataset/core/tensor_shape.cc

@@ -25,7 +25,7 @@
 #else
 #include "mindspore/lite/src/common/log_adapter.h"
 #endif
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 
 namespace mindspore {
 namespace dataset {

mindspore/ccsrc/minddata/dataset/core/tensor_shape.h

@@ -31,7 +31,7 @@
 namespace py = pybind11;
 #endif
 
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/util/status.h"
 #include "minddata/dataset/core/global_context.h"
 #include "minddata/dataset/util/allocator.h"

mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin_arg.cc

@@ -31,7 +31,7 @@
 #include "minddata/dataset/engine/cache/cache_server.h"
 #include "minddata/dataset/engine/cache/cache_ipc.h"
 #include "minddata/dataset/util/path.h"
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 
 namespace mindspore {
 namespace dataset {

mindspore/ccsrc/minddata/dataset/engine/cache/cache_request.cc

@@ -22,7 +22,7 @@
 #include <cstdlib>
 #include <cstring>
 #include <thread>
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/engine/cache/cache_client.h"
 #include "minddata/dataset/engine/cache/cache_fbb.h"
 namespace mindspore {

mindspore/ccsrc/minddata/dataset/engine/cache/cache_server.cc

@@ -18,7 +18,7 @@
 #include <functional>
 #include <limits>
 #include <vector>
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/engine/cache/cache_ipc.h"
 #include "minddata/dataset/engine/cache/cache_service.h"
 #include "minddata/dataset/engine/cache/cache_request.h"

mindspore/ccsrc/minddata/dataset/engine/data_buffer.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -23,7 +23,7 @@
 #include <vector>
 #include "minddata/dataset/util/allocator.h"
 #include "minddata/dataset/util/status.h"
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/core/tensor.h"
 #include "minddata/dataset/core/tensor_row.h"
 
@@ -51,7 +51,7 @@ class DataBuffer {
   ~DataBuffer() = default;
 
   /// \brief A method for debug printing of the buffer
-  /// \param[inout] out The stream to write to
+  /// \param[in/out] out The stream to write to
   /// \param[in] show_all A boolean to toggle between details and summary printing
   void Print(std::ostream &out, bool show_all) const;
 

mindspore/ccsrc/minddata/dataset/engine/data_schema.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -23,7 +23,7 @@
 #include <unordered_map>
 #include <vector>
 #include <nlohmann/json.hpp>
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/core/data_type.h"
 #include "minddata/dataset/core/tensor_shape.h"
 #include "minddata/dataset/util/status.h"
@@ -65,7 +65,7 @@ class ColDescriptor {
   ///     If there is no starting TensorShape in this column, or if there is a shape but it contains
   ///     an unknown dimension, then the output shape returned shall resolve dimensions as needed.
   /// \param[in] num_elements - The number of elements in the data for a Tensor
-  /// \param[inout] out_shape - The materialized output Tensor shape
+  /// \param[in/out] out_shape - The materialized output Tensor shape
   /// \return Status The status code returned
   Status MaterializeTensorShape(int32_t num_elements, TensorShape *out_shape) const;
 
@@ -168,7 +168,7 @@ class DataSchema {
   static const char DEFAULT_DATA_SCHEMA_FILENAME[];
 
   /// \brief Loops through all columns in the schema and returns a map with the column name to column index number.
-  /// \param[inout] out_column_name_map - The output map of columns names to column index
+  /// \param[in/out] out_column_name_map - The output map of columns names to column index
   /// \return Status The status code returned
   Status GetColumnNameMap(std::unordered_map<std::string, int32_t> *out_column_name_map);
 

mindspore/ccsrc/minddata/dataset/engine/datasetops/barrier_op.cc

@@ -16,7 +16,7 @@
 #include "minddata/dataset/engine/datasetops/barrier_op.h"
 #include <iomanip>
 #include <utility>
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/engine/data_buffer.h"
 #include "minddata/dataset/engine/db_connector.h"
 #include "minddata/dataset/core/config_manager.h"

mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_lookup_op.cc

@@ -15,7 +15,7 @@
  */
 #include "minddata/dataset/engine/datasetops/cache_lookup_op.h"
 #include "minddata/dataset/core/config_manager.h"
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/core/global_context.h"
 #include "minddata/dataset/engine/execution_tree.h"
 #ifndef ENABLE_ANDROID

mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_merge_op.cc

@@ -21,7 +21,7 @@
 #include <iomanip>
 #include <utility>
 #include "minddata/dataset/core/config_manager.h"
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/core/global_context.h"
 #include "minddata/dataset/engine/execution_tree.h"
 #include "minddata/dataset/util/system_pool.h"

mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_op.cc

@@ -18,7 +18,7 @@
 #include <memory>
 #include <utility>
 #include "minddata/dataset/core/config_manager.h"
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/core/global_context.h"
 #include "minddata/dataset/engine/datasetops/repeat_op.h"
 #include "minddata/dataset/engine/data_buffer.h"

mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.h

@@ -24,7 +24,7 @@
 #include <utility>
 
 #include "minddata/dataset/callback/callback_manager.h"
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/engine/db_connector.h"
 #include "minddata/dataset/util/status.h"
 

mindspore/ccsrc/minddata/dataset/engine/datasetops/map_op/map_op.cc

@@ -21,7 +21,7 @@
 
 #include "minddata/dataset/callback/callback_param.h"
 #include "minddata/dataset/core/config_manager.h"
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/core/global_context.h"
 #include "minddata/dataset/engine/data_buffer.h"
 #include "minddata/dataset/engine/datasetops/map_op/cpu_map_job.h"

mindspore/ccsrc/minddata/dataset/engine/datasetops/parallel_op.h

@@ -19,7 +19,7 @@
 #include <memory>
 #include <string>
 #include <vector>
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/engine/datasetops/dataset_op.h"
 #include "minddata/dataset/engine/datasetops/source/io_block.h"
 #include "minddata/dataset/util/status.h"

mindspore/ccsrc/minddata/dataset/engine/datasetops/rename_op.cc

@@ -20,7 +20,7 @@
 #include <unordered_map>
 
 #include "minddata/dataset/core/config_manager.h"
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/core/global_context.h"
 #include "minddata/dataset/engine/data_buffer.h"
 #include "minddata/dataset/engine/db_connector.h"

mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.cc

@@ -23,7 +23,7 @@
 
 #include "utils/ms_utils.h"
 #include "minddata/dataset/core/config_manager.h"
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/core/global_context.h"
 #include "minddata/dataset/engine/data_buffer.h"
 #include "minddata/dataset/engine/datasetops/dataset_op.h"

mindspore/ccsrc/minddata/dataset/engine/datasetops/zip_op.cc

@@ -17,7 +17,7 @@
 #include <algorithm>
 #include <utility>
 #include <iomanip>
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/engine/data_buffer.h"
 #include "minddata/dataset/engine/db_connector.h"
 #include "minddata/dataset/core/config_manager.h"

mindspore/ccsrc/minddata/dataset/engine/db_connector.h

@@ -20,7 +20,7 @@
 #include <utility>
 #include "minddata/dataset/engine/connector.h"
 #include "minddata/dataset/engine/data_buffer.h"
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 
 namespace mindspore {
 namespace dataset {

mindspore/ccsrc/minddata/dataset/engine/gpu_item_connector.h

@@ -23,7 +23,7 @@
 #include <vector>
 #include "minddata/dataset/engine/connector.h"
 #include "minddata/dataset/util/status.h"
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "runtime/device/gpu/blocking_queue.h"
 
 using mindspore::device::DataItemGpu;

mindspore/ccsrc/minddata/dataset/engine/jagged_connector.h

@@ -23,7 +23,7 @@
 #include "minddata/dataset/engine/connector.h"
 #include "minddata/dataset/engine/data_buffer.h"
 #include "minddata/dataset/util/status.h"
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 
 namespace mindspore {
 namespace dataset {

mindspore/ccsrc/minddata/dataset/engine/perf/cyclic_array.h

@@ -21,7 +21,7 @@
 #include <algorithm>
 #include <cstring>
 #include <type_traits>
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 
 namespace mindspore {
 namespace dataset {

mindspore/ccsrc/minddata/dataset/engine/perf/perf_data.h

@@ -18,7 +18,7 @@
 #define MINDSPORE_CCSRC_MINDDATA_DATASET_PERF_DATA_H
 
 #include <vector>
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 
 namespace mindspore {
 namespace dataset {

mindspore/ccsrc/minddata/dataset/include/constants.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -13,8 +13,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_CORE_CONSTANTS_H_
-#define MINDSPORE_CCSRC_MINDDATA_DATASET_CORE_CONSTANTS_H_
+#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_CONSTANTS_H_
+#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_CONSTANTS_H_
 
 #include <cstdint>
 #include <limits>
@@ -87,9 +87,10 @@ constexpr int64_t kDeMaxFreq = std::numeric_limits<int64_t>::max();  // 92233720
 constexpr int64_t kDeMaxTopk = std::numeric_limits<int64_t>::max();
 
 constexpr uint32_t kCfgRowsPerBuffer = 1;
-constexpr uint32_t kCfgParallelWorkers = 4;
+constexpr uint32_t kCfgParallelWorkers = 8;
 constexpr uint32_t kCfgWorkerConnectorSize = 16;
 constexpr uint32_t kCfgOpConnectorSize = 16;
+constexpr int32_t kCfgDefaultRankId = -1;
 constexpr uint32_t kCfgDefaultSeed = std::mt19937::default_seed;
 constexpr uint32_t kCfgMonitorSamplingInterval = 10;
 constexpr uint32_t kCfgCallbackTimeout = 60;  // timeout value for callback in seconds
@@ -97,6 +98,7 @@ constexpr int32_t kCfgDefaultCachePort = 50052;
 constexpr char kCfgDefaultCacheHost[] = "127.0.0.1";
 constexpr int32_t kDftPrefetchSize = 20;
 constexpr int32_t kDftNumConnections = 12;
+constexpr int32_t kDftAutoNumWorkers = false;
 
 // Invalid OpenCV type should not be from 0 to 7 (opencv4/opencv2/core/hal/interface.h)
 constexpr uint8_t kCVInvalidType = 255;
@@ -107,4 +109,4 @@ using row_id_type = int64_t;
 }  // namespace dataset
 }  // namespace mindspore
 
-#endif  // MINDSPORE_CCSRC_MINDDATA_DATASET_CORE_CONSTANTS_H_
+#endif  // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_CONSTANTS_H_

mindspore/ccsrc/minddata/dataset/include/data_helper.h

@@ -0,0 +1,448 @@
+/**
+ * Copyright 2021 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATA_HELPER_H_
+#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATA_HELPER_H_
+
+#include <sys/stat.h>
+#include <fstream>
+#include <iostream>
+#include <map>
+#include <memory>
+#include <sstream>
+#include <string>
+#include <unordered_map>
+#include <vector>
+#include "include/api/dual_abi_helper.h"
+#include "include/api/status.h"
+
+namespace mindspore {
+namespace dataset {
+
+/// \brief Simple class to do data manipulation, contains helper function to update json files in dataset
+class DataHelper {
+ public:
+  /// \brief constructor
+  DataHelper() {}
+
+  /// \brief Destructor
+  ~DataHelper() = default;
+
+  /// \brief Create an Album dataset while taking in a path to a image folder
+  ///     Creates the output directory if doesn't exist
+  /// \param[in] in_dir Image folder directory that takes in images
+  /// \param[in] out_dir Directory containing output json files
+  Status CreateAlbum(const std::string &in_dir, const std::string &out_dir) {
+    return CreateAlbumIF(StringToChar(in_dir), StringToChar(out_dir));
+  }
+
+  /// \brief Update a json file field with a vector of string values
+  /// \param in_file The input file name to read in
+  /// \param key Key of field to write to
+  /// \param value Value array to write to file
+  /// \param out_file Optional input for output file path, will write to input file if not specified
+  /// \return Status The status code returned
+  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<std::string> &value,
+                     const std::string &out_file = "") {
+    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), VectorStringToChar(value), StringToChar(out_file));
+  }
+
+  /// \brief Update a json file field with a vector of bool values
+  /// \param in_file The input file name to read in
+  /// \param key Key of field to write to
+  /// \param value Value array to write to file
+  /// \param out_file Optional parameter for output file path, will write to input file if not specified
+  /// \return Status The status code returned
+  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<bool> &value,
+                     const std::string &out_file = "") {
+    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
+  }
+
+  /// \brief Update a json file field with a vector of int8 values
+  /// \param in_file The input file name to read in
+  /// \param key Key of field to write to
+  /// \param value Value array to write to file
+  /// \param out_file Optional parameter for output file path, will write to input file if not specified
+  /// \return Status The status code returned
+  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<int8_t> &value,
+                     const std::string &out_file = "") {
+    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
+  }
+
+  /// \brief Update a json file field with a vector of uint8 values
+  /// \param in_file The input file name to read in
+  /// \param key Key of field to write to
+  /// \param value Value array to write to file
+  /// \param out_file Optional parameter for output file path, will write to input file if not specified
+  /// \return Status The status code returned
+  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<uint8_t> &value,
+                     const std::string &out_file = "") {
+    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
+  }
+
+  /// \brief Update a json file field with a vector of int16 values
+  /// \param in_file The input file name to read in
+  /// \param key Key of field to write to
+  /// \param value Value array to write to file
+  /// \param out_file Optional parameter for output file path, will write to input file if not specified
+  /// \return Status The status code returned
+  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<int16_t> &value,
+                     const std::string &out_file = "") {
+    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
+  }
+
+  /// \brief Update a json file field with a vector of uint16 values
+  /// \param in_file The input file name to read in
+  /// \param key Key of field to write to
+  /// \param value Value array to write to file
+  /// \param out_file Optional parameter for output file path, will write to input file if not specified
+  /// \return Status The status code returned
+  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<uint16_t> &value,
+                     const std::string &out_file = "") {
+    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
+  }
+
+  /// \brief Update a json file field with a vector of int32 values
+  /// \param in_file The input file name to read in
+  /// \param key Key of field to write to
+  /// \param value Value array to write to file
+  /// \param out_file Optional parameter for output file path, will write to input file if not specified
+  /// \return Status The status code returned
+  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<int32_t> &value,
+                     const std::string &out_file = "") {
+    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
+  }
+
+  /// \brief Update a json file field with a vector of uint32 values
+  /// \param in_file The input file name to read in
+  /// \param key Key of field to write to
+  /// \param value Value array to write to file
+  /// \param out_file Optional parameter for output file path, will write to input file if not specified
+  /// \return Status The status code returned
+  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<uint32_t> &value,
+                     const std::string &out_file = "") {
+    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
+  }
+
+  /// \brief Update a json file field with a vector of int64 values
+  /// \param in_file The input file name to read in
+  /// \param key Key of field to write to
+  /// \param value Value array to write to file
+  /// \param out_file Optional parameter for output file path, will write to input file if not specified
+  /// \return Status The status code returned
+  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<int64_t> &value,
+                     const std::string &out_file = "") {
+    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
+  }
+
+  /// \brief Update a json file field with a vector of uint64 values
+  /// \param in_file The input file name to read in
+  /// \param key Key of field to write to
+  /// \param value Value array to write to file
+  /// \param out_file Optional parameter for output file path, will write to input file if not specified
+  /// \return Status The status code returned
+  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<uint64_t> &value,
+                     const std::string &out_file = "") {
+    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
+  }
+
+  /// \brief Update a json file field with a vector of float values
+  /// \param in_file The input file name to read in
+  /// \param key Key of field to write to
+  /// \param value Value array to write to file
+  /// \param out_file Optional parameter for output file path, will write to input file if not specified
+  /// \return Status The status code returned
+  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<float> &value,
+                     const std::string &out_file = "") {
+    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
+  }
+
+  /// \brief Update a json file field with a vector of double values
+  /// \param in_file The input file name to read in
+  /// \param key Key of field to write to
+  /// \param value Value array to write to file
+  /// \param out_file Optional parameter for output file path, will write to input file if not specified
+  /// \return Status The status code returned
+  Status UpdateArray(const std::string &in_file, const std::string &key, const std::vector<double> &value,
+                     const std::string &out_file = "") {
+    return UpdateArrayIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
+  }
+
+  /// \brief Update a json file field with a string value
+  /// \param in_file The input file name to read in
+  /// \param key Key of field to write to
+  /// \param value Value to write to file
+  /// \param out_file Optional parameter for output file path, will write to input file if not specified
+  /// \return Status The status code returned
+  Status UpdateValue(const std::string &in_file, const std::string &key, const std::string &value,
+                     const std::string &out_file = "") {
+    return UpdateValueIF(StringToChar(in_file), StringToChar(key), StringToChar(value), StringToChar(out_file));
+  }
+
+  /// \brief Update a json file field with a bool value
+  /// \param in_file The input file name to read in
+  /// \param key Key of field to write to
+  /// \param value Value to write to file
+  /// \param out_file Optional parameter for output file path, will write to input file if not specified
+  /// \return Status The status code returned
+  Status UpdateValue(const std::string &in_file, const std::string &key, const bool &value,
+                     const std::string &out_file = "") {
+    return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
+  }
+
+  /// \brief Update a json file field with an int8 value
+  /// \param in_file The input file name to read in
+  /// \param key Key of field to write to
+  /// \param value Value to write to file
+  /// \param out_file Optional parameter for output file path, will write to input file if not specified
+  /// \return Status The status code returned
+  Status UpdateValue(const std::string &in_file, const std::string &key, const int8_t &value,
+                     const std::string &out_file = "") {
+    return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
+  }
+
+  /// \brief Update a json file field with an uint8 value
+  /// \param in_file The input file name to read in
+  /// \param key Key of field to write to
+  /// \param value Value to write to file
+  /// \param out_file Optional parameter for output file path, will write to input file if not specified
+  /// \return Status The status code returned
+  Status UpdateValue(const std::string &in_file, const std::string &key, const uint8_t &value,
+                     const std::string &out_file = "") {
+    return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
+  }
+
+  /// \brief Update a json file field with an int16 value
+  /// \param in_file The input file name to read in
+  /// \param key Key of field to write to
+  /// \param value Value to write to file
+  /// \param out_file Optional parameter for output file path, will write to input file if not specified
+  /// \return Status The status code returned
+  Status UpdateValue(const std::string &in_file, const std::string &key, const int16_t &value,
+                     const std::string &out_file = "") {
+    return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
+  }
+
+  /// \brief Update a json file field with an uint16 value
+  /// \param in_file The input file name to read in
+  /// \param key Key of field to write to
+  /// \param value Value to write to file
+  /// \param out_file Optional parameter for output file path, will write to input file if not specified
+  /// \return Status The status code returned
+  Status UpdateValue(const std::string &in_file, const std::string &key, const uint16_t &value,
+                     const std::string &out_file = "") {
+    return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
+  }
+
+  /// \brief Update a json file field with an int32 value
+  /// \param in_file The input file name to read in
+  /// \param key Key of field to write to
+  /// \param value Value to write to file
+  /// \param out_file Optional parameter for output file path, will write to input file if not specified
+  /// \return Status The status code returned
+  Status UpdateValue(const std::string &in_file, const std::string &key, const int32_t &value,
+                     const std::string &out_file = "") {
+    return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
+  }
+
+  /// \brief Update a json file field with an uint32 value
+  /// \param in_file The input file name to read in
+  /// \param key Key of field to write to
+  /// \param value Value to write to file
+  /// \param out_file Optional parameter for output file path, will write to input file if not specified
+  /// \return Status The status code returned
+  Status UpdateValue(const std::string &in_file, const std::string &key, const uint32_t &value,
+                     const std::string &out_file = "") {
+    return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
+  }
+
+  /// \brief Update a json file field with an int64 value
+  /// \param in_file The input file name to read in
+  /// \param key Key of field to write to
+  /// \param value Value to write to file
+  /// \param out_file Optional parameter for output file path, will write to input file if not specified
+  /// \return Status The status code returned
+  Status UpdateValue(const std::string &in_file, const std::string &key, const int64_t &value,
+                     const std::string &out_file = "") {
+    return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
+  }
+
+  /// \brief Update a json file field with an uint64 value
+  /// \param in_file The input file name to read in
+  /// \param key Key of field to write to
+  /// \param value Value to write to file
+  /// \param out_file Optional parameter for output file path, will write to input file if not specified
+  /// \return Status The status code returned
+  Status UpdateValue(const std::string &in_file, const std::string &key, const uint64_t &value,
+                     const std::string &out_file = "") {
+    return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
+  }
+
+  /// \brief Update a json file field with a float value
+  /// \param in_file The input file name to read in
+  /// \param key Key of field to write to
+  /// \param value Value to write to file
+  /// \param out_file Optional parameter for output file path, will write to input file if not specified
+  /// \return Status The status code returned
+  Status UpdateValue(const std::string &in_file, const std::string &key, const float &value,
+                     const std::string &out_file = "") {
+    return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
+  }
+
+  /// \brief Update a json file field with a double value
+  /// \param in_file The input file name to read in
+  /// \param key Key of field to write to
+  /// \param value Value to write to file
+  /// \param out_file Optional parameter for output file path, will write to input file if not specified
+  /// \return Status The status code returned
+  Status UpdateValue(const std::string &in_file, const std::string &key, const double &value,
+                     const std::string &out_file = "") {
+    return UpdateValueIF(StringToChar(in_file), StringToChar(key), value, StringToChar(out_file));
+  }
+
+  /// \brief Template function to write tensor to file
+  /// \param[in] in_file File to write to
+  /// \param[in] data Array of type T values
+  /// \return Status The status code returned
+  template <typename T>
+  Status WriteBinFile(const std::string &in_file, const std::vector<T> &data) {
+    try {
+      std::ofstream o(in_file, std::ios::binary | std::ios::out);
+      if (!o.is_open()) {
+        return Status(kMDUnexpectedError, "Error opening Bin file to write");
+      }
+      size_t length = data.size();
+      o.write(reinterpret_cast<const char *>(&data[0]), std::streamsize(length * sizeof(T)));
+      o.close();
+    }
+    // Catch any exception and convert to Status return code
+    catch (const std::exception &err) {
+      return Status(kMDUnexpectedError, "Write bin file failed ");
+    }
+    return Status::OK();
+  }
+
+  /// \brief Write pointer to bin, use pointer to avoid memcpy
+  /// \param[in] in_file File name to write to
+  /// \param[in] data Pointer to data
+  /// \param[in] length Length of values to write from pointer
+  /// \return Status The status code returned
+  template <typename T>
+  Status WriteBinFile(const std::string &in_file, T *data, size_t length) {
+    try {
+      std::ofstream o(in_file, std::ios::binary | std::ios::out);
+      if (!o.is_open()) {
+        return Status(kMDUnexpectedError, "Error opening Bin file to write");
+      }
+      o.write(reinterpret_cast<const char *>(data), std::streamsize(length * sizeof(T)));
+      o.close();
+    }
+    // Catch any exception and convert to Status return code
+    catch (const std::exception &err) {
+      return Status(kMDUnexpectedError, "Write bin file failed ");
+    }
+    return Status::OK();
+  }
+
+  /// \brief Helper function to copy content of a tensor to buffer
+  /// \note This function iterates over the tensor in bytes, since
+  /// \param[in] tensor_addr The memory held by a tensor
+  /// \param[in] tensor_size The amount of data in bytes in tensor_addr, e.g. tensor->SizeInBytes()
+  /// \param[out] addr The address to copy tensor data to
+  /// \param[in] buffer_size The buffer size of addr
+  /// \return The size of the tensor (bytes copied
+  size_t DumpData(const unsigned char *tensor_addr, const size_t &tensor_size, void *addr, const size_t &buffer_size);
+
+  /// \brief Helper function to delete key in json file
+  /// note This function will return okay even if key not found
+  /// \param[in] in_file Json file to remove key from
+  /// \param[in] key The key to remove
+  /// \return Status The status code returned
+  Status RemoveKey(const std::string &in_file, const std::string &key, const std::string &out_file = "") {
+    return RemoveKeyIF(StringToChar(in_file), StringToChar(key), StringToChar(out_file));
+  }
+
+  /// \brief A print method typically used for debugging
+  /// \param out - The output stream to write output to
+  void Print(std::ostream &out) const;
+
+  /// \brief << Stream output operator overload
+  /// \notes This allows you to write the debug print info using stream operators
+  /// \param out Reference to the output stream being overloaded
+  /// \param ds Reference to the DataSchema to display
+  /// \return The output stream must be returned
+  friend std::ostream &operator<<(std::ostream &out, const DataHelper &dh) {
+    dh.Print(out);
+    return out;
+  }
+
+ private:
+  // Helper function for dual ABI support
+  Status CreateAlbumIF(const std::vector<char> &in_dir, const std::vector<char> &out_dir);
+  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
+                       const std::vector<std::vector<char>> &value, const std::vector<char> &out_file);
+  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key, const std::vector<bool> &value,
+                       const std::vector<char> &out_file);
+  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key, const std::vector<int8_t> &value,
+                       const std::vector<char> &out_file);
+  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
+                       const std::vector<uint8_t> &value, const std::vector<char> &out_file);
+  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
+                       const std::vector<int16_t> &value, const std::vector<char> &out_file);
+  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
+                       const std::vector<uint16_t> &value, const std::vector<char> &out_file);
+  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
+                       const std::vector<int32_t> &value, const std::vector<char> &out_file);
+  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
+                       const std::vector<uint32_t> &value, const std::vector<char> &out_file);
+  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
+                       const std::vector<int64_t> &value, const std::vector<char> &out_file);
+  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key,
+                       const std::vector<uint64_t> &value, const std::vector<char> &out_file);
+  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key, const std::vector<float> &value,
+                       const std::vector<char> &out_file);
+  Status UpdateArrayIF(const std::vector<char> &in_file, const std::vector<char> &key, const std::vector<double> &value,
+                       const std::vector<char> &out_file);
+
+  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const std::vector<char> &value,
+                       const std::vector<char> &out_file);
+  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const bool &value,
+                       const std::vector<char> &out_file);
+  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const int8_t &value,
+                       const std::vector<char> &out_file);
+  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const uint8_t &value,
+                       const std::vector<char> &out_file);
+  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const int16_t &value,
+                       const std::vector<char> &out_file);
+  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const uint16_t &value,
+                       const std::vector<char> &out_file);
+  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const int32_t &value,
+                       const std::vector<char> &out_file);
+  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const uint32_t &value,
+                       const std::vector<char> &out_file);
+  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const int64_t &value,
+                       const std::vector<char> &out_file);
+  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const uint64_t &value,
+                       const std::vector<char> &out_file);
+  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const float &value,
+                       const std::vector<char> &out_file);
+  Status UpdateValueIF(const std::vector<char> &in_file, const std::vector<char> &key, const double &value,
+                       const std::vector<char> &out_file);
+  Status RemoveKeyIF(const std::vector<char> &in_file, const std::vector<char> &key, const std::vector<char> &out_file);
+};
+}  // namespace dataset
+}  // namespace mindspore
+
+#endif  // MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATA_HELPER_H_

mindspore/ccsrc/minddata/dataset/include/datasets.h

@@ -30,11 +30,10 @@
 #include <utility>
 #include <vector>
 
-#include <nlohmann/json.hpp>
-
 #include "include/api/dual_abi_helper.h"
 #include "include/api/types.h"
 #include "minddata/dataset/include/iterator.h"
+#include "minddata/dataset/include/json_fwd.hpp"
 #include "minddata/dataset/include/samplers.h"
 #include "minddata/dataset/include/text.h"
 #include "minddata/dataset/include/type_id.h"

mindspore/ccsrc/minddata/dataset/include/json_fwd.hpp

@@ -0,0 +1,74 @@
+/**
+ * Copyright 2021 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef INCLUDE_NLOHMANN_JSON_FWD_HPP_
+#define INCLUDE_NLOHMANN_JSON_FWD_HPP_
+
+#include <cstdint>  // int64_t, uint64_t
+#include <map>      // map
+#include <memory>   // allocator
+#include <string>   // string
+#include <vector>   // vector
+
+/*!
+@brief namespace for Niels Lohmann
+@see https://github.com/nlohmann
+@since version 1.0.0
+*/
+namespace nlohmann {
+/*!
+@brief default JSONSerializer template argument
+
+This serializer ignores the template arguments and uses ADL
+([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl))
+for serialization.
+*/
+template <typename T = void, typename SFINAE = void>
+struct adl_serializer;
+
+template <template <typename U, typename V, typename... Args> class ObjectType = std::map,
+          template <typename U, typename... Args> class ArrayType = std::vector, class StringType = std::string,
+          class BooleanType = bool, class NumberIntegerType = std::int64_t, class NumberUnsignedType = std::uint64_t,
+          class NumberFloatType = double, template <typename U> class AllocatorType = std::allocator,
+          template <typename T, typename SFINAE = void> class JSONSerializer = adl_serializer>
+class basic_json;
+
+/*!
+@brief JSON Pointer
+
+A JSON pointer defines a string syntax for identifying a specific value
+within a JSON document. It can be used with functions `at` and
+`operator[]`. Furthermore, JSON pointers are the base for JSON patches.
+
+@sa [RFC 6901](https://tools.ietf.org/html/rfc6901)
+
+@since version 2.0.0
+*/
+template <typename BasicJsonType>
+class json_pointer;
+
+/*!
+@brief default JSON class
+
+This type is the default specialization of the @ref basic_json class which
+uses the standard template types.
+
+@since version 1.0.0
+*/
+using json = basic_json<>;
+}  // namespace nlohmann
+
+#endif  // INCLUDE_NLOHMANN_JSON_FWD_HPP_

mindspore/ccsrc/minddata/dataset/kernels/data/data_utils.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,7 +22,7 @@
 #include <vector>
 #include <utility>
 
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/core/data_type.h"
 #ifdef ENABLE_PYTHON
 #include "minddata/dataset/core/pybind_support.h"
@@ -117,101 +117,81 @@ Status OneHotEncoding(std::shared_ptr<Tensor> input, std::shared_ptr<Tensor> *ou
   }
 }
 
-Status Fill(const std::shared_ptr<Tensor> input, std::shared_ptr<Tensor> *output, std::shared_ptr<Tensor> fill_value) {
-  const DataType &fill_type = fill_value->type();
+Status FillHelper(const std::shared_ptr<Tensor> input, std::shared_ptr<Tensor> *out,
+                  std::shared_ptr<Tensor> fill_output, std::shared_ptr<Tensor> fill_value) {
   const DataType &input_type = input->type();
   const TensorShape &input_shape = input->shape();
-
-  CHECK_FAIL_RETURN_UNEXPECTED(!((fill_type == DataType::DE_STRING) && (input_type != DataType::DE_STRING)),
-                               "Fill: fill datatype does not match the input datatype.");
-
-  CHECK_FAIL_RETURN_UNEXPECTED(fill_value->shape() == TensorShape({}),
-                               "Fill: the shape of fill_value is not a scalar.");
-
-  std::shared_ptr<Tensor> out, fill_output;
-
-  if (input_type != DataType::DE_STRING && fill_type != DataType::DE_STRING && input_type != fill_type) {
-    auto op = std::make_unique<TypeCastOp>(input_type);
-    RETURN_IF_NOT_OK(op->Compute(fill_value, &fill_output));
-  } else {
-    fill_output = fill_value;
-  }
-
-  if (input_type.IsNumeric()) {
-    RETURN_IF_NOT_OK(Tensor::CreateEmpty(input_shape, input_type, &out));
-  }
-
   switch (input_type.value()) {
     case DataType::DE_BOOL: {
       bool value = 0;
       RETURN_IF_NOT_OK(fill_output->GetItemAt(&value, {}));
-      out->Fill<bool>(value);
+      (*out)->Fill<bool>(value);
       break;
     }
     case DataType::DE_INT8: {
       int8_t value = 0;
       RETURN_IF_NOT_OK(fill_output->GetItemAt(&value, {}));
-      out->Fill<int8_t>(value);
+      (*out)->Fill<int8_t>(value);
       break;
     }
     case DataType::DE_UINT8: {
       uint8_t value = 0;
       RETURN_IF_NOT_OK(fill_output->GetItemAt(&value, {}));
-      out->Fill<uint8_t>(value);
+      (*out)->Fill<uint8_t>(value);
       break;
     }
     case DataType::DE_UINT16: {
       uint16_t value = 0;
       RETURN_IF_NOT_OK(fill_output->GetItemAt(&value, {}));
-      out->Fill<uint16_t>(value);
+      (*out)->Fill<uint16_t>(value);
       break;
     }
     case DataType::DE_INT16: {
       int16_t value = 0;
       RETURN_IF_NOT_OK(fill_output->GetItemAt(&value, {}));
-      out->Fill<int16_t>(value);
+      (*out)->Fill<int16_t>(value);
       break;
     }
     case DataType::DE_UINT32: {
       uint32_t value = 0;
       RETURN_IF_NOT_OK(fill_output->GetItemAt(&value, {}));
-      out->Fill<uint32_t>(value);
+      (*out)->Fill<uint32_t>(value);
       break;
     }
     case DataType::DE_INT32: {
       int32_t value = 0;
       RETURN_IF_NOT_OK(fill_output->GetItemAt(&value, {}));
-      out->Fill<int32_t>(value);
+      (*out)->Fill<int32_t>(value);
       break;
     }
     case DataType::DE_UINT64: {
       uint64_t value = 0;
       RETURN_IF_NOT_OK(fill_output->GetItemAt(&value, {}));
-      out->Fill<uint64_t>(value);
+      (*out)->Fill<uint64_t>(value);
       break;
     }
     case DataType::DE_INT64: {
       int64_t value = 0;
       RETURN_IF_NOT_OK(fill_output->GetItemAt(&value, {}));
-      out->Fill<int64_t>(value);
+      (*out)->Fill<int64_t>(value);
       break;
     }
     case DataType::DE_FLOAT16: {
       int64_t value = 0;
       RETURN_IF_NOT_OK(fill_output->GetItemAt(&value, {}));
-      out->Fill<float>(value);
+      (*out)->Fill<float>(value);
       break;
     }
     case DataType::DE_FLOAT32: {
       float value = 0;
       RETURN_IF_NOT_OK(fill_output->GetItemAt(&value, {}));
-      out->Fill<float>(value);
+      (*out)->Fill<float>(value);
       break;
     }
     case DataType::DE_FLOAT64: {
       double value = 0;
       RETURN_IF_NOT_OK(fill_output->GetItemAt(&value, {}));
-      out->Fill<double>(value);
+      (*out)->Fill<double>(value);
       break;
     }
     case DataType::DE_STRING: {
@@ -222,7 +202,7 @@ Status Fill(const std::shared_ptr<Tensor> input, std::shared_ptr<Tensor> *output
       for (int i = 0; i < input_shape.NumOfElements(); i++) {
         strings.emplace_back(fill_string);
       }
-      RETURN_IF_NOT_OK(Tensor::CreateFromVector(strings, input_shape, &out));
+      RETURN_IF_NOT_OK(Tensor::CreateFromVector(strings, input_shape, out));
       break;
     }
     case DataType::DE_UNKNOWN: {
@@ -230,10 +210,37 @@ Status Fill(const std::shared_ptr<Tensor> input, std::shared_ptr<Tensor> *output
       break;
     }
   }
+  return Status::OK();
+}
+
+Status Fill(const std::shared_ptr<Tensor> input, std::shared_ptr<Tensor> *output, std::shared_ptr<Tensor> fill_value) {
+  const DataType &fill_type = fill_value->type();
+  const DataType &input_type = input->type();
+  const TensorShape &input_shape = input->shape();
+
+  CHECK_FAIL_RETURN_UNEXPECTED(!((fill_type == DataType::DE_STRING) && (input_type != DataType::DE_STRING)),
+                               "Fill: fill datatype does not match the input datatype.");
+
+  CHECK_FAIL_RETURN_UNEXPECTED(fill_value->shape() == TensorShape({}),
+                               "Fill: the shape of fill_value is not a scalar.");
+
+  std::shared_ptr<Tensor> out, fill_output;
+
+  if (input_type != DataType::DE_STRING && fill_type != DataType::DE_STRING && input_type != fill_type) {
+    auto op = std::make_unique<TypeCastOp>(input_type);
+    RETURN_IF_NOT_OK(op->Compute(fill_value, &fill_output));
+  } else {
+    fill_output = fill_value;
+  }
 
+  if (input_type.IsNumeric()) {
+    RETURN_IF_NOT_OK(Tensor::CreateEmpty(input_shape, input_type, &out));
+  }
+  RETURN_IF_NOT_OK(FillHelper(input, &out, fill_output, fill_value));
   *output = out;
   return Status::OK();
 }
+
 template <typename FROM, typename TO>
 void Cast(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
   auto in_itr = input->begin<FROM>();

mindspore/ccsrc/minddata/dataset/kernels/data/data_utils.h

@@ -20,7 +20,7 @@
 #include <string>
 #include <vector>
 #include <unordered_map>
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #ifndef ENABLE_ANDROID
 #include "minddata/dataset/core/cv_tensor.h"
 #endif

mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/utils/MDAclProcess.cc

@@ -13,7 +13,7 @@
  * limitations under the License.
  */
 
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/core/tensor_shape.h"
 #include "minddata/dataset/kernels/image/image_utils.h"
 #include "MDAclProcess.h"

mindspore/ccsrc/minddata/dataset/kernels/image/image_utils.cc

@@ -22,7 +22,7 @@
 #include <opencv2/imgcodecs.hpp>
 #include "utils/ms_utils.h"
 #include "minddata/dataset/kernels/image/math_utils.h"
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/core/cv_tensor.h"
 #include "minddata/dataset/core/tensor.h"
 #include "minddata/dataset/core/tensor_shape.h"

mindspore/ccsrc/minddata/dataset/kernels/image/lite_image_utils.cc

@@ -21,7 +21,7 @@
 #include "utils/ms_utils.h"
 #include "minddata/dataset/kernels/image/lite_cv/lite_mat.h"
 #include "minddata/dataset/kernels/image/lite_cv/image_process.h"
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/core/tensor.h"
 #include "minddata/dataset/core/tensor_shape.h"
 #include "minddata/dataset/util/random.h"

mindspore/ccsrc/minddata/dataset/kernels/image/pad_op.cc

@@ -16,7 +16,7 @@
 #include "minddata/dataset/kernels/image/pad_op.h"
 
 #include "minddata/dataset/kernels/image/image_utils.h"
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/util/status.h"
 
 namespace mindspore {

mindspore/ccsrc/minddata/dataset/kernels/image/pad_op.h

@@ -22,7 +22,7 @@
 
 #include "minddata/dataset/core/tensor.h"
 #include "minddata/dataset/kernels/tensor_op.h"
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/util/status.h"
 
 namespace mindspore {

mindspore/ccsrc/minddata/dataset/text/kernels/data_utils.h

@@ -20,7 +20,7 @@
 #include <string>
 #include <vector>
 #include "minddata/dataset/util/status.h"
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/core/data_type.h"
 #include "minddata/dataset/core/tensor.h"
 #include "minddata/dataset/core/cv_tensor.h"

mindspore/ccsrc/minddata/dataset/text/kernels/jieba_tokenizer_op.h

@@ -19,7 +19,7 @@
 #include <string>
 #include <memory>
 #include "cppjieba/Jieba.hpp"
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/kernels/tensor_op.h"
 #include "minddata/dataset/util/status.h"
 

mindspore/ccsrc/minddata/dataset/text/kernels/sentence_piece_tokenizer_op.h

@@ -23,7 +23,7 @@
 #include <iostream>
 #include <memory>
 
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/kernels/tensor_op.h"
 #include "minddata/dataset/util/status.h"
 #include "minddata/dataset/text/sentence_piece_vocab.h"

mindspore/ccsrc/minddata/dataset/util/data_helper.cc → mindspore/ccsrc/minddata/dataset/util/json_helper.cc

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020 Huawei Technologies Co., Ltd
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-#include "minddata/dataset/util/data_helper.h"
+#include "minddata/dataset/util/json_helper.h"
 
 #include <algorithm>
 #include <fstream>
@@ -30,7 +30,7 @@
 namespace mindspore {
 namespace dataset {
 // Create a numbered json file from image folder
-Status DataHelper::CreateAlbum(const std::string &in_dir, const std::string &out_dir) {
+Status JsonHelper::CreateAlbum(const std::string &in_dir, const std::string &out_dir) {
   // in check
   Path base_dir = Path(in_dir);
   if (!base_dir.IsDirectory() || !base_dir.Exists()) {
@@ -56,12 +56,12 @@ Status DataHelper::CreateAlbum(const std::string &in_dir, const std::string &out
 }
 
 // A print method typically used for debugging
-void DataHelper::Print(std::ostream &out) const {
+void JsonHelper::Print(std::ostream &out) const {
   out << "  Data Helper"
       << "\n";
 }
 
-Status DataHelper::UpdateArray(const std::string &in_file, const std::string &key,
+Status JsonHelper::UpdateArray(const std::string &in_file, const std::string &key,
                                const std::vector<std::string> &value, const std::string &out_file) {
   try {
     Path in = Path(in_file);
@@ -91,7 +91,7 @@ Status DataHelper::UpdateArray(const std::string &in_file, const std::string &ke
   return Status::OK();
 }
 
-Status DataHelper::RemoveKey(const std::string &in_file, const std::string &key, const std::string &out_file) {
+Status JsonHelper::RemoveKey(const std::string &in_file, const std::string &key, const std::string &out_file) {
   try {
     Path in = Path(in_file);
     nlohmann::json js;
@@ -120,7 +120,7 @@ Status DataHelper::RemoveKey(const std::string &in_file, const std::string &key,
   return Status::OK();
 }
 
-size_t DataHelper::DumpData(const unsigned char *tensor_addr, const size_t &tensor_size, void *addr,
+size_t JsonHelper::DumpData(const unsigned char *tensor_addr, const size_t &tensor_size, void *addr,
                             const size_t &buffer_size) {
   // write to address, input order is: destination, source
   errno_t ret = memcpy_s(addr, buffer_size, tensor_addr, tensor_size);

mindspore/ccsrc/minddata/dataset/util/data_helper.h → mindspore/ccsrc/minddata/dataset/util/json_helper.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2019 Huawei Technologies Co., Ltd
+ * Copyright 2019-2021 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -13,8 +13,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_UTIL_DATA_HELPER_H_
-#define MINDSPORE_CCSRC_MINDDATA_DATASET_UTIL_DATA_HELPER_H_
+#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_JSON_DATA_HELPER_H_
+#define MINDSPORE_CCSRC_MINDDATA_DATASET_JSON_DATA_HELPER_H_
 
 #include <fstream>
 #include <iostream>
@@ -34,13 +34,13 @@ namespace mindspore {
 namespace dataset {
 
 /// \brief Simple class to do data manipulation, contains helper function to update json files in dataset
-class DataHelper {
+class JsonHelper {
  public:
   /// \brief constructor
-  DataHelper() {}
+  JsonHelper() {}
 
   /// \brief Destructor
-  ~DataHelper() = default;
+  ~JsonHelper() = default;
 
   /// \brief Create an Album dataset while taking in a path to a image folder
   ///     Creates the output directory if doesn't exist
@@ -200,7 +200,7 @@ class DataHelper {
   /// \param out Reference to the output stream being overloaded
   /// \param ds Reference to the DataSchema to display
   /// \return The output stream must be returned
-  friend std::ostream &operator<<(std::ostream &out, const DataHelper &dh) {
+  friend std::ostream &operator<<(std::ostream &out, const JsonHelper &dh) {
     dh.Print(out);
     return out;
   }
@@ -208,4 +208,4 @@ class DataHelper {
 }  // namespace dataset
 }  // namespace mindspore
 
-#endif  // MINDSPORE_CCSRC_MINDDATA_DATASET_UTIL_DATA_HELPER_H_
+#endif  // MINDSPORE_CCSRC_MINDDATA_DATASET_UTIL_JSON_HELPER_H_

mindspore/dataset/engine/datasets.py

@@ -4909,7 +4909,7 @@ class _PaddedDataset:
 
 class PaddedDataset(GeneratorDataset):
     """
-    Create a dataset with fake data provided by user. Mainly used to add to the original data set
+    Create a dataset with filler data provided by user. Mainly used to add to the original data set
     and assign it to the corresponding shard.
 
     Args:

mindspore/lite/minddata/CMakeLists.txt

@@ -115,6 +115,7 @@ if(BUILD_MINDDATA STREQUAL "full")
         ${MINDDATA_DIR}/kernels/data/data_utils.cc
         ${MINDDATA_DIR}/api/samplers.cc
         ${MINDDATA_DIR}/api/iterator.cc
+        ${MINDDATA_DIR}/api/data_helper.cc
         ${MINDDATA_DIR}/api/execute.cc
         ${MINDDATA_DIR}/core/de_tensor.cc
         ${MINDDATA_DIR}/core/tensor_shape.cc
@@ -187,7 +188,7 @@ if(BUILD_MINDDATA STREQUAL "full")
         ${MINDDATA_DIR}/util/path.cc
         ${MINDDATA_DIR}/util/status.cc
         ${MINDDATA_DIR}/util/service.cc
-        ${MINDDATA_DIR}/util/data_helper.cc
+        ${MINDDATA_DIR}/util/json_helper.cc
         ${MINDDATA_DIR}/util/cond_var.cc
         ${MINDDATA_DIR}/engine/data_schema.cc
         ${MINDDATA_DIR}/kernels/tensor_op.cc
@@ -267,11 +268,12 @@ elseif(BUILD_MINDDATA STREQUAL "wrapper")
             ${MINDDATA_DIR}/core/tensor_row.cc
             ${MINDDATA_DIR}/core/de_tensor.cc
             ${MINDDATA_DIR}/api/vision.cc
+            ${MINDDATA_DIR}/api/data_helper.cc
             ${MINDDATA_DIR}/api/execute.cc
             ${MINDDATA_DIR}/api/transforms.cc
             ${MINDDATA_DIR}/util/path.cc
             ${MINDDATA_DIR}/util/status.cc
-            ${MINDDATA_DIR}/util/data_helper.cc
+            ${MINDDATA_DIR}/util/json_helper.cc
             ${MINDDATA_DIR}/util/memory_pool.cc
             ${MINDDATA_DIR}/engine/data_schema.cc
             ${MINDDATA_DIR}/kernels/tensor_op.cc

mindspore/lite/minddata/wrapper/MDToDApi.cc

@@ -29,7 +29,7 @@
 #include "minddata/dataset/include/type_id.h"
 #include "minddata/dataset/util/path.h"
 #include "minddata/dataset/include/vision.h"
-#include "minddata/dataset/util/data_helper.h"
+#include "minddata/dataset/include/data_helper.h"
 #include "minddata/dataset/core/de_tensor.h"
 #include "include/api/types.h"
 #if defined(__ANDROID__) || defined(ANDROID)

tests/ut/cpp/dataset/bit_functions_test.cc

@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "common/common.h"
 
 using namespace mindspore::dataset;

tests/ut/cpp/dataset/common/bboxop_common.cc

@@ -28,7 +28,7 @@
 #include "utils/ms_utils.h"
 #include "minddata/dataset/core/cv_tensor.h"
 #include "minddata/dataset/util/path.h"
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "utils/log_adapter.h"
 
 using namespace mindspore::dataset;
@@ -113,7 +113,7 @@ void BBoxOpCommon::SaveImagesWithAnnotations(BBoxOpCommon::FileType type, const
     std::shared_ptr<Tensor> row_to_save;
     Status swap_status = SwapRedAndBlue(row[0], &row_to_save);
     if (!swap_status.IsOk()) {
-      MS_LOG(ERROR) << "Swaping red and blue channels failed in SaveImagesWithAnnotations.";
+      MS_LOG(ERROR) << "Swapping red and blue channels failed in SaveImagesWithAnnotations.";
       EXPECT_TRUE(swap_status.IsOk());
     }
     cv::Mat image = std::static_pointer_cast<CVTensor>(row_to_save)->mat();

tests/ut/cpp/dataset/common/cvop_common.cc

@@ -18,7 +18,7 @@
 #include <string>
 #include <vector>
 #include "cvop_common.h"
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "utils/ms_utils.h"
 #include "minddata/dataset/core/cv_tensor.h"
 #include "utils/log_adapter.h"

tests/ut/cpp/dataset/data_helper_test.cc

@@ -30,7 +30,7 @@
 #include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h"
 #include "minddata/dataset/engine/datasetops/source/sampler/subset_random_sampler.h"
 #include "minddata/dataset/engine/datasetops/source/sampler/weighted_random_sampler.h"
-#include "minddata/dataset/util/data_helper.h"
+#include "minddata/dataset/include/data_helper.h"
 #include "minddata/dataset/util/path.h"
 #include "minddata/dataset/util/status.h"
 #include "gtest/gtest.h"

tests/ut/cpp/dataset/datatype_test.cc

@@ -19,7 +19,7 @@
 #include "common/common.h"
 #include "gtest/gtest.h"
 #include <opencv2/opencv.hpp>
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 
 using namespace mindspore::dataset;
 

tests/ut/cpp/dataset/distributed_sampler_test.cc

@@ -16,7 +16,7 @@
 #include "common/common.h"
 #include "gtest/gtest.h"
 
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/core/tensor.h"
 #include "minddata/dataset/engine/data_buffer.h"
 #include "minddata/dataset/engine/datasetops/source/sampler/sampler.h"

tests/ut/cpp/dataset/rename_op_test.cc

@@ -18,7 +18,7 @@
 #include <memory>
 #include <string>
 #include "minddata/dataset/core/client.h"
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/engine/datasetops/rename_op.h"
 #include "common/common.h"
 #include "utils/ms_utils.h"

tests/ut/cpp/dataset/subset_random_sampler_test.cc

@@ -16,7 +16,7 @@
 #include "common/common.h"
 #include "gtest/gtest.h"
 
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/core/tensor.h"
 #include "minddata/dataset/engine/data_buffer.h"
 #include "minddata/dataset/engine/datasetops/source/sampler/sampler.h"

tests/ut/cpp/dataset/subset_sampler_test.cc

@@ -16,7 +16,7 @@
 #include "common/common.h"
 #include "gtest/gtest.h"
 
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/core/tensor.h"
 #include "minddata/dataset/engine/data_buffer.h"
 #include "minddata/dataset/engine/datasetops/source/sampler/sampler.h"

tests/ut/cpp/dataset/weighted_random_sampler_test.cc

@@ -16,7 +16,7 @@
 #include "common/common.h"
 #include "gtest/gtest.h"
 
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/core/tensor.h"
 #include "minddata/dataset/engine/data_buffer.h"
 #include "minddata/dataset/engine/datasetops/source/sampler/sampler.h"

tests/ut/cpp/dataset/zip_op_test.cc

@@ -22,7 +22,7 @@
 #include <string>
 #include <thread>
 #include "minddata/dataset/core/client.h"
-#include "minddata/dataset/core/constants.h"
+#include "minddata/dataset/include/constants.h"
 #include "minddata/dataset/engine/datasetops/zip_op.h"
 #include "minddata/dataset/core/tensor.h"
 #include "minddata/dataset/core/config_manager.h"