mac build plus

mindspore/ccsrc/CMakeLists.txt

@@ -12,6 +12,8 @@ endif ()
 
 if (NOT(CMAKE_SYSTEM_NAME MATCHES "Darwin"))
     link_directories(${CMAKE_SOURCE_DIR}/build/mindspore/graphengine)
+else()
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-delete-non-abstract-non-virtual-dtor")
 endif ()
 
 if (CMAKE_SYSTEM_NAME MATCHES "Windows")

mindspore/ccsrc/minddata/dataset/CMakeLists.txt

@@ -3,7 +3,7 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-reorder")
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-switch")
 
 if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-uninitialized -Wno-deprecated-declarations -Wno-delete-non-abstract-non-virtual-dtor -Wno-constant-conversion")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-uninitialized -Wno-deprecated-declarations -Wno-delete-non-abstract-non-virtual-dtor -Wno-constant-conversion -Wno-unknown-warning-option")
 else()
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-maybe-uninitialized")
 endif()
@@ -91,6 +91,7 @@ add_subdirectory(core)
 add_subdirectory(kernels)
 add_subdirectory(engine)
 add_subdirectory(api)
+add_subdirectory(text)
 add_subdirectory(callback)
 ######################################################################
 add_dependencies(utils core)
@@ -107,17 +108,13 @@ add_dependencies(engine-perf core)
 add_dependencies(engine-gnn core)
 add_dependencies(engine core)
 add_dependencies(callback core)
+add_dependencies(text core)
+add_dependencies(text-kernels core)
 add_dependencies(cpp-API core)
 add_dependencies(engine-ir-datasetops core)
 add_dependencies(engine-ir-datasetops-source core)
 add_dependencies(engine-ir-cache core)
 
-if (NOT(${CMAKE_SYSTEM_NAME} MATCHES "Darwin"))
-    add_subdirectory(text)
-    add_dependencies(text core)
-    add_dependencies(text-kernels core)
-endif ()
-
 if (ENABLE_ACL)
     add_dependencies(kernels-dvpp-image core dvpp-utils)
 endif ()
@@ -162,14 +159,9 @@ set(submodules
         $<TARGET_OBJECTS:engine-opt>
         $<TARGET_OBJECTS:engine-cache-client>
         $<TARGET_OBJECTS:engine>
+        $<TARGET_OBJECTS:text>
+        $<TARGET_OBJECTS:text-kernels>
         )
-if (NOT(${CMAKE_SYSTEM_NAME} MATCHES "Darwin"))
-    set(submodules
-            ${submodules}
-            $<TARGET_OBJECTS:text>
-            $<TARGET_OBJECTS:text-kernels>
-            )
-endif ()
 
 if (ENABLE_ACL)
     set(submodules
@@ -229,9 +221,9 @@ target_link_libraries(_c_dataengine PUBLIC mindspore::jpeg_turbo mindspore::turb
                        mindspore::opencv_imgproc mindspore::tinyxml2 mindspore::sentencepiece_train ${ICU_LIB})
 if (ENABLE_GPUQUE)
     target_link_libraries(_c_dataengine PRIVATE gpu_queue
-                                  ${CUDNN_LIBRARY_PATH}
-				                  ${CUDA_PATH}/lib64/libcudart.so
-                                  ${CUDA_PATH}/lib64/stubs/libcuda.so)
+                          ${CUDNN_LIBRARY_PATH}
+                          ${CUDA_PATH}/lib64/libcudart.so
+                          ${CUDA_PATH}/lib64/stubs/libcuda.so)
 endif ()
 
 if (ENABLE_TDTQUE)

mindspore/ccsrc/minddata/dataset/api/CMakeLists.txt

@@ -1,85 +1,52 @@
 file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
 set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD)
 if (ENABLE_PYTHON)
-    if (APPLE)
-        add_library(APItoPython OBJECT
-                python/pybind_register.cc
-                python/pybind_conversion.cc
-                python/bindings/dataset/include/datasets_bindings.cc
-                python/bindings/dataset/include/iterator_bindings.cc
-                python/bindings/dataset/include/execute_binding.cc
-                python/bindings/dataset/include/schema_bindings.cc
-                python/bindings/dataset/engine/cache/bindings.cc
-                python/bindings/dataset/core/bindings.cc
-                python/bindings/dataset/callback/bindings.cc
-                python/bindings/dataset/kernels/data/bindings.cc
-                python/bindings/dataset/kernels/bindings.cc
-                python/bindings/dataset/engine/datasetops/bindings.cc
-                python/bindings/dataset/engine/datasetops/source/bindings.cc
-                python/bindings/dataset/engine/gnn/bindings.cc
-                python/bindings/dataset/kernels/image/bindings.cc
-                python/bindings/dataset/engine/datasetops/source/sampler/bindings.cc
-                python/bindings/mindrecord/include/bindings.cc
-                )
-    else()
-        add_library(APItoPython OBJECT
-                python/pybind_register.cc
-                python/pybind_conversion.cc
-                python/bindings/dataset/include/datasets_bindings.cc
-                python/bindings/dataset/include/iterator_bindings.cc
-                python/bindings/dataset/include/execute_binding.cc
-                python/bindings/dataset/include/schema_bindings.cc
-                python/bindings/dataset/engine/cache/bindings.cc
-                python/bindings/dataset/core/bindings.cc
-                python/bindings/dataset/callback/bindings.cc
-                python/bindings/dataset/kernels/data/bindings.cc
-                python/bindings/dataset/kernels/bindings.cc
-                python/bindings/dataset/engine/datasetops/bindings.cc
-                python/bindings/dataset/engine/datasetops/source/bindings.cc
-                python/bindings/dataset/engine/gnn/bindings.cc
-                python/bindings/dataset/kernels/image/bindings.cc
-                python/bindings/dataset/engine/datasetops/source/sampler/bindings.cc
-                python/bindings/dataset/text/bindings.cc
-                python/bindings/dataset/text/kernels/bindings.cc
-                python/bindings/mindrecord/include/bindings.cc
-                )
-    endif()
+    add_library(APItoPython OBJECT
+            python/pybind_register.cc
+            python/pybind_conversion.cc
+            python/bindings/dataset/include/datasets_bindings.cc
+            python/bindings/dataset/include/iterator_bindings.cc
+            python/bindings/dataset/include/execute_binding.cc
+            python/bindings/dataset/include/schema_bindings.cc
+            python/bindings/dataset/engine/cache/bindings.cc
+            python/bindings/dataset/core/bindings.cc
+            python/bindings/dataset/callback/bindings.cc
+            python/bindings/dataset/kernels/data/bindings.cc
+            python/bindings/dataset/kernels/bindings.cc
+            python/bindings/dataset/engine/datasetops/bindings.cc
+            python/bindings/dataset/engine/datasetops/source/bindings.cc
+            python/bindings/dataset/engine/gnn/bindings.cc
+            python/bindings/dataset/kernels/image/bindings.cc
+            python/bindings/dataset/engine/datasetops/source/sampler/bindings.cc
+            python/bindings/dataset/text/bindings.cc
+            python/bindings/dataset/text/kernels/bindings.cc
+            python/bindings/mindrecord/include/bindings.cc
+            )
     target_include_directories(APItoPython PRIVATE ${pybind11_INCLUDE_DIRS})
 endif ()
 
-if (APPLE)
+
+if (ENABLE_ACL)
     add_library(cpp-API OBJECT
             config.cc
             datasets.cc
             execute.cc
             iterator.cc
+            minddata_eager.cc
             transforms.cc
             samplers.cc
+            text.cc
             vision.cc
             )
 else()
-    if (ENABLE_ACL)
-        add_library(cpp-API OBJECT
-                config.cc
-                datasets.cc
-                execute.cc
-                iterator.cc
-                minddata_eager.cc
-                transforms.cc
-                samplers.cc
-                text.cc
-                vision.cc
-                )
-    else()
-        add_library(cpp-API OBJECT
-                config.cc
-                datasets.cc
-                execute.cc
-                iterator.cc
-                transforms.cc
-                samplers.cc
-                text.cc
-                vision.cc
-                )
-    endif()
+    add_library(cpp-API OBJECT
+            config.cc
+            datasets.cc
+            execute.cc
+            iterator.cc
+            transforms.cc
+            samplers.cc
+            text.cc
+            vision.cc
+            )
 endif()

mindspore/ccsrc/minddata/dataset/text/kernels/jieba_tokenizer_op.h

@@ -18,7 +18,6 @@
 
 #include <string>
 #include <memory>
-#if !defined(__APPLE__)
 #include "cppjieba/Jieba.hpp"
 #include "minddata/dataset/core/constants.h"
 #include "minddata/dataset/kernels/tensor_op.h"
@@ -67,4 +66,3 @@ class JiebaTokenizerOp : public TensorOp {
 }  // namespace dataset
 }  // namespace mindspore
 #endif  // MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_TEXT_JIEBA_OP_H_
-#endif

mindspore/ccsrc/minddata/mindrecord/include/shard_category.h

@@ -26,7 +26,7 @@
 
 namespace mindspore {
 namespace mindrecord {
-class ShardCategory : public ShardOperator {
+class __attribute__((visibility("default"))) ShardCategory : public ShardOperator {
  public:
   explicit ShardCategory(const std::vector<std::pair<std::string, std::string>> &categories,
                          int64_t num_elements = std::numeric_limits<int64_t>::max(), bool replacement = false);

mindspore/ccsrc/minddata/mindrecord/include/shard_column.h

@@ -57,7 +57,7 @@ const std::unordered_map<std::string, ColumnDataType> ColumnDataTypeMap = {
   {"bytes", ColumnBytes}, {"string", ColumnString},   {"int32", ColumnInt32},
   {"int64", ColumnInt64}, {"float32", ColumnFloat32}, {"float64", ColumnFloat64}};
 
-class ShardColumn {
+class __attribute__((visibility("default"))) ShardColumn {
  public:
   explicit ShardColumn(const std::shared_ptr<ShardHeader> &shard_header, bool compress_integer = true);
   explicit ShardColumn(const json &schema_json, bool compress_integer = true);

mindspore/ccsrc/minddata/mindrecord/include/shard_distributed_sample.h

@@ -27,7 +27,7 @@
 
 namespace mindspore {
 namespace mindrecord {
-class ShardDistributedSample : public ShardSample {
+class __attribute__((visibility("default"))) ShardDistributedSample : public ShardSample {
  public:
   ShardDistributedSample(int num_shards, int shard_id, int no_of_padded_samples, bool shuffle, uint32_t seed,
                          int no_of_samples = 0, int offset = -1);

mindspore/ccsrc/minddata/mindrecord/include/shard_error.h

@@ -77,7 +77,7 @@ enum MSRStatus {
 };
 
 // convert error no to string message
-std::string ErrnoToMessage(MSRStatus status);
+std::string __attribute__((visibility("default"))) ErrnoToMessage(MSRStatus status);
 }  // namespace mindrecord
 }  // namespace mindspore
 

mindspore/ccsrc/minddata/mindrecord/include/shard_header.h

@@ -31,7 +31,7 @@
 
 namespace mindspore {
 namespace mindrecord {
-class ShardHeader {
+class __attribute__((visibility("default"))) ShardHeader {
  public:
   ShardHeader();
 

mindspore/ccsrc/minddata/mindrecord/include/shard_index.h

@@ -37,7 +37,7 @@ using std::pair;
 using std::string;
 using std::vector;
 
-class Index {
+class __attribute__((visibility("default"))) Index {
  public:
   Index();
 

mindspore/ccsrc/minddata/mindrecord/include/shard_index_generator.h

@@ -32,7 +32,7 @@ namespace mindspore {
 namespace mindrecord {
 using INDEX_FIELDS = std::pair<MSRStatus, std::vector<std::tuple<std::string, std::string, std::string>>>;
 using ROW_DATA = std::pair<MSRStatus, std::vector<std::vector<std::tuple<std::string, std::string, std::string>>>>;
-class ShardIndexGenerator {
+class __attribute__((visibility("default"))) ShardIndexGenerator {
  public:
   explicit ShardIndexGenerator(const std::string &file_path, bool append = false);
 

mindspore/ccsrc/minddata/mindrecord/include/shard_operator.h

@@ -22,7 +22,7 @@
 
 namespace mindspore {
 namespace mindrecord {
-class ShardOperator {
+class __attribute__((visibility("default"))) ShardOperator {
  public:
   virtual ~ShardOperator() = default;
 

mindspore/ccsrc/minddata/mindrecord/include/shard_page.h

@@ -33,7 +33,7 @@ const std::string kPageTypeRaw = "RAW_DATA";
 const std::string kPageTypeBlob = "BLOB_DATA";
 const std::string kPageTypeNewColumn = "NEW_COLUMN_DATA";
 
-class Page {
+class __attribute__((visibility("default"))) Page {
  public:
   Page(const int &page_id, const int &shard_id, const std::string &page_type, const int &page_type_id,
        const uint64_t &start_row_id, const uint64_t end_row_id,

mindspore/ccsrc/minddata/mindrecord/include/shard_pk_sample.h

@@ -27,7 +27,7 @@
 
 namespace mindspore {
 namespace mindrecord {
-class ShardPkSample : public ShardCategory {
+class __attribute__((visibility("default"))) ShardPkSample : public ShardCategory {
  public:
   ShardPkSample(const std::string &category_field, int64_t num_elements, int64_t num_samples);
 

mindspore/ccsrc/minddata/mindrecord/include/shard_reader.h

@@ -65,7 +65,7 @@ using TASK_RETURN_CONTENT =
   std::pair<MSRStatus, std::pair<TaskType, std::vector<std::tuple<std::vector<uint8_t>, json>>>>;
 const int kNumBatchInMap = 1000;  // iterator buffer size in row-reader mode
 
-class ShardReader {
+class __attribute__((visibility("default"))) ShardReader {
  public:
   ShardReader();
 

mindspore/ccsrc/minddata/mindrecord/include/shard_sample.h

@@ -26,7 +26,7 @@
 
 namespace mindspore {
 namespace mindrecord {
-class ShardSample : public ShardOperator {
+class __attribute__((visibility("default"))) ShardSample : public ShardOperator {
  public:
   explicit ShardSample(int n);
 

mindspore/ccsrc/minddata/mindrecord/include/shard_schema.h

@@ -30,7 +30,7 @@
 
 namespace mindspore {
 namespace mindrecord {
-class Schema {
+class __attribute__((visibility("default"))) Schema {
  public:
   ~Schema() = default;
 

mindspore/ccsrc/minddata/mindrecord/include/shard_segment.h

@@ -25,7 +25,7 @@
 
 namespace mindspore {
 namespace mindrecord {
-class ShardSegment : public ShardReader {
+class __attribute__((visibility("default"))) ShardSegment : public ShardReader {
  public:
   ShardSegment();
 

mindspore/ccsrc/minddata/mindrecord/include/shard_sequential_sample.h

@@ -25,7 +25,7 @@
 
 namespace mindspore {
 namespace mindrecord {
-class ShardSequentialSample : public ShardSample {
+class __attribute__((visibility("default"))) ShardSequentialSample : public ShardSample {
  public:
   ShardSequentialSample(int64_t n, int64_t offset);
 

mindspore/ccsrc/minddata/mindrecord/include/shard_shuffle.h

@@ -22,7 +22,7 @@
 
 namespace mindspore {
 namespace mindrecord {
-class ShardShuffle : public ShardOperator {
+class __attribute__((visibility("default"))) ShardShuffle : public ShardOperator {
  public:
   explicit ShardShuffle(uint32_t seed = 0, ShuffleType shuffle_type = kShuffleCategory);
 

mindspore/ccsrc/minddata/mindrecord/include/shard_statistics.h

@@ -32,7 +32,7 @@
 
 namespace mindspore {
 namespace mindrecord {
-class Statistics {
+class __attribute__((visibility("default"))) Statistics {
  public:
   /// \brief save the statistic and its description
   /// \param[in] desc the statistic's description

mindspore/ccsrc/minddata/mindrecord/include/shard_task.h

@@ -27,7 +27,7 @@
 
 namespace mindspore {
 namespace mindrecord {
-class ShardTask {
+class __attribute__((visibility("default"))) ShardTask {
  public:
   ShardTask();
 

mindspore/ccsrc/minddata/mindrecord/include/shard_writer.h

@@ -46,7 +46,7 @@
 
 namespace mindspore {
 namespace mindrecord {
-class ShardWriter {
+class __attribute__((visibility("default"))) ShardWriter {
  public:
   ShardWriter();
 

mindspore/core/CMakeLists.txt

@@ -18,7 +18,7 @@ if (CMAKE_SYSTEM_NAME MATCHES "Windows")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-attributes -DHAVE_SNPRINTF")
     add_compile_definitions(BUILDING_DLL)
 elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin")
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wuser-defined-warnings -Winconsistent-missing-override")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wuser-defined-warnings -Winconsistent-missing-override -Wno-delete-non-abstract-non-virtual-dtor")
 endif ()
 
 set_property(SOURCE ${CORE_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_CORE)

third_party/patch/cppjieba/cppjieba.patch001

@@ -1,6 +1,39 @@
+diff -Npur cppjieba/deps/limonp/StringUtil.hpp cppjiebap/deps/limonp/StringUtil.hpp
+--- cppjieba/deps/limonp/StringUtil.hpp	2020-03-11 09:30:52.000000000 +0800
++++ cppjiebap/deps/limonp/StringUtil.hpp	2020-12-15 16:02:38.000000000 +0800
+@@ -84,12 +84,12 @@ inline bool IsSpace(unsigned c) {
+ }
+ 
+ inline std::string& LTrim(std::string &s) {
+-  s.erase(s.begin(), std::find_if(s.begin(), s.end(), std::not1(std::ptr_fun<unsigned, bool>(IsSpace))));
++  s.erase(s.begin(), std::find_if(s.begin(), s.end(), std::not1(std::function<unsigned(bool)>(IsSpace))));
+   return s;
+ }
+ 
+ inline std::string& RTrim(std::string &s) {
+-  s.erase(std::find_if(s.rbegin(), s.rend(), std::not1(std::ptr_fun<unsigned, bool>(IsSpace))).base(), s.end());
++  s.erase(std::find_if(s.rbegin(), s.rend(), std::not1(std::function<unsigned(bool)>(IsSpace))).base(), s.end());
+   return s;
+ }
+ 
+@@ -98,12 +98,12 @@ inline std::string& Trim(std::string &s)
+ }
+ 
+ inline std::string& LTrim(std::string & s, char x) {
+-  s.erase(s.begin(), std::find_if(s.begin(), s.end(), std::not1(std::bind2nd(std::equal_to<char>(), x))));
++  s.erase(s.begin(), std::find_if(s.begin(), s.end(), [x](char c) -> bool { return c != x; }));
+   return s;
+ }
+ 
+ inline std::string& RTrim(std::string & s, char x) {
+-  s.erase(std::find_if(s.rbegin(), s.rend(), std::not1(std::bind2nd(std::equal_to<char>(), x))).base(), s.end());
++  s.erase(std::find_if(s.rbegin(), s.rend(), [x](char c) -> bool { return c != x; } ).base(), s.end());
+   return s;
+ }
+ 
 diff -Npur cppjieba/include/cppjieba/Jieba.hpp cppjiebap/include/cppjieba/Jieba.hpp
---- cppjieba/include/cppjieba/Jieba.hpp	2020-05-07 15:27:16.490147073 +0800
-+++ cppjiebap/include/cppjieba/Jieba.hpp	2020-05-07 15:51:15.315931163 +0800
+--- cppjieba/include/cppjieba/Jieba.hpp	2020-03-11 09:30:52.000000000 +0800
++++ cppjiebap/include/cppjieba/Jieba.hpp	2020-12-15 16:01:46.000000000 +0800
 @@ -10,17 +10,14 @@ class Jieba {
   public:
    Jieba(const string& dict_path, 
@@ -31,8 +64,8 @@ diff -Npur cppjieba/include/cppjieba/Jieba.hpp cppjiebap/include/cppjieba/Jieba.
  
  } // namespace cppjieba
 diff -Npur cppjieba/test/demo.cpp cppjiebap/test/demo.cpp
---- cppjieba/test/demo.cpp	2020-05-07 15:27:16.490147073 +0800
-+++ cppjiebap/test/demo.cpp	2020-05-07 15:53:21.630248552 +0800
+--- cppjieba/test/demo.cpp	2020-03-11 09:30:52.000000000 +0800
++++ cppjiebap/test/demo.cpp	2020-12-15 16:01:46.000000000 +0800
 @@ -11,9 +11,7 @@ const char* const STOP_WORD_PATH = "../d
  int main(int argc, char** argv) {
    cppjieba::Jieba jieba(DICT_PATH,
@@ -56,8 +89,8 @@ diff -Npur cppjieba/test/demo.cpp cppjiebap/test/demo.cpp
    return EXIT_SUCCESS;
  }
 diff -Npur cppjieba/test/unittest/jieba_test.cpp cppjiebap/test/unittest/jieba_test.cpp
---- cppjieba/test/unittest/jieba_test.cpp	2020-05-07 15:27:16.522146752 +0800
-+++ cppjiebap/test/unittest/jieba_test.cpp	2020-05-07 15:59:11.630860061 +0800
+--- cppjieba/test/unittest/jieba_test.cpp	2020-03-11 09:30:52.000000000 +0800
++++ cppjiebap/test/unittest/jieba_test.cpp	2020-12-15 16:01:46.000000000 +0800
 @@ -6,9 +6,7 @@ using namespace cppjieba;
  TEST(JiebaTest, Test1) {
    cppjieba::Jieba jieba("../dict/jieba.dict.utf8",