Huawei_Technology
/
mindspore
Not watched
1
0
Fork 0
Code
Releases 13 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: a38d6139fa
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'a38d6139fa'
${ noResults }
mindspore/third_party/patch/pslite/ps_lite.patch001

256 lines
9.8 kB
Raw Blame History 

  1. diff -Npur ps-lite-master/include/dmlc/base.h ps-lite-master-new/include/dmlc/base.h

  2. --- ps-lite-master/include/dmlc/base.h	2020-02-29 13:59:55.000000000 +0800

  3. +++ ps-lite-master-new/include/dmlc/base.h	2020-07-01 11:56:50.444833389 +0800

  4. @@ -8,7 +8,7 @@

  5.  

  6.  /*! \brief whether use glog for logging */

  7.  #ifndef DMLC_USE_GLOG

  8. -#define DMLC_USE_GLOG 0

  9. +#define DMLC_USE_GLOG 1

  10.  #endif

  11.  

  12.  /*!

  13. diff -Npur ps-lite-master/include/dmlc/logging.h ps-lite-master-new/include/dmlc/logging.h

  14. --- ps-lite-master/include/dmlc/logging.h	2020-02-29 13:59:55.000000000 +0800

  15. +++ ps-lite-master-new/include/dmlc/logging.h	2020-07-08 21:35:33.334584767 +0800

  16. @@ -52,7 +52,7 @@ struct Error : public std::runtime_error

  17.  

  18.  namespace dmlc {

  19.  inline void InitLogging(const char* argv0) {

  20. -  google::InitGoogleLogging(argv0);

  21. +  //google::InitGoogleLogging(argv0);

  22.  }

  23.  }  // namespace dmlc

  24.  

  25. diff -Npur ps-lite-master/make/deps.mk ps-lite-master-new/make/deps.mk

  26. --- ps-lite-master/make/deps.mk	2020-02-29 13:59:55.000000000 +0800

  27. +++ ps-lite-master-new/make/deps.mk	2020-06-17 10:35:46.253837426 +0800

  28. @@ -1,69 +1,7 @@

  29.  # Install dependencies

  30. -

  31. -URL1=https://raw.githubusercontent.com/mli/deps/master/build

  32. -URL2=https://github.com/google/protobuf/releases/download/v3.5.1

  33. -ifndef WGET

  34. -WGET = wget

  35. -endif

  36. -

  37. -# protobuf

  38. -PROTOBUF = ${DEPS_PATH}/include/google/protobuf/message.h

  39. -${PROTOBUF}:

  40. -	$(eval FILE=protobuf-cpp-3.5.1.tar.gz)

  41. -	$(eval DIR=protobuf-3.5.1)

  42. -	rm -rf $(FILE) $(DIR)

  43. -	$(WGET) $(URL2)/$(FILE) && tar --no-same-owner -zxf $(FILE)

  44. -	cd $(DIR) && export CFLAGS=-fPIC && export CXXFLAGS=-fPIC && ./configure -prefix=$(DEPS_PATH) && $(MAKE) && $(MAKE) install

  45. -	rm -rf $(FILE) $(DIR)

  46. -

  47.  # zmq

  48. -ZMQ = ${DEPS_PATH}/include/zmq.h

  49. +ZMQ = $(MS_ZMQ_INSTALL_PATH)/lib/libzmq.a

  50.  

  51.  ${ZMQ}:

  52. -	$(eval FILE=zeromq-4.1.4.tar.gz)

  53. -	$(eval DIR=zeromq-4.1.4)

  54. -	rm -rf $(FILE) $(DIR)

  55. -	$(WGET) $(URL1)/$(FILE) && tar --no-same-owner -zxf $(FILE)

  56. -	cd $(DIR) && export CFLAGS=-fPIC && export CXXFLAGS=-fPIC && ./configure -prefix=$(DEPS_PATH) --with-libsodium=no --with-libgssapi_krb5=no && $(MAKE) && $(MAKE) install

  57. -	rm -rf $(FILE) $(DIR)

  58. -

  59. -# lz4

  60. -LZ4 = ${DEPS_PATH}/include/lz4.h

  61. -${LZ4}:

  62. -	$(eval FILE=lz4-r129.tar.gz)

  63. -	$(eval DIR=lz4-r129)

  64. -	rm -rf $(FILE) $(DIR)

  65. -	wget $(URL1)/$(FILE) && tar --no-same-owner -zxf $(FILE)

  66. -	cd $(DIR) && $(MAKE) && PREFIX=$(DEPS_PATH) $(MAKE) install

  67. -	rm -rf $(FILE) $(DIR)

  68. -

  69. -# cityhash

  70. -CITYHASH = ${DEPS_PATH}/include/city.h

  71. -${CITYHASH}:

  72. -	$(eval FILE=cityhash-1.1.1.tar.gz)

  73. -	$(eval DIR=cityhash-1.1.1)

  74. -	rm -rf $(FILE) $(DIR)

  75. -	wget $(URL1)/$(FILE) && tar --no-same-owner -zxf $(FILE)

  76. -	cd $(DIR) && ./configure -prefix=$(DEPS_PATH) --enable-sse4.2 && $(MAKE) CXXFLAGS="-g -O3 -msse4.2" && $(MAKE) install

  77. -	rm -rf $(FILE) $(DIR)

  78. -

  79. -

  80. -# # gflags

  81. -# ${DEPS_PATH}/include/google/gflags.h:

  82. -# 	$(eval FILE=gflags-2.0-no-svn-files.tar.gz)

  83. -# 	$(eval DIR=gflags-2.0)

  84. -# 	rm -rf $(FILE) $(DIR)

  85. -# 	wget $(URL)/$(FILE) && tar -zxf $(FILE)

  86. -# 	cd $(DIR) && ./configure -prefix=$(DEPS_PATH) && $(MAKE) && $(MAKE) install

  87. -# 	rm -rf $(FILE) $(DIR)

  88. -# gflags: | ${DEPS_PATH}/include/google/gflags.h

  89. +	cd $(MS_ZMQ_DIR) && export CFLAGS="-fPIC -D_GLIBCXX_USE_CXX11_ABI=0" && export CXXFLAGS=-fPIC && ./configure -prefix=$(MS_ZMQ_INSTALL_PATH) --with-libsodium=no --with-libgssapi_krb5=no && $(MAKE) && $(MAKE) install

  90.  

  91. -# # glog

  92. -# ${DEPS_PATH}/include/glog/logging.h: | ${DEPS_PATH}/include/google/gflags.h

  93. -# 	$(eval FILE=v0.3.4.tar.gz)

  94. -# 	$(eval DIR=glog-0.3.4)

  95. -# 	rm -rf $(FILE) $(DIR)

  96. -# 	wget https://github.com/google/glog/archive/$(FILE) && tar -zxf $(FILE)

  97. -# 	cd $(DIR) && ./configure -prefix=$(DEPS_PATH) --with-gflags=$(DEPS_PATH) && $(MAKE) && $(MAKE) install

  98. -# 	rm -rf $(FILE) $(DIR)

  99. -# glog: | ${DEPS_PATH}/include/glog/logging.h

  100. diff -Npur ps-lite-master/make/ps.mk ps-lite-master-new/make/ps.mk

  101. --- ps-lite-master/make/ps.mk	2020-02-29 13:59:55.000000000 +0800

  102. +++ ps-lite-master-new/make/ps.mk	2020-06-05 09:28:35.337740291 +0800

  103. @@ -9,5 +9,5 @@ ifeq ($(USE_KEY32), 1)

  104.  ADD_CFLAGS += -DUSE_KEY32=1

  105.  endif

  106.  

  107. -PS_LDFLAGS_SO = -L$(DEPS_PATH)/lib -lprotobuf-lite -lzmq

  108. -PS_LDFLAGS_A = $(addprefix $(DEPS_PATH)/lib/, libprotobuf-lite.a libzmq.a)

  109. +PS_LDFLAGS_SO = -L$(MS_ZMQ_INSTALL_PATH)/lib -lzmq -L$(MS_PROTO_LIB_DIR) -lprotobuf-lite

  110. +PS_LDFLAGS_A = $(addprefix $(MS_ZMQ_INSTALL_PATH)/lib -L$(MS_PROTO_LIB_DIR), libprotobuf-lite.a libzmq.a)

  111. diff -Npur ps-lite-master/Makefile ps-lite-master-new/Makefile

  112. --- ps-lite-master/Makefile	2020-02-29 13:59:55.000000000 +0800

  113. +++ ps-lite-master-new/Makefile	2020-06-17 11:09:20.240322660 +0800

  114. @@ -12,13 +12,24 @@ ifndef DEPS_PATH

  115.  DEPS_PATH = $(shell pwd)/deps

  116.  endif

  117.  

  118. +MS_PROTO_DIR = @protobuf_DIRPATH@

  119. +MS_GLOG_DIR = @glog_DIRPATH@

  120. +MS_ZMQ_DIR = @zeromq_DIRPATH@

  121. +

  122. +MS_PROTO_LIB_DIR = @protobuf_LIBPATH@

  123. +MS_GLOG_LIB_DIR = @glog_LIBPATH@

  124. +MS_ZMQ_INSTALL_PATH = $(MS_ZMQ_DIR)/zmq_install

  125.  

  126.  ifndef PROTOC

  127. -PROTOC = ${DEPS_PATH}/bin/protoc

  128. +PROTOC = $(MS_PROTO_DIR)/bin/protoc

  129.  endif

  130.  

  131. -INCPATH = -I./src -I./include -I$(DEPS_PATH)/include

  132. -CFLAGS = -std=c++11 -msse2 -fPIC -O3 -ggdb -Wall -finline-functions $(INCPATH) $(ADD_CFLAGS)

  133. +INCPATH = -I./src -I./include -I$(MS_ZMQ_INSTALL_PATH)/include

  134. +INCPATH += -I$(MS_PROTO_DIR)/include

  135. +INCPATH += -I$(MS_GLOG_DIR)/include

  136. +

  137. +CXXFLAGS = -D_GLIBCXX_USE_CXX11_ABI=0

  138. +CFLAGS = -std=c++11 -fPIC -O3 -ggdb -Wall -finline-functions $(INCPATH) $(ADD_CFLAGS) -D_GLIBCXX_USE_CXX11_ABI=0

  139.  LIBS = -pthread

  140.  

  141.  ifdef USE_IBVERBS

  142. @@ -30,6 +41,7 @@ ifdef ASAN

  143.  CFLAGS += -fsanitize=address -fno-omit-frame-pointer -fno-optimize-sibling-calls

  144.  endif

  145.  

  146. +LIBS += -L$(MS_GLOG_LIB_DIR) -lglog

  147.  

  148.  all: ps test

  149.  

  150. @@ -51,9 +63,9 @@ build/libps.a: $(OBJS)

  151.  build/%.o: src/%.cc ${ZMQ} src/meta.pb.h

  152.  	@mkdir -p $(@D)

  153.  	$(CXX) $(INCPATH) -std=c++11 -MM -MT build/$*.o $< >build/$*.d

  154. -	$(CXX) $(CFLAGS) $(LIBS) -c $< -o $@

  155. +	$(CXX) $(CFLAGS) $(CXXFLAGS) $(LIBS) -c $< -o $@

  156.  

  157. -src/%.pb.cc src/%.pb.h : src/%.proto ${PROTOBUF}

  158. +src/%.pb.cc src/%.pb.h : src/%.proto

  159.  	$(PROTOC) --cpp_out=./src --proto_path=./src $<

  160.  

  161.  -include build/*.d

  162. diff -Npur ps-lite-master/src/ibverbs_van.h ps-lite-master-new/src/ibverbs_van.h

  163. --- ps-lite-master/src/ibverbs_van.h	2020-02-29 13:59:55.000000000 +0800

  164. +++ ps-lite-master-new/src/ibverbs_van.h	2020-06-02 20:52:11.076230014 +0800

  165. @@ -145,15 +145,15 @@ class SimpleMempool {

  166.        total_allocated_size += new_mem_size;

  167.      }

  168.  

  169. -    CHECK_NE(free_list.end(), it) << "Not enough memory";

  170. +    //CHECK_NE(free_list.end(), it) << "Not enough memory";

  171.      CHECK_GE(it->first, proper_size);

  172.  

  173.      char *addr = it->second;

  174.      size_t space_left = it->first - proper_size;

  175.  

  176.      free_list.erase(it);

  177. -    CHECK_EQ(used_list.find(addr), used_list.end())

  178. -        << "Address is already allocated";

  179. +    //CHECK_EQ(used_list.find(addr), used_list.end())

  180. +        //<< "Address is already allocated";

  181.  

  182.      used_list.emplace(addr, proper_size);

  183.  

  184. @@ -173,8 +173,8 @@ class SimpleMempool {

  185.      std::lock_guard<std::mutex> lk(mu_);

  186.  

  187.      auto it = used_list.find(addr);

  188. -    CHECK_NE(used_list.end(), it)

  189. -        << "Cannot find info about address: " << (uintptr_t)addr;

  190. +    //CHECK_NE(used_list.end(), it)

  191. +        //<< "Cannot find info about address: " << (uintptr_t)addr;

  192.  

  193.      size_t size = it->second;

  194.      used_list.erase(it);

  195. @@ -208,7 +208,7 @@ class SimpleMempool {

  196.    // Convert the memory address to its associated RDMA memory region

  197.    inline struct ibv_mr *Addr2MR(char *addr) {

  198.      auto it = mr_list.lower_bound(addr);

  199. -    CHECK_NE(it, mr_list.end()) << "cannot find the associated memory region";

  200. +    //CHECK_NE(it, mr_list.end()) << "cannot find the associated memory region";

  201.      return it->second;

  202.    }

  203.  };

  204. @@ -330,7 +330,7 @@ class AddressPool {

  205.      CHECK(ptr);

  206.      uint32_t idx = indices_.front();

  207.      indices_.pop();

  208. -    CHECK_EQ(table_[idx], nullptr);

  209. +    //CHECK_EQ(table_[idx], nullptr);

  210.      table_[idx] = ptr;

  211.      return idx;

  212.    }

  213. @@ -636,7 +636,7 @@ class IBVerbsVan : public Van {

  214.      PBMeta meta;

  215.      PackMetaPB(msg.meta, &meta);

  216.  

  217. -    CHECK_NE(endpoints_.find(remote_id), endpoints_.end());

  218. +    //CHECK_NE(endpoints_.find(remote_id), endpoints_.end());

  219.      Endpoint *endpoint = endpoints_[remote_id].get();

  220.      MessageBuffer *msg_buf = new MessageBuffer();

  221.  

  222. diff -Npur ps-lite-master/src/van.cc ps-lite-master-new/src/van.cc

  223. --- ps-lite-master/src/van.cc	2020-02-29 13:59:55.000000000 +0800

  224. +++ ps-lite-master-new/src/van.cc	2020-06-02 20:52:43.330405828 +0800

  225. @@ -448,6 +448,7 @@ void Van::PackMetaPB(const Meta& meta, P

  226.    if (meta.timestamp != Meta::kEmpty) pb->set_timestamp(meta.timestamp);

  227.    if (meta.body.size()) pb->set_body(meta.body);

  228.    pb->set_push(meta.push);

  229. +  pb->set_pull(meta.pull);

  230.    pb->set_request(meta.request);

  231.    pb->set_simple_app(meta.simple_app);

  232.    pb->set_priority(meta.priority);

  233. diff -Npur ps-lite-master/tests/test.mk ps-lite-master-new/tests/test.mk

  234. --- ps-lite-master/tests/test.mk	2020-02-29 13:59:55.000000000 +0800

  235. +++ ps-lite-master-new/tests/test.mk	2020-06-16 19:15:06.025087897 +0800

  236. @@ -1,10 +1,10 @@

  237. -TEST_SRC = $(wildcard tests/test_*.cc)

  238. -TEST = $(patsubst tests/test_%.cc, tests/test_%, $(TEST_SRC))

  239. +#TEST_SRC = $(wildcard tests/test_*.cc)

  240. +#TEST = $(patsubst tests/test_%.cc, tests/test_%, $(TEST_SRC))

  241.  

  242. -# -ltcmalloc_and_profiler

  243. -LDFLAGS = -Wl,-rpath,$(DEPS_PATH)/lib $(PS_LDFLAGS_SO) -pthread

  244. -tests/% : tests/%.cc build/libps.a

  245. -	$(CXX) $(CFLAGS) $(LIBS) -MM -MT tests/$* $< >tests/$*.d

  246. -	$(CXX) $(CFLAGS) $(LIBS) -o $@ $(filter %.cc %.a, $^) $(LDFLAGS)

  247. -

  248. --include tests/*.d

  249. +## -ltcmalloc_and_profiler

  250. +#LDFLAGS = -Wl,-rpath,$(DEPS_PATH)/lib $(PS_LDFLAGS_SO) -pthread

  251. +#tests/% : tests/%.cc build/libps.a

  252. +#	$(CXX) $(CFLAGS) $(LIBS) -MM -MT tests/$* $< >tests/$*.d

  253. +#	$(CXX) $(CFLAGS) $(LIBS) -o $@ $(filter %.cc %.a, $^) $(LDFLAGS)

  254. +#

  255. +#-include tests/*.d