|
|
|
@@ -0,0 +1,264 @@ |
|
|
|
diff -Npur ps-lite-master/include/dmlc/base.h ps-lite-master-new/include/dmlc/base.h |
|
|
|
--- ps-lite-master/include/dmlc/base.h 2020-02-29 13:59:55.000000000 +0800 |
|
|
|
+++ ps-lite-master-new/include/dmlc/base.h 2020-07-01 11:56:50.444833389 +0800 |
|
|
|
@@ -8,7 +8,7 @@ |
|
|
|
|
|
|
|
/*! \brief whether use glog for logging */ |
|
|
|
#ifndef DMLC_USE_GLOG |
|
|
|
-#define DMLC_USE_GLOG 0 |
|
|
|
+#define DMLC_USE_GLOG 1 |
|
|
|
#endif |
|
|
|
|
|
|
|
/*! |
|
|
|
diff -Npur ps-lite-master/include/dmlc/logging.h ps-lite-master-new/include/dmlc/logging.h |
|
|
|
--- ps-lite-master/include/dmlc/logging.h 2020-02-29 13:59:55.000000000 +0800 |
|
|
|
+++ ps-lite-master-new/include/dmlc/logging.h 2020-07-01 11:58:00.015919207 +0800 |
|
|
|
@@ -13,7 +13,7 @@ |
|
|
|
#include <string> |
|
|
|
#include <vector> |
|
|
|
#include <memory> |
|
|
|
-#include "./base.h" |
|
|
|
+//#include "./base.h" |
|
|
|
|
|
|
|
#if DMLC_LOG_STACK_TRACE |
|
|
|
#include <cxxabi.h> |
|
|
|
@@ -52,7 +52,7 @@ struct Error : public std::runtime_error |
|
|
|
|
|
|
|
namespace dmlc { |
|
|
|
inline void InitLogging(const char* argv0) { |
|
|
|
- google::InitGoogleLogging(argv0); |
|
|
|
+ //google::InitGoogleLogging(argv0); |
|
|
|
} |
|
|
|
} // namespace dmlc |
|
|
|
|
|
|
|
diff -Npur ps-lite-master/make/deps.mk ps-lite-master-new/make/deps.mk |
|
|
|
--- ps-lite-master/make/deps.mk 2020-02-29 13:59:55.000000000 +0800 |
|
|
|
+++ ps-lite-master-new/make/deps.mk 2020-06-17 10:35:46.253837426 +0800 |
|
|
|
@@ -1,69 +1,7 @@ |
|
|
|
# Install dependencies |
|
|
|
- |
|
|
|
-URL1=https://raw.githubusercontent.com/mli/deps/master/build |
|
|
|
-URL2=https://github.com/google/protobuf/releases/download/v3.5.1 |
|
|
|
-ifndef WGET |
|
|
|
-WGET = wget |
|
|
|
-endif |
|
|
|
- |
|
|
|
-# protobuf |
|
|
|
-PROTOBUF = ${DEPS_PATH}/include/google/protobuf/message.h |
|
|
|
-${PROTOBUF}: |
|
|
|
- $(eval FILE=protobuf-cpp-3.5.1.tar.gz) |
|
|
|
- $(eval DIR=protobuf-3.5.1) |
|
|
|
- rm -rf $(FILE) $(DIR) |
|
|
|
- $(WGET) $(URL2)/$(FILE) && tar --no-same-owner -zxf $(FILE) |
|
|
|
- cd $(DIR) && export CFLAGS=-fPIC && export CXXFLAGS=-fPIC && ./configure -prefix=$(DEPS_PATH) && $(MAKE) && $(MAKE) install |
|
|
|
- rm -rf $(FILE) $(DIR) |
|
|
|
- |
|
|
|
# zmq |
|
|
|
-ZMQ = ${DEPS_PATH}/include/zmq.h |
|
|
|
+ZMQ = $(MS_ZMQ_INSTALL_PATH)/lib/libzmq.a |
|
|
|
|
|
|
|
${ZMQ}: |
|
|
|
- $(eval FILE=zeromq-4.1.4.tar.gz) |
|
|
|
- $(eval DIR=zeromq-4.1.4) |
|
|
|
- rm -rf $(FILE) $(DIR) |
|
|
|
- $(WGET) $(URL1)/$(FILE) && tar --no-same-owner -zxf $(FILE) |
|
|
|
- cd $(DIR) && export CFLAGS=-fPIC && export CXXFLAGS=-fPIC && ./configure -prefix=$(DEPS_PATH) --with-libsodium=no --with-libgssapi_krb5=no && $(MAKE) && $(MAKE) install |
|
|
|
- rm -rf $(FILE) $(DIR) |
|
|
|
- |
|
|
|
-# lz4 |
|
|
|
-LZ4 = ${DEPS_PATH}/include/lz4.h |
|
|
|
-${LZ4}: |
|
|
|
- $(eval FILE=lz4-r129.tar.gz) |
|
|
|
- $(eval DIR=lz4-r129) |
|
|
|
- rm -rf $(FILE) $(DIR) |
|
|
|
- wget $(URL1)/$(FILE) && tar --no-same-owner -zxf $(FILE) |
|
|
|
- cd $(DIR) && $(MAKE) && PREFIX=$(DEPS_PATH) $(MAKE) install |
|
|
|
- rm -rf $(FILE) $(DIR) |
|
|
|
- |
|
|
|
-# cityhash |
|
|
|
-CITYHASH = ${DEPS_PATH}/include/city.h |
|
|
|
-${CITYHASH}: |
|
|
|
- $(eval FILE=cityhash-1.1.1.tar.gz) |
|
|
|
- $(eval DIR=cityhash-1.1.1) |
|
|
|
- rm -rf $(FILE) $(DIR) |
|
|
|
- wget $(URL1)/$(FILE) && tar --no-same-owner -zxf $(FILE) |
|
|
|
- cd $(DIR) && ./configure -prefix=$(DEPS_PATH) --enable-sse4.2 && $(MAKE) CXXFLAGS="-g -O3 -msse4.2" && $(MAKE) install |
|
|
|
- rm -rf $(FILE) $(DIR) |
|
|
|
- |
|
|
|
- |
|
|
|
-# # gflags |
|
|
|
-# ${DEPS_PATH}/include/google/gflags.h: |
|
|
|
-# $(eval FILE=gflags-2.0-no-svn-files.tar.gz) |
|
|
|
-# $(eval DIR=gflags-2.0) |
|
|
|
-# rm -rf $(FILE) $(DIR) |
|
|
|
-# wget $(URL)/$(FILE) && tar -zxf $(FILE) |
|
|
|
-# cd $(DIR) && ./configure -prefix=$(DEPS_PATH) && $(MAKE) && $(MAKE) install |
|
|
|
-# rm -rf $(FILE) $(DIR) |
|
|
|
-# gflags: | ${DEPS_PATH}/include/google/gflags.h |
|
|
|
+ cd $(MS_ZMQ_DIR) && export CFLAGS="-fPIC -D_GLIBCXX_USE_CXX11_ABI=0" && export CXXFLAGS=-fPIC && ./configure -prefix=$(MS_ZMQ_INSTALL_PATH) --with-libsodium=no --with-libgssapi_krb5=no && $(MAKE) && $(MAKE) install |
|
|
|
|
|
|
|
-# # glog |
|
|
|
-# ${DEPS_PATH}/include/glog/logging.h: | ${DEPS_PATH}/include/google/gflags.h |
|
|
|
-# $(eval FILE=v0.3.4.tar.gz) |
|
|
|
-# $(eval DIR=glog-0.3.4) |
|
|
|
-# rm -rf $(FILE) $(DIR) |
|
|
|
-# wget https://github.com/google/glog/archive/$(FILE) && tar -zxf $(FILE) |
|
|
|
-# cd $(DIR) && ./configure -prefix=$(DEPS_PATH) --with-gflags=$(DEPS_PATH) && $(MAKE) && $(MAKE) install |
|
|
|
-# rm -rf $(FILE) $(DIR) |
|
|
|
-# glog: | ${DEPS_PATH}/include/glog/logging.h |
|
|
|
diff -Npur ps-lite-master/make/ps.mk ps-lite-master-new/make/ps.mk |
|
|
|
--- ps-lite-master/make/ps.mk 2020-02-29 13:59:55.000000000 +0800 |
|
|
|
+++ ps-lite-master-new/make/ps.mk 2020-06-05 09:28:35.337740291 +0800 |
|
|
|
@@ -9,5 +9,5 @@ ifeq ($(USE_KEY32), 1) |
|
|
|
ADD_CFLAGS += -DUSE_KEY32=1 |
|
|
|
endif |
|
|
|
|
|
|
|
-PS_LDFLAGS_SO = -L$(DEPS_PATH)/lib -lprotobuf-lite -lzmq |
|
|
|
-PS_LDFLAGS_A = $(addprefix $(DEPS_PATH)/lib/, libprotobuf-lite.a libzmq.a) |
|
|
|
+PS_LDFLAGS_SO = -L$(MS_ZMQ_INSTALL_PATH)/lib -lzmq -L$(MS_PROTO_LIB_DIR) -lprotobuf-lite |
|
|
|
+PS_LDFLAGS_A = $(addprefix $(MS_ZMQ_INSTALL_PATH)/lib -L$(MS_PROTO_LIB_DIR), libprotobuf-lite.a libzmq.a) |
|
|
|
diff -Npur ps-lite-master/Makefile ps-lite-master-new/Makefile |
|
|
|
--- ps-lite-master/Makefile 2020-02-29 13:59:55.000000000 +0800 |
|
|
|
+++ ps-lite-master-new/Makefile 2020-06-17 11:09:20.240322660 +0800 |
|
|
|
@@ -12,13 +12,24 @@ ifndef DEPS_PATH |
|
|
|
DEPS_PATH = $(shell pwd)/deps |
|
|
|
endif |
|
|
|
|
|
|
|
+MS_PROTO_DIR = @protobuf_DIRPATH@ |
|
|
|
+MS_GLOG_DIR = @glog_DIRPATH@ |
|
|
|
+MS_ZMQ_DIR = @zeromq_DIRPATH@ |
|
|
|
+ |
|
|
|
+MS_PROTO_LIB_DIR = @protobuf_LIBPATH@ |
|
|
|
+MS_GLOG_LIB_DIR = @glog_LIBPATH@ |
|
|
|
+MS_ZMQ_INSTALL_PATH = $(MS_ZMQ_DIR)/zmq_install |
|
|
|
|
|
|
|
ifndef PROTOC |
|
|
|
-PROTOC = ${DEPS_PATH}/bin/protoc |
|
|
|
+PROTOC = $(MS_PROTO_DIR)/bin/protoc |
|
|
|
endif |
|
|
|
|
|
|
|
-INCPATH = -I./src -I./include -I$(DEPS_PATH)/include |
|
|
|
-CFLAGS = -std=c++11 -msse2 -fPIC -O3 -ggdb -Wall -finline-functions $(INCPATH) $(ADD_CFLAGS) |
|
|
|
+INCPATH = -I./src -I./include -I$(MS_ZMQ_INSTALL_PATH)/include |
|
|
|
+INCPATH += -I$(MS_PROTO_DIR)/include |
|
|
|
+INCPATH += -I$(MS_GLOG_DIR)/include |
|
|
|
+ |
|
|
|
+CXXFLAGS = -D_GLIBCXX_USE_CXX11_ABI=0 |
|
|
|
+CFLAGS = -std=c++11 -fPIC -O3 -ggdb -Wall -finline-functions $(INCPATH) $(ADD_CFLAGS) -D_GLIBCXX_USE_CXX11_ABI=0 |
|
|
|
LIBS = -pthread |
|
|
|
|
|
|
|
ifdef USE_IBVERBS |
|
|
|
@@ -30,6 +41,7 @@ ifdef ASAN |
|
|
|
CFLAGS += -fsanitize=address -fno-omit-frame-pointer -fno-optimize-sibling-calls |
|
|
|
endif |
|
|
|
|
|
|
|
+LIBS += -L$(MS_GLOG_LIB_DIR) -lglog |
|
|
|
|
|
|
|
all: ps test |
|
|
|
|
|
|
|
@@ -51,9 +63,9 @@ build/libps.a: $(OBJS) |
|
|
|
build/%.o: src/%.cc ${ZMQ} src/meta.pb.h |
|
|
|
@mkdir -p $(@D) |
|
|
|
$(CXX) $(INCPATH) -std=c++11 -MM -MT build/$*.o $< >build/$*.d |
|
|
|
- $(CXX) $(CFLAGS) $(LIBS) -c $< -o $@ |
|
|
|
+ $(CXX) $(CFLAGS) $(CXXFLAGS) $(LIBS) -c $< -o $@ |
|
|
|
|
|
|
|
-src/%.pb.cc src/%.pb.h : src/%.proto ${PROTOBUF} |
|
|
|
+src/%.pb.cc src/%.pb.h : src/%.proto |
|
|
|
$(PROTOC) --cpp_out=./src --proto_path=./src $< |
|
|
|
|
|
|
|
-include build/*.d |
|
|
|
diff -Npur ps-lite-master/src/ibverbs_van.h ps-lite-master-new/src/ibverbs_van.h |
|
|
|
--- ps-lite-master/src/ibverbs_van.h 2020-02-29 13:59:55.000000000 +0800 |
|
|
|
+++ ps-lite-master-new/src/ibverbs_van.h 2020-06-02 20:52:11.076230014 +0800 |
|
|
|
@@ -145,15 +145,15 @@ class SimpleMempool { |
|
|
|
total_allocated_size += new_mem_size; |
|
|
|
} |
|
|
|
|
|
|
|
- CHECK_NE(free_list.end(), it) << "Not enough memory"; |
|
|
|
+ //CHECK_NE(free_list.end(), it) << "Not enough memory"; |
|
|
|
CHECK_GE(it->first, proper_size); |
|
|
|
|
|
|
|
char *addr = it->second; |
|
|
|
size_t space_left = it->first - proper_size; |
|
|
|
|
|
|
|
free_list.erase(it); |
|
|
|
- CHECK_EQ(used_list.find(addr), used_list.end()) |
|
|
|
- << "Address is already allocated"; |
|
|
|
+ //CHECK_EQ(used_list.find(addr), used_list.end()) |
|
|
|
+ //<< "Address is already allocated"; |
|
|
|
|
|
|
|
used_list.emplace(addr, proper_size); |
|
|
|
|
|
|
|
@@ -173,8 +173,8 @@ class SimpleMempool { |
|
|
|
std::lock_guard<std::mutex> lk(mu_); |
|
|
|
|
|
|
|
auto it = used_list.find(addr); |
|
|
|
- CHECK_NE(used_list.end(), it) |
|
|
|
- << "Cannot find info about address: " << (uintptr_t)addr; |
|
|
|
+ //CHECK_NE(used_list.end(), it) |
|
|
|
+ //<< "Cannot find info about address: " << (uintptr_t)addr; |
|
|
|
|
|
|
|
size_t size = it->second; |
|
|
|
used_list.erase(it); |
|
|
|
@@ -208,7 +208,7 @@ class SimpleMempool { |
|
|
|
// Convert the memory address to its associated RDMA memory region |
|
|
|
inline struct ibv_mr *Addr2MR(char *addr) { |
|
|
|
auto it = mr_list.lower_bound(addr); |
|
|
|
- CHECK_NE(it, mr_list.end()) << "cannot find the associated memory region"; |
|
|
|
+ //CHECK_NE(it, mr_list.end()) << "cannot find the associated memory region"; |
|
|
|
return it->second; |
|
|
|
} |
|
|
|
}; |
|
|
|
@@ -330,7 +330,7 @@ class AddressPool { |
|
|
|
CHECK(ptr); |
|
|
|
uint32_t idx = indices_.front(); |
|
|
|
indices_.pop(); |
|
|
|
- CHECK_EQ(table_[idx], nullptr); |
|
|
|
+ //CHECK_EQ(table_[idx], nullptr); |
|
|
|
table_[idx] = ptr; |
|
|
|
return idx; |
|
|
|
} |
|
|
|
@@ -636,7 +636,7 @@ class IBVerbsVan : public Van { |
|
|
|
PBMeta meta; |
|
|
|
PackMetaPB(msg.meta, &meta); |
|
|
|
|
|
|
|
- CHECK_NE(endpoints_.find(remote_id), endpoints_.end()); |
|
|
|
+ //CHECK_NE(endpoints_.find(remote_id), endpoints_.end()); |
|
|
|
Endpoint *endpoint = endpoints_[remote_id].get(); |
|
|
|
MessageBuffer *msg_buf = new MessageBuffer(); |
|
|
|
|
|
|
|
diff -Npur ps-lite-master/src/van.cc ps-lite-master-new/src/van.cc |
|
|
|
--- ps-lite-master/src/van.cc 2020-02-29 13:59:55.000000000 +0800 |
|
|
|
+++ ps-lite-master-new/src/van.cc 2020-06-02 20:52:43.330405828 +0800 |
|
|
|
@@ -448,6 +448,7 @@ void Van::PackMetaPB(const Meta& meta, P |
|
|
|
if (meta.timestamp != Meta::kEmpty) pb->set_timestamp(meta.timestamp); |
|
|
|
if (meta.body.size()) pb->set_body(meta.body); |
|
|
|
pb->set_push(meta.push); |
|
|
|
+ pb->set_pull(meta.pull); |
|
|
|
pb->set_request(meta.request); |
|
|
|
pb->set_simple_app(meta.simple_app); |
|
|
|
pb->set_priority(meta.priority); |
|
|
|
diff -Npur ps-lite-master/tests/test.mk ps-lite-master-new/tests/test.mk |
|
|
|
--- ps-lite-master/tests/test.mk 2020-02-29 13:59:55.000000000 +0800 |
|
|
|
+++ ps-lite-master-new/tests/test.mk 2020-06-16 19:15:06.025087897 +0800 |
|
|
|
@@ -1,10 +1,10 @@ |
|
|
|
-TEST_SRC = $(wildcard tests/test_*.cc) |
|
|
|
-TEST = $(patsubst tests/test_%.cc, tests/test_%, $(TEST_SRC)) |
|
|
|
+#TEST_SRC = $(wildcard tests/test_*.cc) |
|
|
|
+#TEST = $(patsubst tests/test_%.cc, tests/test_%, $(TEST_SRC)) |
|
|
|
|
|
|
|
-# -ltcmalloc_and_profiler |
|
|
|
-LDFLAGS = -Wl,-rpath,$(DEPS_PATH)/lib $(PS_LDFLAGS_SO) -pthread |
|
|
|
-tests/% : tests/%.cc build/libps.a |
|
|
|
- $(CXX) $(CFLAGS) $(LIBS) -MM -MT tests/$* $< >tests/$*.d |
|
|
|
- $(CXX) $(CFLAGS) $(LIBS) -o $@ $(filter %.cc %.a, $^) $(LDFLAGS) |
|
|
|
- |
|
|
|
--include tests/*.d |
|
|
|
+## -ltcmalloc_and_profiler |
|
|
|
+#LDFLAGS = -Wl,-rpath,$(DEPS_PATH)/lib $(PS_LDFLAGS_SO) -pthread |
|
|
|
+#tests/% : tests/%.cc build/libps.a |
|
|
|
+# $(CXX) $(CFLAGS) $(LIBS) -MM -MT tests/$* $< >tests/$*.d |
|
|
|
+# $(CXX) $(CFLAGS) $(LIBS) -o $@ $(filter %.cc %.a, $^) $(LDFLAGS) |
|
|
|
+# |
|
|
|
+#-include tests/*.d |
mindspore-ci-bot
Gitee
5 years ago