Adjust log level for cache server/admin/perf

and fix cache perf tool compilation

mindspore/ccsrc/minddata/dataset/engine/cache/CMakeLists.txt

@@ -27,7 +27,7 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
   endif()
 endif()
 
-if(NUMA_FOUND)
+if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
   ADD_DEFINITIONS(-DCACHE_LOCAL_CLIENT)
 endif()
 

mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin.cc

@@ -15,12 +15,10 @@
 */
 #include <unistd.h>
 #include <iostream>
-#ifdef USE_GLOG
-#include <glog/logging.h>
-#endif
 #include "minddata/dataset/engine/cache/cache_admin_arg.h"
 #include "minddata/dataset/engine/cache/cache_common.h"
 #include "minddata/dataset/util/path.h"
+#include "mindspore/core/utils/log_adapter.h"
 
 namespace ms = mindspore;
 namespace ds = mindspore::dataset;
@@ -31,7 +29,7 @@ int main(int argc, char **argv) {
   std::stringstream arg_stream;
 
 #ifdef USE_GLOG
-  FLAGS_minloglevel = google::WARNING;
+  FLAGS_logtostderr = false;
   FLAGS_log_dir = ds::DefaultLogDir();
   // Create default log dir
   ds::Path log_dir = ds::Path(FLAGS_log_dir);

mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin_arg.cc

@@ -63,6 +63,15 @@ CacheAdminArgHandler::CacheAdminArgHandler()
       port_ = 0;  // cause the port range validation to generate an error during the validation checks
     }
   }
+  const char *env_log_level = std::getenv("GLOG_v");
+  if (env_log_level != nullptr) {
+    char *end = nullptr;
+    log_level_ = strtol(env_log_level, &end, 10);
+    if (*end != '\0') {
+      std::cerr << "Log level from env variable GLOG_v is invalid\n";
+      log_level_ = -1;  // cause the log level range validation to generate an error during the validation checks
+    }
+  }
   // Initialize the command mappings
   arg_map_["-h"] = ArgValue::kArgHost;
   arg_map_["--hostname"] = ArgValue::kArgHost;
@@ -333,7 +342,7 @@ Status CacheAdminArgHandler::Validate() {
   if (num_workers_ < 1 || num_workers_ > max_num_workers)
     return Status(StatusCode::kMDSyntaxError,
                   "Number of workers must be in range of 1 and " + std::to_string(max_num_workers) + ".");
-  if (log_level_ < 0 || log_level_ > 3) return Status(StatusCode::kMDSyntaxError, "Log level must be in range (0..3).");
+  if (log_level_ < 0 || log_level_ > 4) return Status(StatusCode::kMDSyntaxError, "Log level must be in range (0..4).");
   if (memory_cap_ratio_ <= 0 || memory_cap_ratio_ > 1)
     return Status(StatusCode::kMDSyntaxError, "Memory cap ratio should be positive and no greater than 1");
   if (port_ < 1025 || port_ > 65535) return Status(StatusCode::kMDSyntaxError, "Port must be in range (1025..65535).");
@@ -608,7 +617,7 @@ void CacheAdminArgHandler::Help() {
   std::cerr << "                [[-p | --port] <port number>]             Default is " << kCfgDefaultCachePort << ".\n";
   std::cerr << "                [[-w | --workers] <number of workers>]    Default is " << kDefaultNumWorkers << ".\n";
   std::cerr << "                [[-s | --spilldir] <spilling directory>]  Default is no spilling.\n";
-  std::cerr << "                [[-l | --loglevel] <log level>]           Default is 1 (warning level).\n";
+  std::cerr << "                [[-l | --loglevel] <log level>]           Default is 1 (INFO level).\n";
   std::cerr << "            [--destroy_session  | -d] <session id>\n";
   std::cerr << "                [[-p | --port] <port number>]\n";
   std::cerr << "            [--generate_session | -g]\n";

mindspore/ccsrc/minddata/dataset/engine/cache/cache_hw.cc

@@ -32,7 +32,7 @@ CacheServerHW::CacheServerHW() {
   MS_LOG(DEBUG) << "Number of cpu(s) : " << num_cpus_;
 #ifdef NUMA_ENABLED
   if (numa_enabled()) {
-    MS_LOG(WARNING) << "Numa support enabled";
+    MS_LOG(INFO) << "Numa support enabled";
     for (auto i = 0; i <= numa_max_node(); ++i) {
       int64_t free_avail;
       int64_t mem_avail = numa_node_size(i, &free_avail);

mindspore/ccsrc/minddata/dataset/engine/cache/cache_main.cc

@@ -17,14 +17,12 @@
 #include "minddata/dataset/engine/cache/cache_server.h"
 #include <sys/types.h>
 #include <unistd.h>
-#ifdef USE_GLOG
-#include <glog/logging.h>
-#endif
 #include <cstdlib>
 #include <thread>
 #include <chrono>
 #include "minddata/dataset/engine/cache/cache_common.h"
 #include "minddata/dataset/engine/cache/cache_ipc.h"
+#include "mindspore/core/utils/log_adapter.h"
 namespace ms = mindspore;
 namespace ds = mindspore::dataset;
 
@@ -61,8 +59,7 @@ ms::Status StartServer(int argc, char **argv) {
   ds::SharedMessage msg;
   if (daemonize) {
 #ifdef USE_GLOG
-    // temporary setting log level to WARNING to avoid logging when creating dir
-    FLAGS_minloglevel = google::WARNING;
+    FLAGS_logtostderr = false;
     FLAGS_log_dir = ds::DefaultLogDir();
     // Create cache server default log dir
     ds::Path log_dir = ds::Path(FLAGS_log_dir);
@@ -70,7 +67,7 @@ ms::Status StartServer(int argc, char **argv) {
     if (rc.IsError()) {
       return rc;
     }
-    FLAGS_minloglevel = strtol(argv[5], nullptr, 10);
+    ms::g_ms_submodule_log_levels[SUBMODULE_ID] = strtol(argv[5], nullptr, 10);
     google::InitGoogleLogging(argv[0]);
 #endif
     rc = msg.Create();
@@ -120,8 +117,8 @@ ms::Status StartServer(int argc, char **argv) {
   }
 
   // Dump the summary
-  MS_LOG(WARNING) << "Cache server has started successfully and is listening on port " << port << std::endl;
-  MS_LOG(WARNING) << builder << std::endl;
+  MS_LOG(INFO) << "Cache server has started successfully and is listening on port " << port << std::endl;
+  MS_LOG(INFO) << builder << std::endl;
   // Create the instance with some sanity checks built in
   rc = builder.Build();
   if (rc.IsOk()) {

mindspore/ccsrc/minddata/dataset/engine/cache/cache_server.cc

@@ -669,7 +669,7 @@ Status CacheServer::GetCacheMissKeys(CacheRequest *rq, CacheReply *reply) {
 
 inline Status GenerateClientSessionID(session_id_type session_id, CacheReply *reply) {
   reply->set_result(std::to_string(session_id));
-  MS_LOG(WARNING) << "Server generated new session id " << session_id;
+  MS_LOG(INFO) << "Server generated new session id " << session_id;
   return Status::OK();
 }
 
@@ -1081,7 +1081,7 @@ Status CacheServer::DestroySession(CacheRequest *rq) {
   // Finally remove the session itself
   auto n = active_sessions_.erase(drop_session_id);
   if (n > 0) {
-    MS_LOG(WARNING) << "Session destroyed with id " << drop_session_id;
+    MS_LOG(INFO) << "Session destroyed with id " << drop_session_id;
     return Status::OK();
   } else {
     if (found) {

mindspore/ccsrc/minddata/dataset/engine/cache/perf/CMakeLists.txt

@@ -8,6 +8,7 @@ if(ENABLE_CACHE)
   target_link_libraries(cache_perf
       _c_dataengine
       _c_mindrecord
+      mindspore
       mindspore::protobuf
       mindspore_gvar
       ${PYTHON_LIBRARIES}
@@ -21,6 +22,7 @@ if(ENABLE_CACHE)
   target_link_libraries(cache_pipeline
       _c_dataengine
       _c_mindrecord
+      mindspore
       mindspore::protobuf
       mindspore_gvar
       ${PYTHON_LIBRARIES}

mindspore/ccsrc/minddata/dataset/engine/cache/perf/cache_perf.cc

@@ -14,22 +14,22 @@
  * limitations under the License.
 */
 
-#ifdef USE_GLOG
-#include <glog/logging.h>
-#endif
-#include <iostream>
 #include "minddata/dataset/engine/cache/perf/cache_perf_run.h"
+#include <iostream>
+#include "mindspore/core/utils/log_adapter.h"
+namespace ms = mindspore;
 namespace ds = mindspore::dataset;
 
 int main(int argc, char **argv) {
 #ifdef USE_GLOG
+  FLAGS_logtostderr = false;
   FLAGS_log_dir = "/tmp";
   google::InitGoogleLogging(argv[0]);
 #endif
   ds::CachePerfRun cachePerfRun;
   if (cachePerfRun.ProcessArgs(argc, argv) == 0) {
     std::cout << cachePerfRun << std::endl;
-    ds::Status rc = cachePerfRun.Run();
+    ms::Status rc = cachePerfRun.Run();
     if (rc.IsError()) {
       std::cerr << rc.ToString() << std::endl;
     }

mindspore/ccsrc/minddata/dataset/engine/cache/perf/cache_perf_run.cc

@@ -548,7 +548,7 @@ Status CachePerfRun::Run() {
   RETURN_IF_NOT_OK(cache_builder_.Build(&cc_));
   Status rc = cc_->CreateCache(crc_, false);
   // Duplicate key is fine.
-  if (rc.IsError() && rc.get_code() != StatusCode::kDuplicateKey) {
+  if (rc.IsError() && rc != StatusCode::kMDDuplicateKey) {
     return rc;
   }
 

mindspore/ccsrc/minddata/dataset/engine/cache/perf/cache_pipeline.cc

@@ -14,22 +14,22 @@
  * limitations under the License.
 */
 
-#ifdef USE_GLOG
-#include <glog/logging.h>
-#endif
-#include <string.h>
 #include "minddata/dataset/engine/cache/perf/cache_pipeline_run.h"
+#include <string.h>
+#include "mindspore/core/utils/log_adapter.h"
+
+namespace ms = mindspore;
 namespace ds = mindspore::dataset;
 
 int main(int argc, char **argv) {
 #ifdef USE_GLOG
+  FLAGS_logtostderr = false;
   FLAGS_log_dir = "/tmp";
-  FLAGS_minloglevel = google::WARNING;
   google::InitGoogleLogging(argv[0]);
 #endif
   ds::CachePipelineRun cachePipelineRun;
   if (cachePipelineRun.ProcessArgs(argc, argv) == 0) {
-    ds::Status rc = cachePipelineRun.Run();
+    ms::Status rc = cachePipelineRun.Run();
     // If we hit any error, send the rc back to the parent.
     if (rc.IsError()) {
       ds::ErrorMsg proto;

mindspore/ccsrc/minddata/dataset/engine/cache/perf/cache_pipeline_run.cc

@@ -177,7 +177,7 @@ Status CachePipelineRun::Run() {
 
   Status rc = cc_->CreateCache(crc_, false);
   // Duplicate key is fine.
-  if (rc.IsError() && rc.get_code() != StatusCode::kDuplicateKey) {
+  if (rc.IsError() && rc != StatusCode::kMDDuplicateKey) {
     return rc;
   }
 
@@ -313,7 +313,7 @@ Status CachePipelineRun::WriterWorkerEntry(int32_t worker_id) {
       rc = cc_->AsyncWriteBuffer(std::move(buffer));
       auto end_tick = std::chrono::steady_clock::now();
       if (rc.IsError()) {
-        if (rc.IsOutofMemory() || rc.IsNoSpace()) {
+        if (rc == StatusCode::kMDOutOfMemory || rc == StatusCode::kMDNoSpace) {
           MS_LOG(WARNING) << "Pipeline number " << my_pipeline_ + 1 << " worker id " << worker_id << ": "
                           << rc.ToString();
           resource_err = true;