#!/usr/bin/env python3 # coding=UTF-8 # Copyright 2020 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """ Function: Use to control the federated learning cluster Usage: python fl_restful_tool.py [http_type] [ip] [port] [request_name] [server_num] [instance_param] [metrics_file_path] """ import argparse import json import os import warnings import requests class Status: success = "0" failed = "1" class Restful: """ Define restful interface constant """ scale = "scale" scaleout = "scaleout" scalein = "scalein" nodes = "nodes" getInstanceDetail = "getInstanceDetail" newInstance = "newInstance" queryInstance = "queryInstance" enableFLS = "enableFLS" disableFLS = "disableFLS" state = "state" scaleoutRollback = "scaleoutRollback" warnings.filterwarnings('ignore') parser = argparse.ArgumentParser() parser.add_argument("--http_type", type=str, default="http", help="http or https") parser.add_argument("--ip", type=str, default="127.0.0.1") parser.add_argument("--port", type=int, default=6666) # scaleout scalein nodes parser.add_argument("--request_name", type=str, default="") parser.add_argument("--server_num", type=int, default=0) # "start_fl_job_threshold=20,start_fl_job_time_window=2000..." parser.add_argument("--instance_param", type=str, default="") parser.add_argument("--metrics_file_path", type=str, default="/opt/huawei/mindspore/hybrid_albert/metrics.json") args, _ = parser.parse_known_args() http_type = args.http_type ip = args.ip port = args.port request_name = args.request_name server_num = args.server_num instance_param = args.instance_param metrics_file_path = args.metrics_file_path headers = {'Content-Type': 'application/json'} session = requests.Session() base_url = http_type + "://" + ip + ":" + str(port) + "/" def call_scale(): """ call cluster scale out or scale in """ if server_num == 0: return process_self_define_json(Status.failed, "error. server_num is 0") node_ids = json.loads(call_nodes())["result"] cluster_abstract_node_num = len(node_ids) if cluster_abstract_node_num == 0: return process_self_define_json(Status.failed, "error. cluster abstract node num is 0") cluster_server_node_num = 0 cluster_worker_node_num = 0 cluster_server_node_base_name = '' for i in range(0, cluster_abstract_node_num): if node_ids[i]['role'] == 'WORKER': cluster_worker_node_num = cluster_worker_node_num + 1 elif node_ids[i]['role'] == 'SERVER': cluster_server_node_num = cluster_server_node_num + 1 cluster_server_node_name = str(node_ids[i]['nodeId']) index = cluster_server_node_name.rindex('-') cluster_server_node_base_name = cluster_server_node_name[0:index] else: pass if cluster_server_node_num == server_num: return process_self_define_json(Status.failed, "error. cluster server num is same with server_num.") if cluster_server_node_num > server_num: scale_in_len = cluster_server_node_num - server_num scale_in_node_ids = [] for index in range(cluster_server_node_num - scale_in_len, cluster_server_node_num): scale_in_node_name = cluster_server_node_base_name + "-" + str(index) scale_in_node_ids.append(scale_in_node_name) return call_scalein(scale_in_node_ids) return call_scaleout(server_num - cluster_server_node_num) def call_scaleout(scale_out_server_num, scale_out_worker_num=0): url = base_url + "scaleout" data = {"server_num": scale_out_server_num, "worker_num": scale_out_worker_num} res = session.post(url, headers=headers, verify=False, data=json.dumps(data)) res_json = json.loads(res.text) if res_json["code"] == Status.failed: return process_self_define_json(Status.failed, res_json["error_message"]) result = "scale out server num is " + str(scale_out_server_num) return process_result_json(Status.success, res_json["message"], result) def call_scaleout_rollback(): url = base_url + Restful.scaleoutRollback res = session.get(url, verify=False) res_json = json.loads(res.text) if res_json["code"] == Status.failed: return process_self_define_json(Status.failed, res_json["error_message"]) return process_self_define_json(Status.success, res_json["message"]) def call_scalein(scale_in_node_ids): """ call cluster to scale in """ if not scale_in_node_ids: return process_self_define_json(Status.failed, "error. node ids is empty.") url = base_url + "scalein" data = {"node_ids": scale_in_node_ids} res = session.post(url, headers=headers, verify=False, data=json.dumps(data)) res_json = json.loads(res.text) if res_json["code"] == Status.failed: return process_self_define_json(Status.failed, res_json["error_message"]) result = "scale in node ids is " + str(scale_in_node_ids) return process_result_json(Status.success, res_json["message"], result) def call_nodes(): url = base_url + Restful.nodes res = session.get(url, verify=False) res_json = json.loads(res.text) if res_json["code"] == Status.failed: return process_self_define_json(Status.failed, res_json["error_message"]) return process_result_json(Status.success, res_json["message"], res_json["nodeIds"]) def call_get_instance_detail(): """ call cluster get instance detail """ if not os.path.exists(metrics_file_path): return process_self_define_json(Status.failed, "error. metrics file is not existed.") ans_json_obj = {} joined_client_num_list = [] rejected_client_num_list = [] metrics_auc_list = [] metrics_loss_list = [] iteration_execution_time_list = [] with open(metrics_file_path, 'r') as f: metrics_list = f.readlines() if not metrics_list: return process_self_define_json(Status.failed, "error. metrics file has no content") for metrics in metrics_list: json_obj = json.loads(metrics) iteration_execution_time_list.append(json_obj['iterationExecutionTime']) joined_client_num_list.append(json_obj['joinedClientNum']) rejected_client_num_list.append(json_obj['rejectedClientNum']) metrics_auc_list.append(json_obj['metricsAuc']) metrics_loss_list.append(json_obj['metricsLoss']) last_metrics = metrics_list[len(metrics_list) - 1] last_metrics_obj = json.loads(last_metrics) ans_json_obj["code"] = Status.success ans_json_obj["describe"] = "get instance metrics detail successful." ans_json_obj["result"] = {} ans_json_obj["result"]['currentIteration'] = last_metrics_obj['currentIteration'] ans_json_obj["result"]['flIterationNum'] = last_metrics_obj['flIterationNum'] ans_json_obj["result"]['flName'] = last_metrics_obj['flName'] ans_json_obj["result"]['instanceStatus'] = last_metrics_obj['instanceStatus'] ans_json_obj["result"]['iterationExecutionTime'] = iteration_execution_time_list ans_json_obj["result"]['joinedClientNum'] = joined_client_num_list ans_json_obj["result"]['rejectedClientNum'] = rejected_client_num_list ans_json_obj["result"]['metricsAuc'] = metrics_auc_list ans_json_obj["result"]['metricsLoss'] = metrics_loss_list return json.dumps(ans_json_obj) def call_new_instance(): """ call cluster new instance """ if instance_param == "": return process_self_define_json(Status.failed, "error. instance_param is empty.") instance_param_list = instance_param.split(sep=",") instance_param_json_obj = {} url = base_url + Restful.newInstance for cur in instance_param_list: pair = cur.split(sep="=") instance_param_json_obj[pair[0]] = float(pair[1]) data = json.dumps(instance_param_json_obj) res = session.post(url, verify=False, data=data) res_json = json.loads(res.text) if res_json["code"] == Status.failed: return process_self_define_json(Status.failed, res_json["error_message"]) return process_self_define_json(Status.success, res_json["message"]) def call_query_instance(): url = base_url + Restful.queryInstance res = session.post(url, verify=False) res_json = json.loads(res.text) if res_json["code"] == Status.failed: return process_self_define_json(Status.failed, res_json["error_message"]) return process_result_json(Status.success, res_json["message"], res_json["result"]) def call_enable_fls(): url = base_url + Restful.enableFLS res = session.post(url, verify=False) res_json = json.loads(res.text) if res_json["code"] == Status.failed: return process_self_define_json(Status.failed, res_json["error_message"]) return process_self_define_json(Status.success, res_json["message"]) def call_disable_fls(): url = base_url + Restful.disableFLS res = session.post(url, verify=False) res_json = json.loads(res.text) if res_json["code"] == Status.failed: return process_self_define_json(Status.failed, res_json["error_message"]) return process_self_define_json(Status.success, res_json["message"]) def call_state(): url = base_url + Restful.state res = session.get(url, verify=False) res_json = json.loads(res.text) if res_json["code"] == Status.failed: return process_self_define_json(Status.failed, res_json["error_message"]) result = res_json['cluster_state'] return process_result_json(Status.success, res_json["message"], result) def process_result_json(code, describe, result): result_dict = {"code": code, "describe": describe, "result": result} return json.dumps(result_dict) def process_self_define_json(code, describe): result_dict = {"code": code, "describe": describe} return json.dumps(result_dict) if __name__ == '__main__': if request_name == Restful.scale: print(call_scale()) elif request_name == Restful.nodes: print(call_nodes()) elif request_name == Restful.getInstanceDetail: print(call_get_instance_detail()) elif request_name == Restful.newInstance: print(call_new_instance()) elif request_name == Restful.queryInstance: print(call_query_instance()) elif request_name == Restful.enableFLS: print(call_enable_fls()) elif request_name == Restful.disableFLS: print(call_disable_fls()) elif request_name == Restful.state: print(call_state()) elif request_name == Restful.scaleoutRollback: print(call_scaleout_rollback()) else: print(process_self_define_json(1, "error. request_name is not found!"))