From fc18d9a151cf07c197a79bfa1f21dec247183198 Mon Sep 17 00:00:00 2001 From: Gene Date: Fri, 13 Oct 2023 00:37:36 +0800 Subject: [PATCH] [MNT] modify load_learnware in LearnwareClient --- learnware/client/learnware_client.py | 55 ++++++++++++++-------------- tests/test_client/test_download.py | 53 +++++---------------------- 2 files changed, 37 insertions(+), 71 deletions(-) diff --git a/learnware/client/learnware_client.py b/learnware/client/learnware_client.py index d2cd8f1..2d40d70 100644 --- a/learnware/client/learnware_client.py +++ b/learnware/client/learnware_client.py @@ -64,10 +64,9 @@ class LearnwareClient: self.host = C.backend_host else: self.host = host - pass self.chunk_size = 1024 * 1024 - pass + self.tempdir_list = [] def login(self, email, token): url = f"{self.host}/auth/login_by_token" @@ -305,40 +304,36 @@ class LearnwareClient: return semantic_conf[key.value]["Values"] def load_learnware(self, learnware_file: str, load_model: bool = True): - with tempfile.TemporaryDirectory(prefix="learnware_") as tempdir: - with zipfile.ZipFile(learnware_file, "r") as z_file: - z_file.extractall(tempdir) - pass + self.tempdir_list.append(tempfile.TemporaryDirectory(prefix="learnware_")) + tempdir = self.tempdir_list[-1].name + + with zipfile.ZipFile(learnware_file, "r") as z_file: + z_file.extractall(tempdir) - yaml_file = C.learnware_folder_config["yaml_file"] + yaml_file = C.learnware_folder_config["yaml_file"] - with open(os.path.join(tempdir, yaml_file), "r") as fin: - learnware_info = yaml.safe_load(fin) - pass + with open(os.path.join(tempdir, yaml_file), "r") as fin: + learnware_info = yaml.safe_load(fin) - learnware_id = learnware_info.get("id") - if learnware_id is None: - learnware_id = "test_id" - pass + learnware_id = learnware_info.get("id") + if learnware_id is None: + learnware_id = "test_id" - semantic_specification = learnware_info.get("semantic_specification") - if semantic_specification is None: - semantic_specification = {} - pass - else: - semantic_file = semantic_specification.get("file_name") + semantic_specification = learnware_info.get("semantic_specification") + if semantic_specification is None: + semantic_specification = {} + else: + semantic_file = semantic_specification.get("file_name") - with open(os.path.join(tempdir, semantic_file), "r") as fin: - semantic_specification = json.load(fin) - pass - pass + with open(os.path.join(tempdir, semantic_file), "r") as fin: + semantic_specification = json.load(fin) - learnware_obj = learnware.get_learnware_from_dirpath(learnware_id, semantic_specification, tempdir) + learnware_obj = learnware.get_learnware_from_dirpath(learnware_id, semantic_specification, tempdir) - if load_model: - learnware_obj.instantiate_model() + if load_model: + learnware_obj.instantiate_model() - return learnware_obj + return learnware_obj def system(self, command): retcd = os.system(command) @@ -424,3 +419,7 @@ class LearnwareClient: logger.info("test ok") pass + + def __del__(self): + for tempdir in self.tempdir_list: + tempdir.cleanup() \ No newline at end of file diff --git a/tests/test_client/test_download.py b/tests/test_client/test_download.py index 7d2cc53..4a21286 100644 --- a/tests/test_client/test_download.py +++ b/tests/test_client/test_download.py @@ -9,47 +9,6 @@ from learnware.client.container import ModelEnvContainer, LearnwaresContainer from learnware.learnware.reuse import AveragingReuser -def test_container(zip_paths): - semantic_specification = dict() - semantic_specification["Data"] = {"Type": "Class", "Values": ["Text"]} - semantic_specification["Task"] = {"Type": "Class", "Values": ["Ranking"]} - semantic_specification["Library"] = {"Type": "Class", "Values": ["Scikit-learn"]} - semantic_specification["Scenario"] = {"Type": "Tag", "Values": "Financial"} - semantic_specification["Name"] = {"Type": "String", "Values": "test"} - semantic_specification["Description"] = {"Type": "String", "Values": "test"} - - learnware_list = [] - for id, zip_path in enumerate(zip_paths): - dir_path = zip_path[:-4] - with zipfile.ZipFile(zip_path, "r") as z_file: - z_file.extractall(dir_path) - - learnware = get_learnware_from_dirpath(f"test_id{id}", semantic_specification, dir_path) - learnware_list.append(learnware) - - with LearnwaresContainer(learnware_list, zip_paths) as env_container: - learnware_list = env_container.get_learnware_list_with_container() - reuser = AveragingReuser(learnware_list, mode="vote_by_label") - input_array = np.random.random(size=(20, 13)) - print(reuser.predict(input_array)) - - for idx, learnware in enumerate(learnware_list): - print(f"learnware_{idx}", learnware.predict(input_array)) - - -def test_load(zip_paths): - learnware_list = [client.load_learnware(file, load_model=False) for file in zip_paths] - - with LearnwaresContainer(learnware_list, zip_paths) as env_container: - learnware_list = env_container.get_learnware_list_with_container() - reuser = AveragingReuser(learnware_list, mode="vote_by_label") - input_array = np.random.random(size=(20, 13)) - print(reuser.predict(input_array)) - - for idx, learnware in enumerate(learnware_list): - print(f"learnware_{idx}", learnware.predict(input_array)) - - if __name__ == "__main__": email = "liujd@lamda.nju.edu.cn" token = "f7e647146a314c6e8b4e2e1079c4bca4" @@ -64,5 +23,13 @@ if __name__ == "__main__": zip_paths[i] = os.path.join(root, zip_paths[i]) client.download_learnware(learnware_ids[i], zip_paths[i]) - test_container(zip_paths) - # test_load(zip_paths) + learnware_list = [client.load_learnware(file, load_model=False) for file in zip_paths] + + with LearnwaresContainer(learnware_list, zip_paths) as env_container: + learnware_list = env_container.get_learnware_list_with_container() + reuser = AveragingReuser(learnware_list, mode="vote_by_label") + input_array = np.random.random(size=(20, 13)) + print(reuser.predict(input_array)) + + for learnware in learnware_list: + print(learnware.id, learnware.predict(input_array))