From 5a0c01c8b2e6f4b412448a0937516b4c5cfb0cb0 Mon Sep 17 00:00:00 2001 From: bxdd Date: Wed, 11 Oct 2023 18:59:05 +0800 Subject: [PATCH] [Fix] fix container bugs --- learnware/client/container.py | 3 +- learnware/client/utils.py | 4 +- tests/test_learnware_client/test_reuse.py | 27 ++++++------- tests/test_learnware_client/test_reuse2.py | 46 ---------------------- 4 files changed, 14 insertions(+), 66 deletions(-) delete mode 100644 tests/test_learnware_client/test_reuse2.py diff --git a/learnware/client/container.py b/learnware/client/container.py index c59b4f2..85593ae 100644 --- a/learnware/client/container.py +++ b/learnware/client/container.py @@ -62,7 +62,7 @@ class ModelEnvContainer(BaseModel): pickle.dump({"method": method, "kargs": kargs}, input_fp) system_execute( - ["conda", "run", "-n", f"{self.conda_env}", "--no-capture-output", "python3", f"{self.model_script}", f"--model-path", f"{model_path}", f"--input-path", f"{input_path}", f"--output-path", "{output_path}"] + ["conda", "run", "-n", f"{self.conda_env}", "--no-capture-output", "python3", f"{self.model_script}", f"--model-path", f"{model_path}", f"--input-path", f"{input_path}", f"--output-path", f"{output_path}"] ) with open(output_path, "rb") as output_fp: @@ -124,7 +124,6 @@ class LearnwaresContainer: model_list = [_learnware.get_model() for _learnware in self.learnware_list] with ProcessPoolExecutor(max_workers=max(os.cpu_count() // 2, 1)) as executor: executor.map(self._destroy_model_container, model_list) - return self def get_learnware_list_with_container(self): return self.learnware_list \ No newline at end of file diff --git a/learnware/client/utils.py b/learnware/client/utils.py index 5e315f2..4d89140 100644 --- a/learnware/client/utils.py +++ b/learnware/client/utils.py @@ -8,9 +8,9 @@ from .package_utils import filter_nonexist_conda_packages_file, filter_nonexist_ logger = get_module_logger(module_name="client_utils") -def system_execute(args): +def system_execute(args, timeout=None): try: - com_process = subprocess.run(args, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, check=True) + com_process = subprocess.run(args, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, check=True, timeout=timeout) except subprocess.CalledProcessError as err: print(com_process.stderr) raise err diff --git a/tests/test_learnware_client/test_reuse.py b/tests/test_learnware_client/test_reuse.py index 680622d..244c934 100644 --- a/tests/test_learnware_client/test_reuse.py +++ b/tests/test_learnware_client/test_reuse.py @@ -3,7 +3,7 @@ import numpy as np from learnware.learnware import get_learnware_from_dirpath, Learnware from learnware.market import EasyMarket -from learnware.client.container import ModelEnvContainer +from learnware.client.container import ModelEnvContainer, LearnwaresContainer from learnware.learnware.reuse import AveragingReuser if __name__ == "__main__": @@ -24,27 +24,22 @@ if __name__ == "__main__": '/home/bixd/workspace/learnware/Learnware/tests/test_learnware_client/svc_tic', ] + learnware_list = [] for id, (zip_path, dir_path) in enumerate(zip(zip_paths, dir_paths)): with zipfile.ZipFile(zip_path, "r") as z_file: z_file.extractall(dir_path) learnware = get_learnware_from_dirpath(f'test_id{id}', semantic_specification, dir_path) - - model = ModelEnvContainer(learnware.get_model(), zip_path) - model.init_env_and_metadata() - - env_leanware = Learnware(id=learnware.id, model=model, specification=learnware.get_specification()) - - learnware_list.append(env_leanware) - - print('check:', EasyMarket.check_learnware(env_leanware)) - - reuser = AveragingReuser(learnware_list, mode='vote') - input_array = np.random.randint(0, 3, size=(20, 9)) - print(reuser.predict(input_array).argmax(axis=1)) + learnware_list.append(learnware) - for id, ind_learner in enumerate(learnware_list): - print(f"learner_{id}", reuser.predict(input_array).argmax(axis=1)) + with LearnwaresContainer(learnware_list, zip_paths) as env_container: + + learnware_list = env_container.get_learnware_list_with_container() + reuser = AveragingReuser(learnware_list, mode='vote') + input_array = np.random.randint(0, 3, size=(20, 9)) + print(reuser.predict(input_array).argmax(axis=1)) + for id, ind_learner in enumerate(learnware_list): + print(f"learner_{id}", reuser.predict(input_array).argmax(axis=1)) \ No newline at end of file diff --git a/tests/test_learnware_client/test_reuse2.py b/tests/test_learnware_client/test_reuse2.py deleted file mode 100644 index 0dcf293..0000000 --- a/tests/test_learnware_client/test_reuse2.py +++ /dev/null @@ -1,46 +0,0 @@ -import zipfile -import numpy as np - -from learnware.learnware import get_learnware_from_dirpath, Learnware -from learnware.market import EasyMarket -from learnware.client.container import ModelEnvContainer, LearnwaresContainer -from learnware.learnware.reuse import AveragingReuser - -if __name__ == "__main__": - semantic_specification = dict() - semantic_specification["Data"] = {"Type": "Class", "Values": ["Text"]} - semantic_specification["Task"] = {"Type": "Class", "Values": ["Ranking"]} - semantic_specification["Library"] = {"Type": "Class", "Values": ["Scikit-learn"]} - semantic_specification["Scenario"] = {"Type": "Tag", "Values": "Financial"} - semantic_specification["Name"] = {"Type": "String", "Values": "test"} - semantic_specification["Description"] = {"Type": "String", "Values": "test"} - - zip_paths = [ - '/home/bixd/workspace/learnware/Learnware/tests/test_learnware_client/rf_tic.zip', - '/home/bixd/workspace/learnware/Learnware/tests/test_learnware_client/svc_tic.zip', - ] - dir_paths = [ - '/home/bixd/workspace/learnware/Learnware/tests/test_learnware_client/rf_tic', - '/home/bixd/workspace/learnware/Learnware/tests/test_learnware_client/svc_tic', - ] - - - learnware_list = [] - for id, (zip_path, dir_path) in enumerate(zip(zip_paths, dir_paths)): - with zipfile.ZipFile(zip_path, "r") as z_file: - z_file.extractall(dir_path) - - learnware = get_learnware_from_dirpath(f'test_id{id}', semantic_specification, dir_path) - learnware_list.append(learnware) - - with LearnwaresContainer(learnware_list, zip_paths) as env_container: - learnware_list = env_container.get_learnware_list_with_container() - - reuser = AveragingReuser(learnware_list, mode='vote') - input_array = np.random.randint(0, 3, size=(20, 9)) - print(reuser.predict(input_array).argmax(axis=1)) - - for id, ind_learner in enumerate(learnware_list): - print(f"learner_{id}", reuser.predict(input_array).argmax(axis=1)) - - \ No newline at end of file