From 7e36e63f59deb2165a847896df67c3384adca64b Mon Sep 17 00:00:00 2001 From: Gene Date: Tue, 24 Oct 2023 20:47:04 +0800 Subject: [PATCH] [ENH] add check_learnware in client --- learnware/client/learnware_client.py | 122 +++++---------------------- 1 file changed, 19 insertions(+), 103 deletions(-) diff --git a/learnware/client/learnware_client.py b/learnware/client/learnware_client.py index 1f43540..8a6efb0 100644 --- a/learnware/client/learnware_client.py +++ b/learnware/client/learnware_client.py @@ -19,7 +19,8 @@ from .container import LearnwaresContainer from ..market.easy import EasyMarket from ..logger import get_module_logger from ..specification import Specification -from ..learnware import BaseReuser, Learnware +from ..learnware import BaseReuser, Learnware, get_learnware_from_dirpath +from ..test import get_semantic_specification CHUNK_SIZE = 1024 * 1024 logger = get_module_logger(module_name="LearnwareClient") @@ -264,29 +265,6 @@ class LearnwareClient: raise Exception("delete failed: " + json.dumps(result)) pass - def check_learnware(self, path, semantic_specification): - if os.path.isfile(path): - with tempfile.TemporaryDirectory() as tempdir: - with zipfile.ZipFile(path, "r") as z_file: - z_file.extractall(tempdir) - pass - return self.check_learnware_folder(tempdir, semantic_specification) - pass - else: - return self.check_learnware_folder(path, semantic_specification) - pass - pass - - def check_learnware_folder(self, folder, semantic_specification): - learnware_obj = learnware.get_learnware_from_dirpath("test_id", semantic_specification, folder) - - check_result = EasyMarket.check_learnware(learnware_obj) - if check_result == EasyMarket.USABLE_LEARWARE: - return True - else: - return False - pass - def create_semantic_specification( self, name, description, data_type, task_type, library_type, senarioes, input_description, output_description ): @@ -408,91 +386,29 @@ class LearnwareClient: return learnware_list[0] else: return learnware_list - - def system(self, command): - retcd = os.system(command) - if retcd != 0: - raise RuntimeError(f"Command {command} failed with return code {retcd}") - pass - - def install_environment(self, zip_path, conda_env=None): - """Install environment of a learnware - - Parameters - ---------- - zip_path : str - Path of the learnware zip file - conda_env : optional - If it is not None, a new conda environment will be created with the given name; - If it is None, use current environment. - - Raises - ------ - Exception - Lack of the environment configuration file. - """ - with tempfile.TemporaryDirectory(prefix="learnware_") as tempdir: - with zipfile.ZipFile(zip_path, "r") as z_file: - logger.info(f"zip_file namelist: {z_file.namelist}") - if "environment.yaml" in z_file.namelist(): - z_file.extract("environment.yaml", tempdir) - yaml_path = os.path.join(tempdir, "environment.yaml") - yaml_path_filter = os.path.join(tempdir, "environment_filter.yaml") - package_utils.filter_nonexist_conda_packages_file(yaml_path, yaml_path_filter) - # create environment - if conda_env is not None: - self.system(f"conda env update --name {conda_env} --file {yaml_path_filter}") - pass - else: - self.system(f"conda env update --file {yaml_path_filter}") - pass - pass - elif "requirements.txt" in z_file.namelist(): - z_file.extract("requirements.txt", tempdir) - requirements_path = os.path.join(tempdir, "requirements.txt") - requirements_path_filter = os.path.join(tempdir, "requirements_filter.txt") - package_utils.filter_nonexist_pip_packages_file(requirements_path, requirements_path_filter) - - if conda_env is not None: - self.system(f"conda create -y --name {conda_env} python=3.8") - self.system( - f"conda run --name {conda_env} --no-capture-output python3 -m pip install -r {requirements_path_filter}" - ) - else: - self.system(f"python3 -m pip install -r {requirements_path_filter}") - pass - pass - else: - raise Exception("Environment.yaml or requirements.txt not found in the learnware zip file.") - pass - pass - pass - - def test_learnware(self, zip_path, semantic_specification=None): + + @staticmethod + def check_learnware(zip_path, semantic_specification=None): if semantic_specification is None: - semantic_specification = dict() - pass - + semantic_specification = get_semantic_specification() + with tempfile.TemporaryDirectory(prefix="learnware_") as tempdir: with zipfile.ZipFile(zip_path, mode="r") as z_file: z_file.extractall(tempdir) - pass - - learnware_obj = learnware.get_learnware_from_dirpath("test_id", semantic_specification, tempdir) - - if learnware_obj is None: + + learnware = get_learnware_from_dirpath( + id="test", semantic_spec=semantic_specification, learnware_dirpath=tempdir + ) + + if learnware is None: raise Exception("The learnware is not valid.") - - learnware_obj.instantiate_model() - - if len(semantic_specification) > 0: - if EasyMarket.check_learnware(learnware_obj) != EasyMarket.USABLE_LEARWARE: + + with LearnwaresContainer(learnware, zip_path) as env_container: + learnware = env_container.get_learnwares_with_container()[0] + if EasyMarket.check_learnware(learnware) == EasyMarket.USABLE_LEARWARE: + logger.info("The learnware passed the local test.") + else: raise Exception("The learnware is not usable.") - pass - pass - - logger.info("test ok") - pass def cleanup(self): for tempdir in self.tempdir_list: