diff --git a/learnware/client/container.py b/learnware/client/container.py index 243d580..47bc045 100644 --- a/learnware/client/container.py +++ b/learnware/client/container.py @@ -22,10 +22,10 @@ logger = get_module_logger(module_name="client_container") class ModelContainer(BaseModel): - def __init__(self, model_config: dict, learnware_zippath: str, build: bool = True): + def __init__(self, model_config: dict, learnware_dirpath: str, build: bool = True): self.model_script = os.path.join(C.package_path, "client", "scripts", "run_model.py") self.model_config = model_config - self.learnware_zippath = learnware_zippath + self.learnware_dirpath = learnware_dirpath self.build = build self.cleanup_flag = False @@ -78,12 +78,12 @@ class ModelContainer(BaseModel): class ModelCondaContainer(ModelContainer): - def __init__(self, model_config: dict, learnware_zippath: str, conda_env: str = None, build: bool = True): + def __init__(self, model_config: dict, learnware_dirpath: str, conda_env: str = None, build: bool = True): self.conda_env = f"learnware_{shortuuid.uuid()}" if conda_env is None else conda_env - super(ModelCondaContainer, self).__init__(model_config, learnware_zippath, build) + super(ModelCondaContainer, self).__init__(model_config, learnware_dirpath, build) def _init_env(self): - install_environment(self.learnware_zippath, self.conda_env) + install_environment(self.learnware_dirpath, self.conda_env) def _remove_env(self): remove_enviroment(self.conda_env) @@ -175,7 +175,7 @@ class ModelDockerContainer(ModelContainer): def __init__( self, model_config: dict, - learnware_zippath: str, + learnware_dirpath: str, docker_container: object = None, build: bool = True, ): @@ -192,7 +192,7 @@ class ModelDockerContainer(ModelContainer): self.docker_model_config = None self.docker_model_script_path = None # call init method of parent of parent class - super(ModelDockerContainer, self).__init__(model_config, learnware_zippath, build) + super(ModelDockerContainer, self).__init__(model_config, learnware_dirpath, build) @staticmethod def _generate_docker_container(): @@ -377,7 +377,7 @@ class ModelDockerContainer(ModelContainer): def _setup_env_and_metadata(self): """setup env and set the input and output shape by communicating with docker""" - self._install_environment(self.learnware_zippath, self.conda_env) + self._install_environment(self.learnware_dirpath, self.conda_env) with tempfile.TemporaryDirectory(prefix="learnware_") as tempdir: output_path = os.path.join(tempdir, "output.pkl") model_path = os.path.join(tempdir, "model.pkl") diff --git a/learnware/client/utils.py b/learnware/client/utils.py index 95a10c7..a425313 100644 --- a/learnware/client/utils.py +++ b/learnware/client/utils.py @@ -22,13 +22,13 @@ def remove_enviroment(conda_env): system_execute(args=["conda", "env", "remove", "-n", f"{conda_env}"]) -def install_environment(zip_path, conda_env): +def install_environment(learnware_dirpath, conda_env): """Install environment of a learnware Parameters ---------- - zip_path : str - Path of the learnware zip file + learnware_dirpath : str + Path of the learnware folder conda_env : str a new conda environment will be created with the given name; @@ -38,48 +38,45 @@ def install_environment(zip_path, conda_env): Lack of the environment configuration file. """ with tempfile.TemporaryDirectory(prefix="learnware_") as tempdir: - with zipfile.ZipFile(file=zip_path, mode="r") as z_file: - logger.info(f"zip_file namelist: {z_file.namelist()}") - if "environment.yaml" in z_file.namelist(): - z_file.extract(member="environment.yaml", path=tempdir) - yaml_path: str = os.path.join(tempdir, "environment.yaml") - yaml_path_filter: str = os.path.join(tempdir, "environment_filter.yaml") - logger.info(f"checking the avaliabe conda packages for {conda_env}") - filter_nonexist_conda_packages_file(yaml_file=yaml_path, output_yaml_file=yaml_path_filter) - # create environment - logger.info(f"create conda env [{conda_env}] according to .yaml file") - system_execute( - args=["conda", "env", "create", "--name", f"{conda_env}", "--file", f"{yaml_path_filter}"] - ) + logger.info(f"zip_file namelist: {os.listdir(learnware_dirpath)}") + if "environment.yaml" in os.listdir(learnware_dirpath): + yaml_path: str = os.path.join(learnware_dirpath, "environment.yaml") + yaml_path_filter: str = os.path.join(tempdir, "environment_filter.yaml") + logger.info(f"checking the avaliabe conda packages for {conda_env}") + filter_nonexist_conda_packages_file(yaml_file=yaml_path, output_yaml_file=yaml_path_filter) + # create environment + logger.info(f"create conda env [{conda_env}] according to .yaml file") + system_execute( + args=["conda", "env", "create", "--name", f"{conda_env}", "--file", f"{yaml_path_filter}"] + ) - elif "requirements.txt" in z_file.namelist(): - z_file.extract(member="requirements.txt", path=tempdir) - requirements_path: str = os.path.join(tempdir, "requirements.txt") - requirements_path_filter: str = os.path.join(tempdir, "requirements_filter.txt") - logger.info(f"checking the avaliabe pip packages for {conda_env}") - filter_nonexist_pip_packages_file( - requirements_file=requirements_path, output_file=requirements_path_filter - ) - logger.info(f"create empty conda env [{conda_env}]") - system_execute(args=["conda", "create", "-y", "--name", f"{conda_env}", "python=3.8"]) - logger.info(f"install pip requirements for conda env [{conda_env}]") - system_execute( - args=[ - "conda", - "run", - "-n", - f"{conda_env}", - "--no-capture-output", - "python3", - "-m", - "pip", - "install", - "-r", - f"{requirements_path_filter}", - ] - ) - else: - raise Exception("Environment.yaml or requirements.txt not found in the learnware zip file.") + elif "requirements.txt" in os.listdir(learnware_dirpath): + requirements_path: str = os.path.join(learnware_dirpath, "requirements.txt") + requirements_path_filter: str = os.path.join(tempdir, "requirements_filter.txt") + logger.info(f"checking the avaliabe pip packages for {conda_env}") + filter_nonexist_pip_packages_file( + requirements_file=requirements_path, output_file=requirements_path_filter + ) + logger.info(f"create empty conda env [{conda_env}]") + system_execute(args=["conda", "create", "-y", "--name", f"{conda_env}", "python=3.8"]) + logger.info(f"install pip requirements for conda env [{conda_env}]") + system_execute( + args=[ + "conda", + "run", + "-n", + f"{conda_env}", + "--no-capture-output", + "python3", + "-m", + "pip", + "install", + "-r", + f"{requirements_path_filter}", + ] + ) + else: + raise Exception("Environment.yaml or requirements.txt not found in the learnware folder.") logger.info(f"install learnware package for conda env [{conda_env}]") system_execute(