From c4ca0e60dcbf0a89b38fb2ce47a793fbc1a72f4d Mon Sep 17 00:00:00 2001 From: bxdd Date: Fri, 3 Nov 2023 13:39:57 +0800 Subject: [PATCH] [MNT] modify conda container to fit with relative path of module path --- learnware/client/container.py | 140 +++++++++++++++++----------------- learnware/learnware/base.py | 10 ++- 2 files changed, 79 insertions(+), 71 deletions(-) diff --git a/learnware/client/container.py b/learnware/client/container.py index 47bc045..0f09e6d 100644 --- a/learnware/client/container.py +++ b/learnware/client/container.py @@ -93,8 +93,10 @@ class ModelCondaContainer(ModelContainer): output_path = os.path.join(tempdir, "output.pkl") model_path = os.path.join(tempdir, "model.pkl") + model_config = self.model_config.copy() + model_config["module_path"] = Learnware.get_model_module_abspath(self.learnware_dirpath, model_config["module_path"]) with open(model_path, "wb") as model_fp: - pickle.dump(self.model_config, model_fp) + pickle.dump(model_config, model_fp) system_execute( [ @@ -128,9 +130,12 @@ class ModelCondaContainer(ModelContainer): input_path = os.path.join(tempdir, "input.pkl") output_path = os.path.join(tempdir, "output.pkl") model_path = os.path.join(tempdir, "model.pkl") - + + model_config = self.model_config.copy() + model_config["module_path"] = Learnware.get_model_module_abspath(self.learnware_dirpath, model_config["module_path"]) + with open(model_path, "wb") as model_fp: - pickle.dump(self.model_config, model_fp) + pickle.dump(model_config, model_fp) with open(input_path, "wb") as input_fp: pickle.dump({"method": method, "kargs": kargs}, input_fp) @@ -263,7 +268,7 @@ class ModelDockerContainer(ModelContainer): except docker.errors.NotFound as err: logger.error(f"Copy file from container failed due to {err}") - def _install_environment(self, zip_path, conda_env): + def _install_environment(self, learnware_dirpath, conda_env): """Install environment of a learnware in docker container Parameters @@ -280,74 +285,71 @@ class ModelDockerContainer(ModelContainer): """ run_cmd_times = 10 with tempfile.TemporaryDirectory(prefix="learnware_") as tempdir: - with zipfile.ZipFile(file=zip_path, mode="r") as z_file: - success_flag = False - logger.info(f"zip_file namelist: {z_file.namelist()}") - - if "environment.yaml" in z_file.namelist(): - z_file.extract(member="environment.yaml", path=tempdir) - yaml_path: str = os.path.join(tempdir, "environment.yaml") - yaml_path_filter: str = os.path.join(tempdir, "environment_filter.yaml") - logger.info(f"checking the avaliabe conda packages for {conda_env}") - filter_nonexist_conda_packages_file(yaml_file=yaml_path, output_yaml_file=yaml_path_filter) - self._copy_file_to_container(yaml_path_filter, yaml_path_filter) - - # create environment - logger.info(f"Create and update conda env [{conda_env}] according to .yaml file") - for i in range(run_cmd_times): - result = self.docker_container.exec_run( - " ".join( - ["conda", "env", "update", "--name", f"{conda_env}", "--file", f"{yaml_path_filter}"] - ) + success_flag = False + logger.info(f"zip_file namelist: {os.namelist()}") + + if "environment.yaml" in os.listdir(learnware_dirpath): + yaml_path: str = os.path.join(learnware_dirpath, "environment.yaml") + yaml_path_filter: str = os.path.join(tempdir, "environment_filter.yaml") + logger.info(f"checking the avaliabe conda packages for {conda_env}") + filter_nonexist_conda_packages_file(yaml_file=yaml_path, output_yaml_file=yaml_path_filter) + self._copy_file_to_container(yaml_path_filter, yaml_path_filter) + + # create environment + logger.info(f"Create and update conda env [{conda_env}] according to .yaml file") + for i in range(run_cmd_times): + result = self.docker_container.exec_run( + " ".join( + ["conda", "env", "update", "--name", f"{conda_env}", "--file", f"{yaml_path_filter}"] ) - if result.exit_code == 0: - success_flag = True - break - - elif "requirements.txt" in z_file.namelist(): - z_file.extract(member="requirements.txt", path=tempdir) - requirements_path: str = os.path.join(tempdir, "requirements.txt") - requirements_path_filter: str = os.path.join(tempdir, "requirements_filter.txt") - logger.info(f"checking the avaliabe pip packages for {conda_env}.") - filter_nonexist_pip_packages_file( - requirements_file=requirements_path, output_file=requirements_path_filter ) - logger.info(f"Create empty conda env [{conda_env}] in docker.") - for i in range(run_cmd_times): - result = self.docker_container.exec_run( - " ".join(["conda", "create", "-y", "--name", f"{conda_env}", "python=3.8"]) - ) - if result.exit_code == 0: - break - logger.info(f"install pip requirements for conda env [{conda_env}] in docker.") - - self._copy_file_to_container(requirements_path_filter, requirements_path_filter) - for i in range(run_cmd_times): - result = self.docker_container.exec_run( - " ".join( - [ - "conda", - "run", - "-n", - f"{conda_env}", - "--no-capture-output", - "python3", - "-m", - "pip", - "install", - "-r", - f"{requirements_path_filter}", - ] - ) + if result.exit_code == 0: + success_flag = True + break + + elif "requirements.txt" in os.listdir(learnware_dirpath): + requirements_path: str = os.path.join(learnware_dirpath, "requirements.txt") + requirements_path_filter: str = os.path.join(tempdir, "requirements_filter.txt") + logger.info(f"checking the avaliabe pip packages for {conda_env}.") + filter_nonexist_pip_packages_file( + requirements_file=requirements_path, output_file=requirements_path_filter + ) + logger.info(f"Create empty conda env [{conda_env}] in docker.") + for i in range(run_cmd_times): + result = self.docker_container.exec_run( + " ".join(["conda", "create", "-y", "--name", f"{conda_env}", "python=3.8"]) + ) + if result.exit_code == 0: + break + logger.info(f"install pip requirements for conda env [{conda_env}] in docker.") + + self._copy_file_to_container(requirements_path_filter, requirements_path_filter) + for i in range(run_cmd_times): + result = self.docker_container.exec_run( + " ".join( + [ + "conda", + "run", + "-n", + f"{conda_env}", + "--no-capture-output", + "python3", + "-m", + "pip", + "install", + "-r", + f"{requirements_path_filter}", + ] ) - if result.exit_code == 0: - success_flag = True - break - else: - raise Exception("Environment.yaml or requirements.txt not found in the learnware zip file.") + ) + if result.exit_code == 0: + success_flag = True + break + else: + raise Exception("Environment.yaml or requirements.txt not found in the learnware zip file.") - if not success_flag: - logger.error(f"Install conda env [{conda_env}] in docker failed!") + if not success_flag: + logger.error(f"Install conda env [{conda_env}] in docker failed!") success_flag = False logger.info(f"Install learnware package for conda env [{conda_env}] in docker.") @@ -384,6 +386,8 @@ class ModelDockerContainer(ModelContainer): self.docker_model_script_path = os.path.join(tempdir, "run_model.py") docker_model_config = self.model_config.copy() + + with tempfile.TemporaryDirectory(prefix="learnware_model_config_") as config_tempdir: basename = os.path.basename(self.model_config["module_path"]) docker_model_config["module_path"] = os.path.join(config_tempdir, basename) diff --git a/learnware/learnware/base.py b/learnware/learnware/base.py index 8400fe7..fc490c8 100644 --- a/learnware/learnware/base.py +++ b/learnware/learnware/base.py @@ -47,13 +47,17 @@ class Learnware: def __repr__(self) -> str: return "{}({}, {}, {})".format(type(self).__name__, self.id, type(self.model).__name__, self.specification) + @staticmethod + def get_model_module_abspath(learnware_dirpath, module_path): + if isinstance(module_path, str) and module_path.endswith(".py") and not os.path.isabs(module_path): + module_path = os.path.join(learnware_dirpath, module_path) + return module_path + def instantiate_model(self): if isinstance(self.model, BaseModel): logger.info("The learnware had been instantiated, thus the instantiation operation is ignored!") elif isinstance(self.model, dict): - module_path = self.model["module_path"] - if isinstance(module_path, str) and module_path.endswith(".py") and not os.path.isabs(module_path): - module_path = os.path.join(self.learnware_dirpath, module_path) + module_path = Learnware.get_model_module_abspath(self.learnware_dirpath, self.model["module_path"]) model_module = get_module_by_module_path(module_path) cls = getattr(model_module, self.model["class_name"]) setattr(sys.modules["__main__"], self.model["class_name"], cls)