diff --git a/learnware/client/container.py b/learnware/client/container.py index a69620a..375546c 100644 --- a/learnware/client/container.py +++ b/learnware/client/container.py @@ -94,7 +94,9 @@ class ModelCondaContainer(ModelContainer): model_path = os.path.join(tempdir, "model.pkl") model_config = self.model_config.copy() - model_config["module_path"] = Learnware.get_model_module_abspath(self.learnware_dirpath, model_config["module_path"]) + model_config["module_path"] = Learnware.get_model_module_abspath( + self.learnware_dirpath, model_config["module_path"] + ) with open(model_path, "wb") as model_fp: pickle.dump(model_config, model_fp) @@ -130,10 +132,12 @@ class ModelCondaContainer(ModelContainer): input_path = os.path.join(tempdir, "input.pkl") output_path = os.path.join(tempdir, "output.pkl") model_path = os.path.join(tempdir, "model.pkl") - + model_config = self.model_config.copy() - model_config["module_path"] = Learnware.get_model_module_abspath(self.learnware_dirpath, model_config["module_path"]) - + model_config["module_path"] = Learnware.get_model_module_abspath( + self.learnware_dirpath, model_config["module_path"] + ) + with open(model_path, "wb") as model_fp: pickle.dump(model_config, model_fp) @@ -299,9 +303,7 @@ class ModelDockerContainer(ModelContainer): logger.info(f"Create and update conda env [{conda_env}] according to .yaml file") for i in range(run_cmd_times): result = self.docker_container.exec_run( - " ".join( - ["conda", "env", "update", "--name", f"{conda_env}", "--file", f"{yaml_path_filter}"] - ) + " ".join(["conda", "env", "update", "--name", f"{conda_env}", "--file", f"{yaml_path_filter}"]) ) if result.exit_code == 0: success_flag = True @@ -386,7 +388,9 @@ class ModelDockerContainer(ModelContainer): self.docker_model_script_path = os.path.join(tempdir, "run_model.py") docker_model_config = self.model_config.copy() - docker_model_config["module_path"] = Learnware.get_model_module_abspath(self.learnware_dirpath, docker_model_config["module_path"]) + docker_model_config["module_path"] = Learnware.get_model_module_abspath( + self.learnware_dirpath, docker_model_config["module_path"] + ) self._copy_file_to_container(self.learnware_dirpath, self.learnware_dirpath) with open(model_path, "wb") as model_fp: @@ -436,7 +440,9 @@ class ModelDockerContainer(ModelContainer): model_path = os.path.join(tempdir, "model.pkl") docker_model_config = self.model_config.copy() - docker_model_config["module_path"] = Learnware.get_model_module_abspath(self.learnware_dirpath, docker_model_config["module_path"]) + docker_model_config["module_path"] = Learnware.get_model_module_abspath( + self.learnware_dirpath, docker_model_config["module_path"] + ) with open(model_path, "wb") as model_fp: pickle.dump(docker_model_config, model_fp) @@ -518,7 +524,10 @@ class LearnwaresContainer: if self.mode == "conda": self.learnware_containers = [ Learnware( - _learnware.id, ModelCondaContainer(_learnware.get_model(), _learnware.get_dirpath()), _learnware.get_specification(), _learnware.get_dirpath() + _learnware.id, + ModelCondaContainer(_learnware.get_model(), _learnware.get_dirpath()), + _learnware.get_specification(), + _learnware.get_dirpath(), ) for _learnware in self.learnware_list ] @@ -527,9 +536,11 @@ class LearnwaresContainer: self.learnware_containers = [ Learnware( _learnware.id, - ModelDockerContainer(_learnware.get_model(), _learnware.get_dirpath(), self._docker_container, build=False), + ModelDockerContainer( + _learnware.get_model(), _learnware.get_dirpath(), self._docker_container, build=False + ), _learnware.get_specification(), - _learnware.get_dirpath() + _learnware.get_dirpath(), ) for _learnware in self.learnware_list ] diff --git a/learnware/client/learnware_client.py b/learnware/client/learnware_client.py index c7e99fc..25ef02a 100644 --- a/learnware/client/learnware_client.py +++ b/learnware/client/learnware_client.py @@ -388,7 +388,7 @@ class LearnwareClient: def _check_semantic_specification(semantic_spec): semantic_checker = EasySemanticChecker() return semantic_checker(semantic_spec) != BaseChecker.INVALID_LEARNWARE - + @staticmethod def _check_stat_specification(learnware): stat_checker = CondaChecker(inner_checker=EasyStatChecker()) @@ -399,8 +399,10 @@ class LearnwareClient: semantic_specification = ( get_semantic_specification() if semantic_specification is None else semantic_specification ) - - assert LearnwareClient._check_semantic_specification(semantic_specification), "Semantic specification check failed!" + + assert LearnwareClient._check_semantic_specification( + semantic_specification + ), "Semantic specification check failed!" with tempfile.TemporaryDirectory(prefix="learnware_") as tempdir: with zipfile.ZipFile(learnware_zip_path, mode="r") as z_file: diff --git a/learnware/client/utils.py b/learnware/client/utils.py index d2fd6d6..7a83ec7 100644 --- a/learnware/client/utils.py +++ b/learnware/client/utils.py @@ -46,17 +46,13 @@ def install_environment(learnware_dirpath, conda_env): filter_nonexist_conda_packages_file(yaml_file=yaml_path, output_yaml_file=yaml_path_filter) # create environment logger.info(f"create conda env [{conda_env}] according to .yaml file") - system_execute( - args=["conda", "env", "create", "--name", f"{conda_env}", "--file", f"{yaml_path_filter}"] - ) + system_execute(args=["conda", "env", "create", "--name", f"{conda_env}", "--file", f"{yaml_path_filter}"]) elif "requirements.txt" in os.listdir(learnware_dirpath): requirements_path: str = os.path.join(learnware_dirpath, "requirements.txt") requirements_path_filter: str = os.path.join(tempdir, "requirements_filter.txt") logger.info(f"checking the avaliabe pip packages for {conda_env}") - filter_nonexist_pip_packages_file( - requirements_file=requirements_path, output_file=requirements_path_filter - ) + filter_nonexist_pip_packages_file(requirements_file=requirements_path, output_file=requirements_path_filter) logger.info(f"create empty conda env [{conda_env}]") system_execute(args=["conda", "create", "-y", "--name", f"{conda_env}", "python=3.8"]) logger.info(f"install pip requirements for conda env [{conda_env}]") diff --git a/learnware/learnware/__init__.py b/learnware/learnware/__init__.py index 54f7025..dcd3dc9 100644 --- a/learnware/learnware/__init__.py +++ b/learnware/learnware/__init__.py @@ -46,7 +46,7 @@ def get_learnware_from_dirpath(id: str, semantic_spec: dict, learnware_dirpath) try: yaml_config = read_yaml_to_dict(os.path.join(learnware_dirpath, C.learnware_folder_config["yaml_file"])) - + if "name" in yaml_config: learnware_config["name"] = yaml_config["name"] if "model" in yaml_config: @@ -57,7 +57,6 @@ def get_learnware_from_dirpath(id: str, semantic_spec: dict, learnware_dirpath) if "module_path" not in learnware_config["model"]: learnware_config["model"]["module_path"] = C.learnware_folder_config["module_file"] - learnware_spec = Specification() for _stat_spec in learnware_config["stat_specifications"]: stat_spec = _stat_spec.copy() @@ -71,4 +70,6 @@ def get_learnware_from_dirpath(id: str, semantic_spec: dict, learnware_dirpath) logger.warning(f"Load Learnware {id} failed! Due to {repr(e)}") return None - return Learnware(id=id, model=learnware_config["model"], specification=learnware_spec, learnware_dirpath=learnware_dirpath) + return Learnware( + id=id, model=learnware_config["model"], specification=learnware_spec, learnware_dirpath=learnware_dirpath + ) diff --git a/learnware/learnware/base.py b/learnware/learnware/base.py index 34fbd06..5ec1c2b 100644 --- a/learnware/learnware/base.py +++ b/learnware/learnware/base.py @@ -37,7 +37,7 @@ class Learnware: specification : Specification The specification including the semantic specification and the statistic specification dirpath: str - The path of the learnware directory + The path of the learnware directory """ self.id = id self.model = model @@ -52,7 +52,7 @@ class Learnware: if isinstance(module_path, str) and module_path.endswith(".py") and not os.path.isabs(module_path): module_path = os.path.join(learnware_dirpath, module_path) return module_path - + def instantiate_model(self): if isinstance(self.model, BaseModel): logger.info("The learnware had been instantiated, thus the instantiation operation is ignored!") @@ -72,13 +72,13 @@ class Learnware: def get_model(self) -> Union[dict, BaseModel]: return self.model - + def get_specification(self) -> Specification: return self.specification def get_dirpath(self) -> str: return self.learnware_dirpath - + def update_stat_spec(self, name, new_stat_spec: BaseStatSpecification): self.specification.update_stat_spec(**{name: new_stat_spec}) diff --git a/learnware/market/__init__.py b/learnware/market/__init__.py index b20753f..87e8c28 100644 --- a/learnware/market/__init__.py +++ b/learnware/market/__init__.py @@ -7,4 +7,4 @@ from .hetergeneous import HeterogeneousOrganizer, MappingFunction from .easy import EasyMarket from .module import instatiate_learnware_market -from .classes import CondaChecker \ No newline at end of file +from .classes import CondaChecker diff --git a/learnware/market/classes.py b/learnware/market/classes.py index 65261a3..e8ca530 100644 --- a/learnware/market/classes.py +++ b/learnware/market/classes.py @@ -7,13 +7,12 @@ class CondaChecker(BaseChecker): def __init__(self, inner_checker, **kwargs): self.inner_checker = inner_checker super(CondaChecker, self).__init__(**kwargs) - + def __call__(self, learnware: Learnware) -> int: - with LearnwaresContainer(learnware) as env_container: learnwares = env_container.get_learnwares_with_container() if len(learnwares) == 0: raise AssertionError(f"The env of learnware {learnware} installed failed") check_status = self.inner_checker(learnware[0]) - - return check_status \ No newline at end of file + + return check_status