From 3a00a2dbf64bc3ea55a28a95fc31f33e9ff4538b Mon Sep 17 00:00:00 2001 From: Gene Date: Wed, 25 Oct 2023 16:43:01 +0800 Subject: [PATCH 1/6] [ENH] add docker runnable_option in load --- learnware/client/learnware_client.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/learnware/client/learnware_client.py b/learnware/client/learnware_client.py index e3dd28e..097852b 100644 --- a/learnware/client/learnware_client.py +++ b/learnware/client/learnware_client.py @@ -306,14 +306,15 @@ class LearnwareClient: the option for instantiating learnwares - "normal": instantiate learnware without installing environment - "conda_env": instantiate learnware with installing conda virtual environment + - "docker": instantiate learnware with creating docker container Returns ------- Learnware The contructed learnware object or object list """ - if runnable_option is not None and runnable_option not in ["normal", "conda_env"]: - raise logger.warning(f"runnable_option must be one of ['normal', 'conda_env'], but got {runnable_option}") + if runnable_option is not None and runnable_option not in ["normal", "conda_env", "docker"]: + raise logger.warning(f"runnable_option must be one of ['normal', 'conda_env', 'docker'], but got {runnable_option}") if learnware_path is None and learnware_id is None: raise ValueError("Requires one of learnware_path or learnware_id") @@ -379,7 +380,10 @@ class LearnwareClient: for i in range(len(learnware_list)): learnware_list[i].instantiate_model() elif runnable_option == "conda_env": - with LearnwaresContainer(learnware_list, zip_paths, cleanup=False) as env_container: + with LearnwaresContainer(learnware_list, zip_paths, cleanup=False, mode="conda") as env_container: + learnware_list = env_container.get_learnwares_with_container() + elif runnable_option == "docker": + with LearnwaresContainer(learnware_list, zip_paths, cleanup=False, mode="docker") as env_container: learnware_list = env_container.get_learnwares_with_container() if len(learnware_list) == 1: From 28a7125d8233cd95a97bb30f42595e80330ba553 Mon Sep 17 00:00:00 2001 From: Gene Date: Wed, 25 Oct 2023 16:45:39 +0800 Subject: [PATCH 2/6] [ENH | FIX] add the auto destroy of containers --- learnware/client/container.py | 46 ++++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/learnware/client/container.py b/learnware/client/container.py index b4ae280..33357db 100644 --- a/learnware/client/container.py +++ b/learnware/client/container.py @@ -222,8 +222,18 @@ class ModelDockerContainer(ModelContainer): @staticmethod def _destroy_docker_container(docker_container): - docker_container.stop() - docker_container.remove() + if isinstance(docker_container, docker.models.containers.Container): + client = docker.from_env() + container_ids = [container.id for container in client.containers.list()] + + if docker_container.id in container_ids: + docker_container.stop() + docker_container.remove() + logger.info("Docker container is stopped and removed.") + else: + logger.info("Docker container has already been removed.") + else: + logger.error("Type of docker_container is not docker.models.containers.Container.") def _copy_file_to_container(self, local_path, container_path): directory_path = os.path.dirname(container_path) @@ -268,9 +278,10 @@ class ModelDockerContainer(ModelContainer): Exception Lack of the environment configuration file. """ - run_cmd_times = 5 + run_cmd_times = 10 with tempfile.TemporaryDirectory(prefix="learnware_") as tempdir: with zipfile.ZipFile(file=zip_path, mode="r") as z_file: + success_flag = False logger.info(f"zip_file namelist: {z_file.namelist()}") if "environment.yaml" in z_file.namelist(): @@ -282,7 +293,7 @@ class ModelDockerContainer(ModelContainer): self._copy_file_to_container(yaml_path_filter, yaml_path_filter) # create environment - logger.info(f"create and update conda env [{conda_env}] according to .yaml file") + logger.info(f"Create and update conda env [{conda_env}] according to .yaml file") for i in range(run_cmd_times): result = self.docker_container.exec_run( " ".join( @@ -290,24 +301,25 @@ class ModelDockerContainer(ModelContainer): ) ) if result.exit_code == 0: + success_flag = True break elif "requirements.txt" in z_file.namelist(): z_file.extract(member="requirements.txt", path=tempdir) requirements_path: str = os.path.join(tempdir, "requirements.txt") requirements_path_filter: str = os.path.join(tempdir, "requirements_filter.txt") - logger.info(f"checking the avaliabe pip packages for {conda_env}") + logger.info(f"checking the avaliabe pip packages for {conda_env}.") filter_nonexist_pip_packages_file( requirements_file=requirements_path, output_file=requirements_path_filter ) - logger.info(f"create empty conda env [{conda_env}]") + logger.info(f"Create empty conda env [{conda_env}] in docker.") for i in range(run_cmd_times): result = self.docker_container.exec_run( " ".join(["conda", "create", "-y", "--name", f"{conda_env}", "python=3.8"]) ) if result.exit_code == 0: break - logger.info(f"install pip requirements for conda env [{conda_env}]") + logger.info(f"install pip requirements for conda env [{conda_env}] in docker.") self._copy_file_to_container(requirements_path_filter, requirements_path_filter) for i in range(run_cmd_times): @@ -329,11 +341,16 @@ class ModelDockerContainer(ModelContainer): ) ) if result.exit_code == 0: + success_flag = True break else: raise Exception("Environment.yaml or requirements.txt not found in the learnware zip file.") + + if not success_flag: + logger.error(f"Install conda env [{conda_env}] in docker failed!") - logger.info(f"install learnware package for conda env [{conda_env}]") + success_flag = False + logger.info(f"Install learnware package for conda env [{conda_env}] in docker.") for i in range(run_cmd_times): result = self.docker_container.exec_run( " ".join( @@ -352,7 +369,11 @@ class ModelDockerContainer(ModelContainer): ) ) if result.exit_code == 0: + success_flag = True break + + if not success_flag: + logger.error(f"Install learnware package for conda env [{conda_env}] in docker failed!") def _setup_env_and_metadata(self): """setup env and set the input and output shape by communicating with docker""" @@ -410,7 +431,7 @@ class ModelDockerContainer(ModelContainer): def _remove_env(self): """remove the docker container""" - ModelDockerContainer._destroy_docker_container(docker_container) + ModelDockerContainer._destroy_docker_container(self.docker_container) def _run_model_with_script(self, method, **kargs): with tempfile.TemporaryDirectory(prefix="learnware_") as tempdir: @@ -517,6 +538,7 @@ class LearnwaresContainer: ) for _learnware, _zippath in zip(self.learnware_list, self.learnware_zippaths) ] + atexit.register(ModelDockerContainer._destroy_docker_container, self._docker_container) model_list = [_learnware.get_model() for _learnware in self.learnware_containers] with ThreadPoolExecutor(max_workers=max(os.cpu_count() // 2, 1)) as executor: @@ -527,6 +549,12 @@ class LearnwaresContainer: logger.warning( f"{len(self.learnware_list) - sum(results)} of {len(self.learnware_list)} learnwares init failed! This learnware will be ignored" ) + + # if not self.cleanup and self.mode == "docker": + # _model_docker_container = self.learnware_containers[0].get_model() + # _model_docker_container.cleanup_flag = True + # atexit.register(_model_docker_container.remove_env) + return self def __exit__(self, exc_type, exc_val, exc_tb): From 0849096e35de42367a05c0f9e3fb3a03970b095d Mon Sep 17 00:00:00 2001 From: Gene Date: Wed, 25 Oct 2023 16:46:03 +0800 Subject: [PATCH 3/6] [MNT] modify filename --- .../test_learnware_client/test_load_conda.py | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 tests/test_learnware_client/test_load_conda.py diff --git a/tests/test_learnware_client/test_load_conda.py b/tests/test_learnware_client/test_load_conda.py new file mode 100644 index 0000000..26b0bd5 --- /dev/null +++ b/tests/test_learnware_client/test_load_conda.py @@ -0,0 +1,75 @@ +import os +import unittest +import zipfile +import numpy as np + +import learnware +from learnware.learnware import get_learnware_from_dirpath +from learnware.client import LearnwareClient +from learnware.client.container import ModelCondaContainer, LearnwaresContainer +from learnware.learnware.reuse import AveragingReuser + + +class TestLearnwareLoad(unittest.TestCase): + def setUp(self): + unittest.TestCase.setUpClass() + email = "liujd@lamda.nju.edu.cn" + token = "f7e647146a314c6e8b4e2e1079c4bca4" + + self.client = LearnwareClient() + self.client.login(email, token) + + root = os.path.dirname(__file__) + self.learnware_ids = ["00000084", "00000154", "00000155"] + self.zip_paths = [os.path.join(root, x) for x in ["1.zip", "2.zip", "3.zip"]] + + def test_load_single_learnware_by_zippath(self): + for learnware_id, zip_path in zip(self.learnware_ids, self.zip_paths): + self.client.download_learnware(learnware_id, zip_path) + + learnware_list = [ + self.client.load_learnware(learnware_path=zippath, runnable_option="conda_env") + for zippath in self.zip_paths + ] + reuser = AveragingReuser(learnware_list, mode="vote_by_label") + input_array = np.random.random(size=(20, 13)) + print(reuser.predict(input_array)) + + for learnware in learnware_list: + print(learnware.id, learnware.predict(input_array)) + + def test_load_multi_learnware_by_zippath(self): + for learnware_id, zip_path in zip(self.learnware_ids, self.zip_paths): + self.client.download_learnware(learnware_id, zip_path) + + learnware_list = self.client.load_learnware(learnware_path=self.zip_paths, runnable_option="conda_env") + reuser = AveragingReuser(learnware_list, mode="vote_by_label") + input_array = np.random.random(size=(20, 13)) + print(reuser.predict(input_array)) + + for learnware in learnware_list: + print(learnware.id, learnware.predict(input_array)) + + def test_load_single_learnware_by_id(self): + learnware_list = [ + self.client.load_learnware(learnware_id=idx, runnable_option="conda_env") for idx in self.learnware_ids + ] + reuser = AveragingReuser(learnware_list, mode="vote_by_label") + input_array = np.random.random(size=(20, 13)) + print(reuser.predict(input_array)) + + for learnware in learnware_list: + print(learnware.id, learnware.predict(input_array)) + + def test_load_multi_learnware_by_id(self): + learnware_list = self.client.load_learnware(learnware_id=self.learnware_ids, runnable_option="conda_env") + reuser = AveragingReuser(learnware_list, mode="vote_by_label") + input_array = np.random.random(size=(20, 13)) + print(reuser.predict(input_array)) + + for learnware in learnware_list: + print(learnware.id, learnware.predict(input_array)) + + +if __name__ == "__main__": + unittest.main() From eb3a6d85564a66c2237cd31668ada19c6f54c9df Mon Sep 17 00:00:00 2001 From: Gene Date: Wed, 25 Oct 2023 16:46:32 +0800 Subject: [PATCH 4/6] [MNT] add docker load test --- .../test_learnware_client/test_load_docker.py | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 tests/test_learnware_client/test_load_docker.py diff --git a/tests/test_learnware_client/test_load_docker.py b/tests/test_learnware_client/test_load_docker.py new file mode 100644 index 0000000..f48c719 --- /dev/null +++ b/tests/test_learnware_client/test_load_docker.py @@ -0,0 +1,53 @@ +import os +import unittest +import zipfile +import numpy as np + +import learnware +from learnware.learnware import get_learnware_from_dirpath +from learnware.client import LearnwareClient +from learnware.client.container import ModelCondaContainer, LearnwaresContainer +from learnware.learnware.reuse import AveragingReuser + + +class TestLearnwareLoad(unittest.TestCase): + def setUp(self): + unittest.TestCase.setUpClass() + email = "liujd@lamda.nju.edu.cn" + token = "f7e647146a314c6e8b4e2e1079c4bca4" + + self.client = LearnwareClient() + self.client.login(email, token) + + root = os.path.dirname(__file__) + self.learnware_ids = ["00000084", "00000154", "00000155"] + self.zip_paths = [os.path.join(root, x) for x in ["1.zip", "2.zip", "3.zip"]] + + def test_load_multi_learnware_by_zippath(self): + for learnware_id, zip_path in zip(self.learnware_ids, self.zip_paths): + self.client.download_learnware(learnware_id, zip_path) + + learnware_list = self.client.load_learnware(learnware_path=self.zip_paths, runnable_option="docker") + reuser = AveragingReuser(learnware_list, mode="vote_by_label") + input_array = np.random.random(size=(20, 13)) + print(reuser.predict(input_array)) + + for learnware in learnware_list: + print(learnware.id, learnware.predict(input_array)) + + def test_load_multi_learnware_by_id(self): + learnware_list = self.client.load_learnware(learnware_id=self.learnware_ids, runnable_option="docker") + docker_container = learnware_list[0].get_model().docker_container + + reuser = AveragingReuser(learnware_list, mode="vote_by_label") + input_array = np.random.random(size=(20, 13)) + print(reuser.predict(input_array)) + + for learnware in learnware_list: + print(learnware.id, learnware.predict(input_array)) + + learnware_list[0].get_model()._destroy_docker_container(docker_container) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file From f6c91c7caae7a7dde16093b5ad3f4f1da3ce981e Mon Sep 17 00:00:00 2001 From: Gene Date: Wed, 25 Oct 2023 16:46:57 +0800 Subject: [PATCH 5/6] [MNT] update load test --- tests/test_learnware_client/test_docker.py | 33 ---------- tests/test_learnware_client/test_load.py | 75 ---------------------- 2 files changed, 108 deletions(-) delete mode 100644 tests/test_learnware_client/test_docker.py delete mode 100644 tests/test_learnware_client/test_load.py diff --git a/tests/test_learnware_client/test_docker.py b/tests/test_learnware_client/test_docker.py deleted file mode 100644 index 5a916cc..0000000 --- a/tests/test_learnware_client/test_docker.py +++ /dev/null @@ -1,33 +0,0 @@ -import os -import zipfile -import numpy as np - -import learnware -from learnware.client import LearnwareClient -from learnware.client.container import LearnwaresContainer -from learnware.learnware.reuse import AveragingReuser - - -if __name__ == "__main__": - email = "liujd@lamda.nju.edu.cn" - token = "f7e647146a314c6e8b4e2e1079c4bca4" - - client = LearnwareClient() - client.login(email, token) - - root = os.path.dirname(__file__) - learnware_ids = ["00000084", "00000154", "00000155"] - zip_paths = [os.path.join(root, x) for x in ["1.zip", "2.zip", "3.zip"]] - - for learnware_id, zip_path in zip(learnware_ids, zip_paths): - client.download_learnware(learnware_id, zip_path) - - learnware_list = [client.load_learnware(learnware_path=zippath) for zippath in zip_paths] - with LearnwaresContainer(learnware_list, zip_paths, mode="docker") as env_container: - learnware_list = env_container.get_learnwares_with_container() - reuser = AveragingReuser(learnware_list, mode="vote_by_label") - input_array = np.random.random(size=(20, 13)) - print(reuser.predict(input_array)) - - for learnware in learnware_list: - print(learnware.id, learnware.predict(input_array)) diff --git a/tests/test_learnware_client/test_load.py b/tests/test_learnware_client/test_load.py deleted file mode 100644 index 26b0bd5..0000000 --- a/tests/test_learnware_client/test_load.py +++ /dev/null @@ -1,75 +0,0 @@ -import os -import unittest -import zipfile -import numpy as np - -import learnware -from learnware.learnware import get_learnware_from_dirpath -from learnware.client import LearnwareClient -from learnware.client.container import ModelCondaContainer, LearnwaresContainer -from learnware.learnware.reuse import AveragingReuser - - -class TestLearnwareLoad(unittest.TestCase): - def setUp(self): - unittest.TestCase.setUpClass() - email = "liujd@lamda.nju.edu.cn" - token = "f7e647146a314c6e8b4e2e1079c4bca4" - - self.client = LearnwareClient() - self.client.login(email, token) - - root = os.path.dirname(__file__) - self.learnware_ids = ["00000084", "00000154", "00000155"] - self.zip_paths = [os.path.join(root, x) for x in ["1.zip", "2.zip", "3.zip"]] - - def test_load_single_learnware_by_zippath(self): - for learnware_id, zip_path in zip(self.learnware_ids, self.zip_paths): - self.client.download_learnware(learnware_id, zip_path) - - learnware_list = [ - self.client.load_learnware(learnware_path=zippath, runnable_option="conda_env") - for zippath in self.zip_paths - ] - reuser = AveragingReuser(learnware_list, mode="vote_by_label") - input_array = np.random.random(size=(20, 13)) - print(reuser.predict(input_array)) - - for learnware in learnware_list: - print(learnware.id, learnware.predict(input_array)) - - def test_load_multi_learnware_by_zippath(self): - for learnware_id, zip_path in zip(self.learnware_ids, self.zip_paths): - self.client.download_learnware(learnware_id, zip_path) - - learnware_list = self.client.load_learnware(learnware_path=self.zip_paths, runnable_option="conda_env") - reuser = AveragingReuser(learnware_list, mode="vote_by_label") - input_array = np.random.random(size=(20, 13)) - print(reuser.predict(input_array)) - - for learnware in learnware_list: - print(learnware.id, learnware.predict(input_array)) - - def test_load_single_learnware_by_id(self): - learnware_list = [ - self.client.load_learnware(learnware_id=idx, runnable_option="conda_env") for idx in self.learnware_ids - ] - reuser = AveragingReuser(learnware_list, mode="vote_by_label") - input_array = np.random.random(size=(20, 13)) - print(reuser.predict(input_array)) - - for learnware in learnware_list: - print(learnware.id, learnware.predict(input_array)) - - def test_load_multi_learnware_by_id(self): - learnware_list = self.client.load_learnware(learnware_id=self.learnware_ids, runnable_option="conda_env") - reuser = AveragingReuser(learnware_list, mode="vote_by_label") - input_array = np.random.random(size=(20, 13)) - print(reuser.predict(input_array)) - - for learnware in learnware_list: - print(learnware.id, learnware.predict(input_array)) - - -if __name__ == "__main__": - unittest.main() From 0d2b4017b9a11c7155a0d3e49247822b338b44ce Mon Sep 17 00:00:00 2001 From: Gene Date: Wed, 25 Oct 2023 16:47:22 +0800 Subject: [PATCH 6/6] [MNT] format code by black --- learnware/client/container.py | 10 +++++----- learnware/client/learnware_client.py | 4 +++- tests/test_learnware_client/test_load_docker.py | 6 +++--- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/learnware/client/container.py b/learnware/client/container.py index 33357db..243d580 100644 --- a/learnware/client/container.py +++ b/learnware/client/container.py @@ -225,7 +225,7 @@ class ModelDockerContainer(ModelContainer): if isinstance(docker_container, docker.models.containers.Container): client = docker.from_env() container_ids = [container.id for container in client.containers.list()] - + if docker_container.id in container_ids: docker_container.stop() docker_container.remove() @@ -345,7 +345,7 @@ class ModelDockerContainer(ModelContainer): break else: raise Exception("Environment.yaml or requirements.txt not found in the learnware zip file.") - + if not success_flag: logger.error(f"Install conda env [{conda_env}] in docker failed!") @@ -371,7 +371,7 @@ class ModelDockerContainer(ModelContainer): if result.exit_code == 0: success_flag = True break - + if not success_flag: logger.error(f"Install learnware package for conda env [{conda_env}] in docker failed!") @@ -549,12 +549,12 @@ class LearnwaresContainer: logger.warning( f"{len(self.learnware_list) - sum(results)} of {len(self.learnware_list)} learnwares init failed! This learnware will be ignored" ) - + # if not self.cleanup and self.mode == "docker": # _model_docker_container = self.learnware_containers[0].get_model() # _model_docker_container.cleanup_flag = True # atexit.register(_model_docker_container.remove_env) - + return self def __exit__(self, exc_type, exc_val, exc_tb): diff --git a/learnware/client/learnware_client.py b/learnware/client/learnware_client.py index 097852b..5b213dd 100644 --- a/learnware/client/learnware_client.py +++ b/learnware/client/learnware_client.py @@ -314,7 +314,9 @@ class LearnwareClient: The contructed learnware object or object list """ if runnable_option is not None and runnable_option not in ["normal", "conda_env", "docker"]: - raise logger.warning(f"runnable_option must be one of ['normal', 'conda_env', 'docker'], but got {runnable_option}") + raise logger.warning( + f"runnable_option must be one of ['normal', 'conda_env', 'docker'], but got {runnable_option}" + ) if learnware_path is None and learnware_id is None: raise ValueError("Requires one of learnware_path or learnware_id") diff --git a/tests/test_learnware_client/test_load_docker.py b/tests/test_learnware_client/test_load_docker.py index f48c719..8fedfa7 100644 --- a/tests/test_learnware_client/test_load_docker.py +++ b/tests/test_learnware_client/test_load_docker.py @@ -38,16 +38,16 @@ class TestLearnwareLoad(unittest.TestCase): def test_load_multi_learnware_by_id(self): learnware_list = self.client.load_learnware(learnware_id=self.learnware_ids, runnable_option="docker") docker_container = learnware_list[0].get_model().docker_container - + reuser = AveragingReuser(learnware_list, mode="vote_by_label") input_array = np.random.random(size=(20, 13)) print(reuser.predict(input_array)) for learnware in learnware_list: print(learnware.id, learnware.predict(input_array)) - + learnware_list[0].get_model()._destroy_docker_container(docker_container) if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main()