From 2e69b63275efb9c704fba18a3cf116b5ba42df96 Mon Sep 17 00:00:00 2001 From: Gene Date: Fri, 17 Nov 2023 14:07:04 +0800 Subject: [PATCH 1/4] [FIX] fix bugs in conda filter --- learnware/client/package_utils.py | 72 +++++++++++++++++-------------- 1 file changed, 40 insertions(+), 32 deletions(-) diff --git a/learnware/client/package_utils.py b/learnware/client/package_utils.py index 9909891..9dea79d 100644 --- a/learnware/client/package_utils.py +++ b/learnware/client/package_utils.py @@ -1,4 +1,5 @@ import os +import re import json import yaml import tempfile @@ -95,43 +96,50 @@ def filter_nonexist_conda_packages(packages: list) -> Tuple[List[str], List[str] exist_packages: list of exist packages nonexist_packages: list of non-exist packages """ - test_yaml = { + + def _process_dependency(dependency): + modified_dependency = re.sub(r"=+", "=", dependency) + match = re.search(r"([^=]*=[^=]*)=", modified_dependency) + return match.group(1) if match else modified_dependency + + def _dry_run(test_yaml): + with tempfile.TemporaryDirectory(prefix="conda_filter_") as tempdir: + test_yaml_file = os.path.join(tempdir, "environment.yaml") + with open(test_yaml_file, "w") as fout: + yaml.safe_dump(test_yaml, fout) + + command = f"conda env create --name env_test --file {test_yaml_file} --dry-run --json" + result = subprocess.run(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + stdout = result.stdout.strip() + last_bracket = stdout.rfind("\n{") + if last_bracket != -1: + stdout = stdout[last_bracket:] + return json.loads(stdout).get("bad_deps", []) + + org_yaml = { "channels": ["defaults"], "dependencies": packages, } + bad_deps = _dry_run(org_yaml) + if len(bad_deps) == 0: + return packages, [] - with tempfile.TemporaryDirectory(prefix="conda_filter_") as tempdir: - test_yaml_file = os.path.join(tempdir, "environment.yaml") - with open(test_yaml_file, "w") as fout: - yaml.safe_dump(test_yaml, fout) - - command = f"conda env create --name env_test --file {test_yaml_file} --dry-run --json" - result = subprocess.run(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) - stdout = result.stdout.strip() - last_bracket = stdout.rfind("\n{") - if last_bracket != -1: - stdout = stdout[last_bracket:] - print(stdout) - output = json.loads(stdout).get("bad_deps", []) - - if len(output) > 0: - exist_packages = [] - nonexist_packages = [] - error_packages = set([package.replace("=", "") for package in output]) - - for package in packages: - if package.replace("=", "") in error_packages: - nonexist_packages.append(package) - else: - exist_packages.append(package) - logger.info(f"Filtered out {len(nonexist_packages)} non-exist conda dependencies.") - - if not any(package.startswith("python=") for package in exist_packages): - exist_packages = ["python=3.8"] + exist_packages - - return exist_packages, nonexist_packages + exist_packages = [] + nonexist_packages = [] + bad_deps = set([package.replace("=", "") for package in bad_deps]) + for package in packages: + if package.replace("=", "") in bad_deps: + if package.startswith("python="): + exist_packages.append(_process_dependency(package)) + else: + nonexist_packages.append(package) else: - return packages, [] + exist_packages.append(package) + logger.info(f"Filtered out {len(nonexist_packages)} non-exist conda dependencies.") + + if not any(package.startswith("python=") for package in exist_packages): + exist_packages = ["python=3.8"] + exist_packages + return exist_packages, nonexist_packages def read_conda_packages_from_dict(env_desc: dict) -> Tuple[List[str], List[str]]: From 83355b64fa83720c2e39c637cd54c8f2007f5fcd Mon Sep 17 00:00:00 2001 From: Gene Date: Fri, 17 Nov 2023 14:07:45 +0800 Subject: [PATCH 2/4] [MNT] add a log in check_learnware --- learnware/client/learnware_client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/learnware/client/learnware_client.py b/learnware/client/learnware_client.py index 0249c53..fa5aef1 100644 --- a/learnware/client/learnware_client.py +++ b/learnware/client/learnware_client.py @@ -402,10 +402,12 @@ class LearnwareClient: ) if learnware is None: - raise Exception("The learnware is not valid.") + raise Exception("The learnware has passed the check.") assert LearnwareClient._check_stat_specification(learnware), "Stat specification check failed!" + logger.info("The learnware passed the test.") + def cleanup(self): for tempdir in self.tempdir_list: tempdir.cleanup() From e22505e431c3ffd514facf89a2db0ce6d126aa9f Mon Sep 17 00:00:00 2001 From: Gene Date: Fri, 17 Nov 2023 14:08:01 +0800 Subject: [PATCH 3/4] [FIX] fix bugs in docker test --- tests/test_learnware_client/test_load_docker.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/test_learnware_client/test_load_docker.py b/tests/test_learnware_client/test_load_docker.py index ac16afd..b64ffed 100644 --- a/tests/test_learnware_client/test_load_docker.py +++ b/tests/test_learnware_client/test_load_docker.py @@ -37,8 +37,6 @@ class TestLearnwareLoad(unittest.TestCase): def test_load_multi_learnware_by_id(self): learnware_list = self.client.load_learnware(learnware_id=self.learnware_ids, runnable_option="docker") - docker_container = learnware_list[0].get_model().docker_container - reuser = AveragingReuser(learnware_list, mode="vote_by_label") input_array = np.random.random(size=(20, 13)) print(reuser.predict(input_array)) @@ -46,8 +44,6 @@ class TestLearnwareLoad(unittest.TestCase): for learnware in learnware_list: print(learnware.id, learnware.predict(input_array)) - learnware_list[0].get_model()._destroy_docker_container(docker_container) - def test_load_single_learnware_by_id_pip(self): learnware_id = "00000147" learnware = self.client.load_learnware(learnware_id=learnware_id, runnable_option="docker") From 8f1a7e4992131f7c7f3e9147b4b3b512ede7e3c6 Mon Sep 17 00:00:00 2001 From: Gene Date: Fri, 17 Nov 2023 14:10:11 +0800 Subject: [PATCH 4/4] [FIX] fix log description in check_learnware --- learnware/client/learnware_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/learnware/client/learnware_client.py b/learnware/client/learnware_client.py index fa5aef1..2e040ec 100644 --- a/learnware/client/learnware_client.py +++ b/learnware/client/learnware_client.py @@ -402,11 +402,11 @@ class LearnwareClient: ) if learnware is None: - raise Exception("The learnware has passed the check.") + raise Exception("The learnware is not valid.") assert LearnwareClient._check_stat_specification(learnware), "Stat specification check failed!" - logger.info("The learnware passed the test.") + logger.info("The learnware has passed the test.") def cleanup(self): for tempdir in self.tempdir_list: