diff --git a/learnware/client/learnware_client.py b/learnware/client/learnware_client.py index 0249c53..2e040ec 100644 --- a/learnware/client/learnware_client.py +++ b/learnware/client/learnware_client.py @@ -406,6 +406,8 @@ class LearnwareClient: assert LearnwareClient._check_stat_specification(learnware), "Stat specification check failed!" + logger.info("The learnware has passed the test.") + def cleanup(self): for tempdir in self.tempdir_list: tempdir.cleanup() diff --git a/learnware/client/package_utils.py b/learnware/client/package_utils.py index 9909891..9dea79d 100644 --- a/learnware/client/package_utils.py +++ b/learnware/client/package_utils.py @@ -1,4 +1,5 @@ import os +import re import json import yaml import tempfile @@ -95,43 +96,50 @@ def filter_nonexist_conda_packages(packages: list) -> Tuple[List[str], List[str] exist_packages: list of exist packages nonexist_packages: list of non-exist packages """ - test_yaml = { + + def _process_dependency(dependency): + modified_dependency = re.sub(r"=+", "=", dependency) + match = re.search(r"([^=]*=[^=]*)=", modified_dependency) + return match.group(1) if match else modified_dependency + + def _dry_run(test_yaml): + with tempfile.TemporaryDirectory(prefix="conda_filter_") as tempdir: + test_yaml_file = os.path.join(tempdir, "environment.yaml") + with open(test_yaml_file, "w") as fout: + yaml.safe_dump(test_yaml, fout) + + command = f"conda env create --name env_test --file {test_yaml_file} --dry-run --json" + result = subprocess.run(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + stdout = result.stdout.strip() + last_bracket = stdout.rfind("\n{") + if last_bracket != -1: + stdout = stdout[last_bracket:] + return json.loads(stdout).get("bad_deps", []) + + org_yaml = { "channels": ["defaults"], "dependencies": packages, } + bad_deps = _dry_run(org_yaml) + if len(bad_deps) == 0: + return packages, [] - with tempfile.TemporaryDirectory(prefix="conda_filter_") as tempdir: - test_yaml_file = os.path.join(tempdir, "environment.yaml") - with open(test_yaml_file, "w") as fout: - yaml.safe_dump(test_yaml, fout) - - command = f"conda env create --name env_test --file {test_yaml_file} --dry-run --json" - result = subprocess.run(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) - stdout = result.stdout.strip() - last_bracket = stdout.rfind("\n{") - if last_bracket != -1: - stdout = stdout[last_bracket:] - print(stdout) - output = json.loads(stdout).get("bad_deps", []) - - if len(output) > 0: - exist_packages = [] - nonexist_packages = [] - error_packages = set([package.replace("=", "") for package in output]) - - for package in packages: - if package.replace("=", "") in error_packages: - nonexist_packages.append(package) - else: - exist_packages.append(package) - logger.info(f"Filtered out {len(nonexist_packages)} non-exist conda dependencies.") - - if not any(package.startswith("python=") for package in exist_packages): - exist_packages = ["python=3.8"] + exist_packages - - return exist_packages, nonexist_packages + exist_packages = [] + nonexist_packages = [] + bad_deps = set([package.replace("=", "") for package in bad_deps]) + for package in packages: + if package.replace("=", "") in bad_deps: + if package.startswith("python="): + exist_packages.append(_process_dependency(package)) + else: + nonexist_packages.append(package) else: - return packages, [] + exist_packages.append(package) + logger.info(f"Filtered out {len(nonexist_packages)} non-exist conda dependencies.") + + if not any(package.startswith("python=") for package in exist_packages): + exist_packages = ["python=3.8"] + exist_packages + return exist_packages, nonexist_packages def read_conda_packages_from_dict(env_desc: dict) -> Tuple[List[str], List[str]]: diff --git a/tests/test_learnware_client/test_load_docker.py b/tests/test_learnware_client/test_load_docker.py index ac16afd..b64ffed 100644 --- a/tests/test_learnware_client/test_load_docker.py +++ b/tests/test_learnware_client/test_load_docker.py @@ -37,8 +37,6 @@ class TestLearnwareLoad(unittest.TestCase): def test_load_multi_learnware_by_id(self): learnware_list = self.client.load_learnware(learnware_id=self.learnware_ids, runnable_option="docker") - docker_container = learnware_list[0].get_model().docker_container - reuser = AveragingReuser(learnware_list, mode="vote_by_label") input_array = np.random.random(size=(20, 13)) print(reuser.predict(input_array)) @@ -46,8 +44,6 @@ class TestLearnwareLoad(unittest.TestCase): for learnware in learnware_list: print(learnware.id, learnware.predict(input_array)) - learnware_list[0].get_model()._destroy_docker_container(docker_container) - def test_load_single_learnware_by_id_pip(self): learnware_id = "00000147" learnware = self.client.load_learnware(learnware_id=learnware_id, runnable_option="docker")