Merge pull request #77 from Learnware-LAMDA/fix_conda_win

[FIX] fix bugs in conda filter

learnware/client/learnware_client.py

@@ -406,6 +406,8 @@ class LearnwareClient:
 
             assert LearnwareClient._check_stat_specification(learnware), "Stat specification check failed!"
 
+        logger.info("The learnware has passed the test.")
+
     def cleanup(self):
         for tempdir in self.tempdir_list:
             tempdir.cleanup()

learnware/client/package_utils.py

@@ -1,4 +1,5 @@
 import os
+import re
 import json
 import yaml
 import tempfile
@@ -95,43 +96,50 @@ def filter_nonexist_conda_packages(packages: list) -> Tuple[List[str], List[str]
         exist_packages: list of exist packages
         nonexist_packages: list of non-exist packages
     """
-    test_yaml = {
+
+    def _process_dependency(dependency):
+        modified_dependency = re.sub(r"=+", "=", dependency)
+        match = re.search(r"([^=]*=[^=]*)=", modified_dependency)
+        return match.group(1) if match else modified_dependency
+
+    def _dry_run(test_yaml):
+        with tempfile.TemporaryDirectory(prefix="conda_filter_") as tempdir:
+            test_yaml_file = os.path.join(tempdir, "environment.yaml")
+            with open(test_yaml_file, "w") as fout:
+                yaml.safe_dump(test_yaml, fout)
+
+            command = f"conda env create --name env_test --file {test_yaml_file} --dry-run --json"
+            result = subprocess.run(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+            stdout = result.stdout.strip()
+            last_bracket = stdout.rfind("\n{")
+            if last_bracket != -1:
+                stdout = stdout[last_bracket:]
+            return json.loads(stdout).get("bad_deps", [])
+
+    org_yaml = {
         "channels": ["defaults"],
         "dependencies": packages,
     }
+    bad_deps = _dry_run(org_yaml)
+    if len(bad_deps) == 0:
+        return packages, []
 
-    with tempfile.TemporaryDirectory(prefix="conda_filter_") as tempdir:
-        test_yaml_file = os.path.join(tempdir, "environment.yaml")
-        with open(test_yaml_file, "w") as fout:
-            yaml.safe_dump(test_yaml, fout)
-
-        command = f"conda env create --name env_test --file {test_yaml_file} --dry-run --json"
-        result = subprocess.run(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
-        stdout = result.stdout.strip()
-        last_bracket = stdout.rfind("\n{")
-        if last_bracket != -1:
-            stdout = stdout[last_bracket:]
-        print(stdout)
-        output = json.loads(stdout).get("bad_deps", [])
-
-        if len(output) > 0:
-            exist_packages = []
-            nonexist_packages = []
-            error_packages = set([package.replace("=", "") for package in output])
-
-            for package in packages:
-                if package.replace("=", "") in error_packages:
-                    nonexist_packages.append(package)
-                else:
-                    exist_packages.append(package)
-            logger.info(f"Filtered out {len(nonexist_packages)} non-exist conda dependencies.")
-
-            if not any(package.startswith("python=") for package in exist_packages):
-                exist_packages = ["python=3.8"] + exist_packages
-
-            return exist_packages, nonexist_packages
+    exist_packages = []
+    nonexist_packages = []
+    bad_deps = set([package.replace("=", "") for package in bad_deps])
+    for package in packages:
+        if package.replace("=", "") in bad_deps:
+            if package.startswith("python="):
+                exist_packages.append(_process_dependency(package))
+            else:
+                nonexist_packages.append(package)
         else:
-            return packages, []
+            exist_packages.append(package)
+    logger.info(f"Filtered out {len(nonexist_packages)} non-exist conda dependencies.")
+
+    if not any(package.startswith("python=") for package in exist_packages):
+        exist_packages = ["python=3.8"] + exist_packages
+    return exist_packages, nonexist_packages
 
 
 def read_conda_packages_from_dict(env_desc: dict) -> Tuple[List[str], List[str]]:

tests/test_learnware_client/test_load_docker.py

@@ -37,8 +37,6 @@ class TestLearnwareLoad(unittest.TestCase):
 
     def test_load_multi_learnware_by_id(self):
         learnware_list = self.client.load_learnware(learnware_id=self.learnware_ids, runnable_option="docker")
-        docker_container = learnware_list[0].get_model().docker_container
-
         reuser = AveragingReuser(learnware_list, mode="vote_by_label")
         input_array = np.random.random(size=(20, 13))
         print(reuser.predict(input_array))
@@ -46,8 +44,6 @@ class TestLearnwareLoad(unittest.TestCase):
         for learnware in learnware_list:
             print(learnware.id, learnware.predict(input_array))
 
-        learnware_list[0].get_model()._destroy_docker_container(docker_container)
-
     def test_load_single_learnware_by_id_pip(self):
         learnware_id = "00000147"
         learnware = self.client.load_learnware(learnware_id=learnware_id, runnable_option="docker")