Merge pull request #23 from Learnware-LAMDA/test_all_learnware

[ENH | MNT] speed up conda search and test all learnware in system

learnware/client/package_utils.py

@@ -1,4 +1,7 @@
+import os
+import json
 import yaml
+import tempfile
 import subprocess
 from typing import List, Tuple
 
@@ -96,18 +99,35 @@ def filter_nonexist_conda_packages(packages: list) -> Tuple[List[str], List[str]
         nonexist_packages: list of non-exist packages
     """
 
-    exist_packages = []
-    nonexist_packages = []
-    for package in packages:
-        try:
-            try_to_run(args=["conda", "search", package], timeout=5)
-            exist_packages.append(package)
-        except Exception as e:
-            nonexist_packages.append(package)
-            pass
-        pass
-
-    return exist_packages, nonexist_packages
+    test_yaml = {
+        "channels": ["defaults"],
+        "dependencies": packages,
+    }
+
+    with tempfile.TemporaryDirectory(prefix="conda_filter_") as tempdir:
+        test_yaml_file = os.path.join(tempdir, "environment.yaml")
+        with open(test_yaml_file, "w") as fout:
+            yaml.safe_dump(test_yaml, fout)
+
+        command = f"conda env create --name env_test --file {test_yaml_file} --dry-run --json"
+        result = subprocess.run(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+        output = json.loads(result.stdout.strip()).get("bad_deps", [])
+
+        if len(output) > 0:
+            exist_packages = []
+            nonexist_packages = []
+            error_packages = set([package.replace("=", "") for package in output])
+
+            for package in packages:
+                if package.replace("=", "") in error_packages:
+                    nonexist_packages.append(package)
+                else:
+                    exist_packages.append(package)
+            logger.info(f"Filtered out {len(nonexist_packages)} non-exist conda dependencies.")
+
+            return exist_packages, nonexist_packages
+        else:
+            return packages, []
 
 
 def read_conda_packages_from_dict(env_desc: dict) -> Tuple[List[str], List[str]]:

learnware/client/utils.py

@@ -14,7 +14,7 @@ def system_execute(args, timeout=None):
     try:
         com_process.check_returncode()
     except subprocess.CalledProcessError as err:
-        print("System Execute Error:", com_process.stderr.decode())
+        logger.error(f"System Execute Error: {com_process.stderr.decode()}")
         raise err
 
 
@@ -47,9 +47,9 @@ def install_environment(zip_path, conda_env):
                 logger.info(f"checking the avaliabe conda packages for {conda_env}")
                 filter_nonexist_conda_packages_file(yaml_file=yaml_path, output_yaml_file=yaml_path_filter)
                 # create environment
-                logger.info(f"create and update conda env [{conda_env}] according to .yaml file")
+                logger.info(f"create conda env [{conda_env}] according to .yaml file")
                 system_execute(
-                    args=["conda", "env", "update", "--name", f"{conda_env}", "--file", f"{yaml_path_filter}"]
+                    args=["conda", "env", "create", "--name", f"{conda_env}", "--file", f"{yaml_path_filter}"]
                 )
 
             elif "requirements.txt" in z_file.namelist():

learnware/test/module.py

@@ -3,7 +3,7 @@ def get_semantic_specification():
     semantic_specification["Data"] = {"Type": "Class", "Values": ["Text"]}
     semantic_specification["Task"] = {"Type": "Class", "Values": ["Ranking"]}
     semantic_specification["Library"] = {"Type": "Class", "Values": ["Scikit-learn"]}
-    semantic_specification["Scenario"] = {"Type": "Tag", "Values": "Financial"}
+    semantic_specification["Scenario"] = {"Type": "Tag", "Values": ["Financial"]}
     semantic_specification["Name"] = {"Type": "String", "Values": "test"}
     semantic_specification["Description"] = {"Type": "String", "Values": "test"}
     return semantic_specification

tests/test_learnware_client/test_all_learnware.py

@@ -0,0 +1,50 @@
+import os
+import unittest
+import tempfile
+
+from learnware.client import LearnwareClient
+from learnware.specification import Specification
+
+
+class TestAllLearnware(unittest.TestCase):
+    def setUp(self):
+        unittest.TestCase.setUpClass()
+        email = "liujd@lamda.nju.edu.cn"
+        token = "f7e647146a314c6e8b4e2e1079c4bca4"
+
+        self.client = LearnwareClient()
+        self.client.login(email, token)
+
+    def test_all_learnware(self):
+        max_learnware_num = 1000
+        semantic_spec = dict()
+        semantic_spec["Data"] = {"Type": "Class", "Values": []}
+        semantic_spec["Task"] = {"Type": "Class", "Values": []}
+        semantic_spec["Library"] = {"Type": "Class", "Values": []}
+        semantic_spec["Scenario"] = {"Type": "Tag", "Values": []}
+        semantic_spec["Name"] = {"Type": "String", "Values": ""}
+        semantic_spec["Description"] = {"Type": "String", "Values": ""}
+
+        specification = Specification(semantic_spec=semantic_spec)
+        result = self.client.search_learnware(specification, page_size=max_learnware_num)
+        print(f"result size: {len(result)}")
+        print(f"key in result: {[key for key in result[0]]}")
+
+        failed_ids = []
+        learnware_ids = [res["learnware_id"] for res in result]
+        with tempfile.TemporaryDirectory(prefix="learnware_") as tempdir:
+            for idx in learnware_ids:
+                zip_path = os.path.join(tempdir, f"test_{idx}.zip")
+                self.client.download_learnware(idx, zip_path)
+                try:
+                    LearnwareClient.check_learnware(zip_path)
+                    print(f"check learnware {idx} succeed")
+                except:
+                    failed_ids.append(idx)
+                    print(f"check learnware {idx} failed!!!")
+
+        print(f"failed learnware ids: {failed_ids}")
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_learnware_client/test_check_learnware.py

@@ -14,12 +14,19 @@ class TestCheckLearnware(unittest.TestCase):
 
         self.client = LearnwareClient()
         self.client.login(email, token)
-        self.learnware_id = "00000154"
 
-    def test_check_learnware(self):
+    def test_check_learnware_pip(self):
+        learnware_id = "00000154"
         with tempfile.TemporaryDirectory(prefix="learnware_") as tempdir:
             self.zip_path = os.path.join(tempdir, "test.zip")
-            self.client.download_learnware(self.learnware_id, self.zip_path)
+            self.client.download_learnware(learnware_id, self.zip_path)
+            LearnwareClient.check_learnware(self.zip_path)
+
+    def test_check_learnware_conda(self):
+        learnware_id = "00000148"
+        with tempfile.TemporaryDirectory(prefix="learnware_") as tempdir:
+            self.zip_path = os.path.join(tempdir, "test.zip")
+            self.client.download_learnware(learnware_id, self.zip_path)
             LearnwareClient.check_learnware(self.zip_path)