Merge branch 'dev' of git.nju.edu.cn:learnware/learnware-market into dev

examples/example_market_db/example_db.py

@@ -16,7 +16,8 @@ def prepare_learnware(learnware_num=10):
         os.makedirs(dir_path, exist_ok=True)
 
         print("Preparing Learnware: %d" % (i))
-        data_X = np.random.randn(5000, 20)
+        data_X = np.random.randn(5000, 20) * i
+        # print(data_X[:10])
         data_y = np.random.randn(5000)
         data_y = np.where(data_y > 0, 1, 0)
 
@@ -32,6 +33,7 @@ def prepare_learnware(learnware_num=10):
 
 
 def test_market():
+    database_ops.clear_learnware_table()
     easy_market = EasyMarket()
     print("Total Item:", len(easy_market))
     test_learnware_num = 10
@@ -56,7 +58,7 @@ def test_market():
     print("Available ids:", curr_inds)
 
 
-def test_search():
+def test_search_sementics():
     easy_market = EasyMarket()
     print("Total Item:", len(easy_market))
     test_learnware_num = 3
@@ -153,8 +155,27 @@ def test_search():
     user_info = BaseUserInfo(id='user', semantic_spec=user_senmantic, stat_info = dict())
     learnware_list = easy_market.search_learnware(user_info)
     print(learnware_list)
+    
+def test_search():
+    easy_market = EasyMarket()
+    print("Total Item:", len(easy_market))
+    test_learnware_num = 3
+    prepare_learnware(test_learnware_num)
+    root_path = "./learnware_pool"
+    os.makedirs(root_path, exist_ok=True)
+    for i in range(10):
+        user_spec = specification.rkme.RKMEStatSpecification()
+        user_spec.load(f"./learnware_pool/svm{i}/spec.json")
+        user_info = BaseUserInfo(id="user_0", semantic_spec={"desc": "test_user_number_0"}, stat_info={"RKME": user_spec})
+        sorted_dist_list, single_learnware_list, mixture_learnware_list = easy_market.search_learnware(user_info)
+
+        print(f"search result of user{i}:")
+        for dist, learnware in zip(sorted_dist_list, single_learnware_list):
+            print(f"dist: {dist}, learnware_id: {learnware.id}, learnware_name: {learnware.name}")
+        mixture_id = " ".join([learnware.id for learnware in mixture_learnware_list])
+        print(f"mixture_learnware: {mixture_id}\n")
 
 
 if __name__ == "__main__":
-    # test_market()
+    test_market()
     test_search()

learnware/config.py

@@ -50,6 +50,8 @@ class Config:
 
 ROOT_DIRPATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 SPEC_DIRPATH = None
+LEARNWARE_POOL_PATH = os.path.join(ROOT_DIRPATH, "learnware_pool")
+os.makedirs(LEARNWARE_POOL_PATH, exist_ok=True)
 
 semantic_config = {
     "Data": {
@@ -102,6 +104,7 @@ _DEFAULT_CONFIG = {
     "logging_level": logging.INFO,
     "specification_path": SPEC_DIRPATH,
     "semantic_specs": semantic_config,
+    "model_pool_path": LEARNWARE_POOL_PATH,
 }
 
 C = Config(_DEFAULT_CONFIG)

learnware/learnware/base.py

@@ -29,7 +29,7 @@ class Learnware:
         Raises
         ------
         TypeError
-            The type of model must be dict or BaseModel, else raise error
+            The type of model must be str or BaseModel, else raise error
         """
         if isinstance(model, BaseModel):
             return model
@@ -42,7 +42,7 @@ class Learnware:
             model_module = get_module_by_module_path(model_dict["module_path"])
             return getattr(model_module, model_dict["class_name"])()
         else:
-            raise TypeError("model must be BaseModel or dict")
+            raise TypeError("model must be BaseModel or str")
 
     def predict(self, X: np.ndarray) -> np.ndarray:
         return self.model.predict(X)

learnware/market/database_ops.py

@@ -36,6 +36,16 @@ def init_empty_db(func):
 
     return wrapper
 
+# Clear Learnware Database
+# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+# !!!!!                                    !!!!!
+# !!!!! Do NOT use unless highly necessary !!!!!
+# !!!!!                                    !!!!!
+# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+@init_empty_db
+def clear_learnware_table(cur):
+    LOGGER.warning("!!! Drop Learnware Table !!!")
+    cur.execute("DROP TABLE LEARNWARE")
 
 @init_empty_db
 def add_learnware_to_db(id: str, name: str, model_path: str, stat_spec_path: str, semantic_spec: dict, cur):

learnware/market/easy.py

@@ -153,7 +153,7 @@ class EasyMarket(BaseMarket):
         # else:
         weight = torch.linalg.inv(K + torch.eye(K.shape[0]).to(user_rkme.device) * 1e-5) @ C
 
-        term1 = user_rkme.eval_Phi(user_rkme)
+        term1 = user_rkme.inner_prod(user_rkme)
         term2 = weight.T @ C
         term3 = weight.T @ K @ weight
         score = float(term1 - 2 * term2 + term3)
@@ -274,7 +274,10 @@ class EasyMarket(BaseMarket):
         for RKME in RKME_list:
             mmd_dist = RKME.dist(user_rkme)
             mmd_dist_list.append(mmd_dist)
-        sorted_dist_list, sorted_learnware_list = (list(t) for t in zip(*sorted(zip(mmd_dist_list, learnware_list))))
+
+        sorted_idx_list = sorted(range(len(learnware_list)), key=lambda k: mmd_dist_list[k])
+        sorted_dist_list = [mmd_dist_list[idx] for idx in sorted_idx_list]
+        sorted_learnware_list = [learnware_list[idx] for idx in sorted_idx_list]
 
         return sorted_dist_list, sorted_learnware_list
 
@@ -312,6 +315,7 @@ class EasyMarket(BaseMarket):
                 match_learnwares.append(learnware)
         return match_learnwares
     
+<<<<<<< HEAD
     def search_learnware(self, user_info: BaseUserInfo) -> Tuple[Any, List[Learnware]]:
         learnware_list = [self.learnware_list[key] for key in self.learnware_list]
         learnware_list_tags = self._search_by_semantic_tags(learnware_list, user_info)
@@ -319,6 +323,34 @@ class EasyMarket(BaseMarket):
         print(learnware_list_tags, learnware_list_description)
         learnware_list = list(set(learnware_list_tags + learnware_list_description))
         return learnware_list
+=======
+    def search_learnware(self, user_info: BaseUserInfo, search_num=3) -> Tuple[List[float], List[Learnware], List[Learnware]]:
+        """Search learnwares based on user_info
+
+        Parameters
+        ----------
+        user_info : BaseUserInfo
+            user_info contains semantic_spec and stat_info
+        search_num : int
+            The number of the returned learnwares
+
+        Returns
+        -------
+        Tuple[List[float], List[Learnware], List[float], List[Learnware]]
+            the first is the sorted list of rkme dist
+            the second is the sorted list of Learnware (single) by the rkme dist
+            the third is the list of Learnware (mixture), the size is search_num
+        """
+        learnware_list = self._search_by_semantic_spec(user_info)
+
+        if "RKME" not in user_info.stat_info:
+            return None, learnware_list, None
+        else:
+            user_rkme = user_info.stat_info["RKME"]
+            sorted_dist_list, single_learnware_list = self._search_by_rkme_spec_single(learnware_list, user_rkme)
+            weight_list, mixture_learnware_list = self._search_by_rkme_spec_mixture(learnware_list, user_rkme, search_num)
+            return sorted_dist_list, single_learnware_list, mixture_learnware_list
+>>>>>>> 2a2f62b2f98ae79caf42c02ed49ba053b3964ae9
 
     def delete_learnware(self, id: str) -> bool:
         if not id in self.learnware_list:
@@ -331,6 +363,12 @@ class EasyMarket(BaseMarket):
     def get_semantic_spec_list(self) -> dict:
         return self.semantic_spec_list
 
+    def get_learnware_by_ids(self, id:str):
+        pass
+
+    def get_learnware_path_by_ids(self, id:str) -> str:
+        pass
+
     def __len__(self):
         return len(self.learnware_list.keys())