diff --git a/examples/example_market_db/example_db.py b/examples/example_market_db/example_db.py index 61557c8..7e30ad7 100644 --- a/examples/example_market_db/example_db.py +++ b/examples/example_market_db/example_db.py @@ -16,7 +16,8 @@ def prepare_learnware(learnware_num=10): os.makedirs(dir_path, exist_ok=True) print("Preparing Learnware: %d" % (i)) - data_X = np.random.randn(5000, 20) + data_X = np.random.randn(5000, 20) * i + # print(data_X[:10]) data_y = np.random.randn(5000) data_y = np.where(data_y > 0, 1, 0) @@ -32,6 +33,7 @@ def prepare_learnware(learnware_num=10): def test_market(): + database_ops.clear_learnware_table() easy_market = EasyMarket() print("Total Item:", len(easy_market)) test_learnware_num = 10 @@ -56,7 +58,7 @@ def test_market(): print("Available ids:", curr_inds) -def test_search(): +def test_search_sementics(): easy_market = EasyMarket() print("Total Item:", len(easy_market)) test_learnware_num = 3 @@ -153,8 +155,27 @@ def test_search(): user_info = BaseUserInfo(id='user', semantic_spec=user_senmantic, stat_info = dict()) learnware_list = easy_market.search_learnware(user_info) print(learnware_list) + +def test_search(): + easy_market = EasyMarket() + print("Total Item:", len(easy_market)) + test_learnware_num = 3 + prepare_learnware(test_learnware_num) + root_path = "./learnware_pool" + os.makedirs(root_path, exist_ok=True) + for i in range(10): + user_spec = specification.rkme.RKMEStatSpecification() + user_spec.load(f"./learnware_pool/svm{i}/spec.json") + user_info = BaseUserInfo(id="user_0", semantic_spec={"desc": "test_user_number_0"}, stat_info={"RKME": user_spec}) + sorted_dist_list, single_learnware_list, mixture_learnware_list = easy_market.search_learnware(user_info) + + print(f"search result of user{i}:") + for dist, learnware in zip(sorted_dist_list, single_learnware_list): + print(f"dist: {dist}, learnware_id: {learnware.id}, learnware_name: {learnware.name}") + mixture_id = " ".join([learnware.id for learnware in mixture_learnware_list]) + print(f"mixture_learnware: {mixture_id}\n") if __name__ == "__main__": - # test_market() + test_market() test_search() diff --git a/learnware/config.py b/learnware/config.py index 84f99b1..9b6874a 100644 --- a/learnware/config.py +++ b/learnware/config.py @@ -50,6 +50,8 @@ class Config: ROOT_DIRPATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) SPEC_DIRPATH = None +LEARNWARE_POOL_PATH = os.path.join(ROOT_DIRPATH, "learnware_pool") +os.makedirs(LEARNWARE_POOL_PATH, exist_ok=True) semantic_config = { "Data": { @@ -102,6 +104,7 @@ _DEFAULT_CONFIG = { "logging_level": logging.INFO, "specification_path": SPEC_DIRPATH, "semantic_specs": semantic_config, + "model_pool_path": LEARNWARE_POOL_PATH, } C = Config(_DEFAULT_CONFIG) diff --git a/learnware/learnware/base.py b/learnware/learnware/base.py index e8f9f35..63d8136 100644 --- a/learnware/learnware/base.py +++ b/learnware/learnware/base.py @@ -29,7 +29,7 @@ class Learnware: Raises ------ TypeError - The type of model must be dict or BaseModel, else raise error + The type of model must be str or BaseModel, else raise error """ if isinstance(model, BaseModel): return model @@ -42,7 +42,7 @@ class Learnware: model_module = get_module_by_module_path(model_dict["module_path"]) return getattr(model_module, model_dict["class_name"])() else: - raise TypeError("model must be BaseModel or dict") + raise TypeError("model must be BaseModel or str") def predict(self, X: np.ndarray) -> np.ndarray: return self.model.predict(X) diff --git a/learnware/market/database_ops.py b/learnware/market/database_ops.py index 56c5298..03009af 100644 --- a/learnware/market/database_ops.py +++ b/learnware/market/database_ops.py @@ -36,6 +36,16 @@ def init_empty_db(func): return wrapper +# Clear Learnware Database +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# !!!!! !!!!! +# !!!!! Do NOT use unless highly necessary !!!!! +# !!!!! !!!!! +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +@init_empty_db +def clear_learnware_table(cur): + LOGGER.warning("!!! Drop Learnware Table !!!") + cur.execute("DROP TABLE LEARNWARE") @init_empty_db def add_learnware_to_db(id: str, name: str, model_path: str, stat_spec_path: str, semantic_spec: dict, cur): diff --git a/learnware/market/easy.py b/learnware/market/easy.py index 2b36f06..3dd5697 100644 --- a/learnware/market/easy.py +++ b/learnware/market/easy.py @@ -153,7 +153,7 @@ class EasyMarket(BaseMarket): # else: weight = torch.linalg.inv(K + torch.eye(K.shape[0]).to(user_rkme.device) * 1e-5) @ C - term1 = user_rkme.eval_Phi(user_rkme) + term1 = user_rkme.inner_prod(user_rkme) term2 = weight.T @ C term3 = weight.T @ K @ weight score = float(term1 - 2 * term2 + term3) @@ -274,7 +274,10 @@ class EasyMarket(BaseMarket): for RKME in RKME_list: mmd_dist = RKME.dist(user_rkme) mmd_dist_list.append(mmd_dist) - sorted_dist_list, sorted_learnware_list = (list(t) for t in zip(*sorted(zip(mmd_dist_list, learnware_list)))) + + sorted_idx_list = sorted(range(len(learnware_list)), key=lambda k: mmd_dist_list[k]) + sorted_dist_list = [mmd_dist_list[idx] for idx in sorted_idx_list] + sorted_learnware_list = [learnware_list[idx] for idx in sorted_idx_list] return sorted_dist_list, sorted_learnware_list @@ -312,6 +315,7 @@ class EasyMarket(BaseMarket): match_learnwares.append(learnware) return match_learnwares +<<<<<<< HEAD def search_learnware(self, user_info: BaseUserInfo) -> Tuple[Any, List[Learnware]]: learnware_list = [self.learnware_list[key] for key in self.learnware_list] learnware_list_tags = self._search_by_semantic_tags(learnware_list, user_info) @@ -319,6 +323,34 @@ class EasyMarket(BaseMarket): print(learnware_list_tags, learnware_list_description) learnware_list = list(set(learnware_list_tags + learnware_list_description)) return learnware_list +======= + def search_learnware(self, user_info: BaseUserInfo, search_num=3) -> Tuple[List[float], List[Learnware], List[Learnware]]: + """Search learnwares based on user_info + + Parameters + ---------- + user_info : BaseUserInfo + user_info contains semantic_spec and stat_info + search_num : int + The number of the returned learnwares + + Returns + ------- + Tuple[List[float], List[Learnware], List[float], List[Learnware]] + the first is the sorted list of rkme dist + the second is the sorted list of Learnware (single) by the rkme dist + the third is the list of Learnware (mixture), the size is search_num + """ + learnware_list = self._search_by_semantic_spec(user_info) + + if "RKME" not in user_info.stat_info: + return None, learnware_list, None + else: + user_rkme = user_info.stat_info["RKME"] + sorted_dist_list, single_learnware_list = self._search_by_rkme_spec_single(learnware_list, user_rkme) + weight_list, mixture_learnware_list = self._search_by_rkme_spec_mixture(learnware_list, user_rkme, search_num) + return sorted_dist_list, single_learnware_list, mixture_learnware_list +>>>>>>> 2a2f62b2f98ae79caf42c02ed49ba053b3964ae9 def delete_learnware(self, id: str) -> bool: if not id in self.learnware_list: @@ -331,6 +363,12 @@ class EasyMarket(BaseMarket): def get_semantic_spec_list(self) -> dict: return self.semantic_spec_list + def get_learnware_by_ids(self, id:str): + pass + + def get_learnware_path_by_ids(self, id:str) -> str: + pass + def __len__(self): return len(self.learnware_list.keys())