From 0f827172cda2a47e4da5c3d9833ce18f04e2604a Mon Sep 17 00:00:00 2001 From: chenzx Date: Wed, 19 Apr 2023 18:25:52 +0800 Subject: [PATCH 1/6] [MNT] Update Image example --- examples/example_image/main.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/examples/example_image/main.py b/examples/example_image/main.py index d6d5f7c..2c68dbb 100644 --- a/examples/example_image/main.py +++ b/examples/example_image/main.py @@ -22,7 +22,7 @@ tmp_dir = "./data/tmp" learnware_pool_dir = "./data/learnware_pool" dataset = "cifar10" n_uploaders = 50 -n_users = 10 +n_users = 20 n_classes = 10 data_root = os.path.join(origin_data_root, dataset) data_save_root = os.path.join(processed_data_root, dataset) @@ -149,9 +149,9 @@ def prepare_market(): semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (i) image_market.add_learnware(new_learnware_path, semantic_spec) - logger.info("Total Item:", len(image_market)) + logger.info("Total Item: %d" % (len(image_market))) curr_inds = image_market._get_ids() - logger.info("Available ids:", curr_inds) + logger.info("Available ids: " + str(curr_inds)) def test_search(load_market=True): @@ -162,6 +162,9 @@ def test_search(load_market=True): image_market = EasyMarket() logger.info("Number of items in the market: %d" % len(image_market)) + select_list = [] + avg_list = [] + improve_list = [] for i in range(n_users): user_data_path = os.path.join(user_save_root, "user_%d_X.npy" % (i)) user_label_path = os.path.join(user_save_root, "user_%d_y.npy" % (i)) @@ -174,15 +177,25 @@ def test_search(load_market=True): logger.info("Searching Market for user: %d" % (i)) sorted_score_list, single_learnware_list, mixture_learnware_list = image_market.search_learnware(user_info) l = len(sorted_score_list) - for idx in range(min(l, 10)): + acc_list = [] + for idx in range(l): learnware = single_learnware_list[idx] score = sorted_score_list[idx] pred_y = learnware.predict(user_data) acc = eval_prediction(pred_y, user_label) + acc_list.append(acc) logger.info("search rank: %d, score: %.3f, learnware_id: %s, acc: %.3f" % (idx, score, learnware.id, acc)) + select_list.append(acc_list[0]) + avg_list.append(np.mean(acc_list)) + improve_list.append((acc_list[0] - np.mean(acc_list)) / np.mean(acc_list)) + logger.info( + "Accuracy of selected learnware: %.3f, Average performance: %.3f" % (np.mean(select_list), np.mean(avg_list)) + ) + logger.info("Average performance improvement: %.3f" % (np.mean(improve_list))) + if __name__ == "__main__": # prepare_data() # prepare_model() - test_search(False) + test_search() From 1584cdaf9599df3bb9560578ddcd187385d4de24 Mon Sep 17 00:00:00 2001 From: xiey Date: Wed, 19 Apr 2023 18:37:20 +0800 Subject: [PATCH 2/6] [MNT] change semantic_spec in examples --- examples/example_image/main.py | 38 ++++------------------------- examples/example_m5/main.py | 40 +++++-------------------------- examples/example_pfs/main.py | 38 ++++------------------------- examples/workflow_by_code/main.py | 33 ++++--------------------- learnware/market/easy.py | 12 +++++----- 5 files changed, 26 insertions(+), 135 deletions(-) diff --git a/examples/example_image/main.py b/examples/example_image/main.py index d6d5f7c..048a9e7 100644 --- a/examples/example_image/main.py +++ b/examples/example_image/main.py @@ -38,45 +38,17 @@ os.makedirs(model_save_root, exist_ok=True) semantic_specs = [ { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, - "Device": {"Values": ["GPU"], "Type": "Tag"}, - "Scenario": {"Values": ["Nature"], "Type": "Tag"}, - "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_1", "Type": "String"}, - }, - { - "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, - "Device": {"Values": ["GPU"], "Type": "Tag"}, - "Scenario": {"Values": ["Business", "Nature"], "Type": "Tag"}, - "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_2", "Type": "String"}, - }, - { - "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class"}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_3", "Type": "String"}, - }, + "Name": {"Values": "learnware_1", "Type": "String"}, + } ] user_senmantic = { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class"}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "String"}, @@ -144,7 +116,7 @@ def prepare_market(): new_learnware_path = prepare_learnware( data_path, model_path, init_file_path, yaml_file_path, tmp_dir, "%s_%d" % (dataset, i) ) - semantic_spec = semantic_specs[i % 3] + semantic_spec = semantic_specs[0] semantic_spec["Name"]["Values"] = "learnware_%d" % (i) semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (i) image_market.add_learnware(new_learnware_path, semantic_spec) diff --git a/examples/example_m5/main.py b/examples/example_m5/main.py index 82ff2d3..761582c 100644 --- a/examples/example_m5/main.py +++ b/examples/example_m5/main.py @@ -15,45 +15,17 @@ from m5 import DataLoader semantic_specs = [ { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, - "Device": {"Values": ["GPU"], "Type": "Tag"}, - "Scenario": {"Values": ["Nature"], "Type": "Tag"}, - "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_1", "Type": "String"}, - }, - { - "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, - "Device": {"Values": ["GPU"], "Type": "Tag"}, - "Scenario": {"Values": ["Business", "Nature"], "Type": "Tag"}, - "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_2", "Type": "String"}, - }, - { - "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class"}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_3", "Type": "String"}, - }, + "Name": {"Values": "learnware_1", "Type": "String"}, + } ] user_senmantic = { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class"}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "String"}, @@ -86,7 +58,7 @@ class M5DatasetWorkflow: zip_path_list.append(os.path.join(curr_root, zip_path)) for idx, zip_path in enumerate(zip_path_list): - semantic_spec = semantic_specs[idx % 3] + semantic_spec = semantic_specs[0] semantic_spec["Name"]["Values"] = "learnware_%d" % (idx) semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx) easy_market.add_learnware(zip_path, semantic_spec) @@ -101,7 +73,7 @@ class M5DatasetWorkflow: m5 = DataLoader() idx_list = m5.get_idx_list() - algo_list = ['lgb'] # algo_list = ["ridge", "lgb"] + algo_list = ["lgb"] # algo_list = ["ridge", "lgb"] curr_root = os.path.dirname(os.path.abspath(__file__)) curr_root = os.path.join(curr_root, "learnware_pool") diff --git a/examples/example_pfs/main.py b/examples/example_pfs/main.py index fd943c9..580ce8d 100644 --- a/examples/example_pfs/main.py +++ b/examples/example_pfs/main.py @@ -15,45 +15,17 @@ from pfs import Dataloader semantic_specs = [ { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, - "Device": {"Values": ["GPU"], "Type": "Tag"}, - "Scenario": {"Values": ["Nature"], "Type": "Tag"}, - "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_1", "Type": "String"}, - }, - { - "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, - "Device": {"Values": ["GPU"], "Type": "Tag"}, - "Scenario": {"Values": ["Business", "Nature"], "Type": "Tag"}, - "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_2", "Type": "String"}, - }, - { - "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class"}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_3", "Type": "String"}, - }, + "Name": {"Values": "learnware_1", "Type": "String"}, + } ] user_senmantic = { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class"}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "String"}, @@ -86,7 +58,7 @@ class PFSDatasetWorkflow: zip_path_list.append(os.path.join(curr_root, zip_path)) for idx, zip_path in enumerate(zip_path_list): - semantic_spec = semantic_specs[idx % 3] + semantic_spec = semantic_specs[0] semantic_spec["Name"]["Values"] = "learnware_%d" % (idx) semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx) easy_market.add_learnware(zip_path, semantic_spec) diff --git a/examples/workflow_by_code/main.py b/examples/workflow_by_code/main.py index 2b72e27..d2baedc 100644 --- a/examples/workflow_by_code/main.py +++ b/examples/workflow_by_code/main.py @@ -18,37 +18,12 @@ curr_root = os.path.dirname(os.path.abspath(__file__)) semantic_specs = [ { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, - "Device": {"Values": ["GPU"], "Type": "Tag"}, - "Scenario": {"Values": ["Nature"], "Type": "Tag"}, - "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_1", "Type": "String"}, - }, - { - "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, - "Device": {"Values": ["GPU"], "Type": "Tag"}, - "Scenario": {"Values": ["Business", "Nature"], "Type": "Tag"}, - "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_2", "Type": "String"}, - }, - { - "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class"}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_3", "Type": "String"}, - }, + "Name": {"Values": "learnware_1", "Type": "String"}, + } ] user_senmantic = { @@ -118,7 +93,7 @@ class LearnwareMarketWorkflow: print("Total Item:", len(easy_market)) for idx, zip_path in enumerate(self.zip_path_list): - semantic_spec = semantic_specs[idx % 3] + semantic_spec = semantic_specs[0] semantic_spec["Name"]["Values"] = "learnware_%d" % (idx) semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx) easy_market.add_learnware(zip_path, semantic_spec) diff --git a/learnware/market/easy.py b/learnware/market/easy.py index 130d918..0133fa1 100644 --- a/learnware/market/easy.py +++ b/learnware/market/easy.py @@ -333,7 +333,7 @@ class EasyMarket(BaseMarket): learnware_list: List[Learnware], user_rkme: RKMEStatSpecification, max_search_num: int, - weight_cutoff: float = 0.95 + weight_cutoff: float = 0.95, ) -> Tuple[List[float], List[Learnware]]: """Select learnwares based on a total mixture ratio, then recalculate their mixture weights @@ -372,15 +372,15 @@ class EasyMarket(BaseMarket): mixture_list.append(learnware_list[idx]) else: break - + if len(mixture_list) <= 1: mixture_list = [learnware_list[sort_by_weight_idx_list[0]]] mixture_weight = [1] else: if len(mixture_list) > max_search_num: - mixture_list = mixture_list[:max_search_num] + mixture_list = mixture_list[:max_search_num] mixture_weight, _ = self._calculate_rkme_spec_mixture_weight(mixture_list, user_rkme) - + return mixture_weight, mixture_list def _filter_by_rkme_spec_single( @@ -618,11 +618,11 @@ class EasyMarket(BaseMarket): sorted_score_list, single_learnware_list = self._filter_by_rkme_spec_single( sorted_score_list, single_learnware_list ) - if search_method == 'auto': + if search_method == "auto": weight_list, mixture_learnware_list = self._search_by_rkme_spec_mixture_auto( learnware_list, user_rkme, max_search_num ) - elif search_method == 'greedy': + elif search_method == "greedy": weight_list, mixture_learnware_list = self._search_by_rkme_spec_mixture_greedy( learnware_list, user_rkme, max_search_num ) From 445930c4021a6c7333a6b9cf394ee2753a4ec93f Mon Sep 17 00:00:00 2001 From: xiey Date: Wed, 19 Apr 2023 18:52:36 +0800 Subject: [PATCH 3/6] [MNT] Add a log info in semantic search --- learnware/market/easy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/learnware/market/easy.py b/learnware/market/easy.py index 0133fa1..1e4d291 100644 --- a/learnware/market/easy.py +++ b/learnware/market/easy.py @@ -581,6 +581,7 @@ class EasyMarket(BaseMarket): user_semantic_spec = user_info.get_semantic_spec() if match_semantic_spec(learnware_semantic_spec, user_semantic_spec): match_learnwares.append(learnware) + logger.info("semantic_spec search: choose %d from %d learnwares" % (len(match_learnwares), len(learnware_list))) return match_learnwares def search_learnware( From 180747757284a7ae91ff9d827bbea5e0c24bd848 Mon Sep 17 00:00:00 2001 From: liuht Date: Wed, 19 Apr 2023 19:06:59 +0800 Subject: [PATCH 4/6] [] --- learnware/market/easy.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/learnware/market/easy.py b/learnware/market/easy.py index 0133fa1..19e2c6f 100644 --- a/learnware/market/easy.py +++ b/learnware/market/easy.py @@ -23,7 +23,7 @@ class EasyMarket(BaseMarket): NOPREDICTION_LEARNWARE = 0 PREDICTION_LEARWARE = 1 - def __init__(self, market_id: str = None, rebuild: bool = False): + def __init__(self, rebuild: bool = False): """Initialize Learnware Market. Automatically reload from db if available. Build an empty db otherwise. @@ -333,7 +333,7 @@ class EasyMarket(BaseMarket): learnware_list: List[Learnware], user_rkme: RKMEStatSpecification, max_search_num: int, - weight_cutoff: float = 0.95, + weight_cutoff: float = 0.98 ) -> Tuple[List[float], List[Learnware]]: """Select learnwares based on a total mixture ratio, then recalculate their mixture weights @@ -372,15 +372,15 @@ class EasyMarket(BaseMarket): mixture_list.append(learnware_list[idx]) else: break - + if len(mixture_list) <= 1: mixture_list = [learnware_list[sort_by_weight_idx_list[0]]] mixture_weight = [1] else: if len(mixture_list) > max_search_num: - mixture_list = mixture_list[:max_search_num] + mixture_list = mixture_list[:max_search_num] mixture_weight, _ = self._calculate_rkme_spec_mixture_weight(mixture_list, user_rkme) - + return mixture_weight, mixture_list def _filter_by_rkme_spec_single( @@ -444,12 +444,12 @@ class EasyMarket(BaseMarket): return filtered_learnware_list - def _search_by_rkme_spec_mixture_greedy( + def _search_by_rkme_spec_mixture( self, learnware_list: List[Learnware], user_rkme: RKMEStatSpecification, max_search_num: int, - score_cutoff: float = 0.01, + score_cutoff: float = 0.001, ) -> Tuple[List[float], List[Learnware]]: """Greedily match learnwares such that their mixture become more and more closer to user's rkme @@ -618,11 +618,11 @@ class EasyMarket(BaseMarket): sorted_score_list, single_learnware_list = self._filter_by_rkme_spec_single( sorted_score_list, single_learnware_list ) - if search_method == "auto": + if search_method == 'auto': weight_list, mixture_learnware_list = self._search_by_rkme_spec_mixture_auto( learnware_list, user_rkme, max_search_num ) - elif search_method == "greedy": + elif search_method == 'greedy': weight_list, mixture_learnware_list = self._search_by_rkme_spec_mixture_greedy( learnware_list, user_rkme, max_search_num ) From eb31bf15b2d70df3871161c395461b2b4ad92728 Mon Sep 17 00:00:00 2001 From: liuht Date: Wed, 19 Apr 2023 19:11:52 +0800 Subject: [PATCH 5/6] [MNT] change parameters --- learnware/market/easy.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/learnware/market/easy.py b/learnware/market/easy.py index fee3af9..0efd5bb 100644 --- a/learnware/market/easy.py +++ b/learnware/market/easy.py @@ -23,7 +23,7 @@ class EasyMarket(BaseMarket): NOPREDICTION_LEARNWARE = 0 PREDICTION_LEARWARE = 1 - def __init__(self, rebuild: bool = False): + def __init__(self, market_id: str = None, rebuild: bool = False): """Initialize Learnware Market. Automatically reload from db if available. Build an empty db otherwise. @@ -333,7 +333,7 @@ class EasyMarket(BaseMarket): learnware_list: List[Learnware], user_rkme: RKMEStatSpecification, max_search_num: int, - weight_cutoff: float = 0.98 + weight_cutoff: float = 0.98, ) -> Tuple[List[float], List[Learnware]]: """Select learnwares based on a total mixture ratio, then recalculate their mixture weights @@ -372,15 +372,15 @@ class EasyMarket(BaseMarket): mixture_list.append(learnware_list[idx]) else: break - + if len(mixture_list) <= 1: mixture_list = [learnware_list[sort_by_weight_idx_list[0]]] mixture_weight = [1] else: if len(mixture_list) > max_search_num: - mixture_list = mixture_list[:max_search_num] + mixture_list = mixture_list[:max_search_num] mixture_weight, _ = self._calculate_rkme_spec_mixture_weight(mixture_list, user_rkme) - + return mixture_weight, mixture_list def _filter_by_rkme_spec_single( @@ -444,7 +444,7 @@ class EasyMarket(BaseMarket): return filtered_learnware_list - def _search_by_rkme_spec_mixture( + def _search_by_rkme_spec_mixture_greedy( self, learnware_list: List[Learnware], user_rkme: RKMEStatSpecification, @@ -619,11 +619,11 @@ class EasyMarket(BaseMarket): sorted_score_list, single_learnware_list = self._filter_by_rkme_spec_single( sorted_score_list, single_learnware_list ) - if search_method == 'auto': + if search_method == "auto": weight_list, mixture_learnware_list = self._search_by_rkme_spec_mixture_auto( learnware_list, user_rkme, max_search_num ) - elif search_method == 'greedy': + elif search_method == "greedy": weight_list, mixture_learnware_list = self._search_by_rkme_spec_mixture_greedy( learnware_list, user_rkme, max_search_num ) From 48c54223203d60ea9a846bc636fbeb818d156a92 Mon Sep 17 00:00:00 2001 From: xiey Date: Wed, 19 Apr 2023 19:15:54 +0800 Subject: [PATCH 6/6] [MNT] add a raise by bixd --- learnware/market/easy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/learnware/market/easy.py b/learnware/market/easy.py index 1e4d291..ddcbb2e 100644 --- a/learnware/market/easy.py +++ b/learnware/market/easy.py @@ -73,6 +73,7 @@ class EasyMarket(BaseMarket): learnware.instantiate_model() except Exception as e: logger.warning(f"The learnware [{learnware.id}] is instantiated failed! Due to {repr(e)}") + raise return cls.INVALID_LEARNWARE try: