diff --git a/examples/example_image/main.py b/examples/example_image/main.py index d6d5f7c..9df532a 100644 --- a/examples/example_image/main.py +++ b/examples/example_image/main.py @@ -22,7 +22,7 @@ tmp_dir = "./data/tmp" learnware_pool_dir = "./data/learnware_pool" dataset = "cifar10" n_uploaders = 50 -n_users = 10 +n_users = 20 n_classes = 10 data_root = os.path.join(origin_data_root, dataset) data_save_root = os.path.join(processed_data_root, dataset) @@ -38,45 +38,17 @@ os.makedirs(model_save_root, exist_ok=True) semantic_specs = [ { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, - "Device": {"Values": ["GPU"], "Type": "Tag"}, - "Scenario": {"Values": ["Nature"], "Type": "Tag"}, - "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_1", "Type": "String"}, - }, - { - "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, - "Device": {"Values": ["GPU"], "Type": "Tag"}, - "Scenario": {"Values": ["Business", "Nature"], "Type": "Tag"}, - "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_2", "Type": "String"}, - }, - { - "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class"}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_3", "Type": "String"}, - }, + "Name": {"Values": "learnware_1", "Type": "String"}, + } ] user_senmantic = { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class"}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "String"}, @@ -144,14 +116,14 @@ def prepare_market(): new_learnware_path = prepare_learnware( data_path, model_path, init_file_path, yaml_file_path, tmp_dir, "%s_%d" % (dataset, i) ) - semantic_spec = semantic_specs[i % 3] + semantic_spec = semantic_specs[0] semantic_spec["Name"]["Values"] = "learnware_%d" % (i) semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (i) image_market.add_learnware(new_learnware_path, semantic_spec) - logger.info("Total Item:", len(image_market)) + logger.info("Total Item: %d" % (len(image_market))) curr_inds = image_market._get_ids() - logger.info("Available ids:", curr_inds) + logger.info("Available ids: " + str(curr_inds)) def test_search(load_market=True): @@ -162,6 +134,9 @@ def test_search(load_market=True): image_market = EasyMarket() logger.info("Number of items in the market: %d" % len(image_market)) + select_list = [] + avg_list = [] + improve_list = [] for i in range(n_users): user_data_path = os.path.join(user_save_root, "user_%d_X.npy" % (i)) user_label_path = os.path.join(user_save_root, "user_%d_y.npy" % (i)) @@ -174,15 +149,25 @@ def test_search(load_market=True): logger.info("Searching Market for user: %d" % (i)) sorted_score_list, single_learnware_list, mixture_learnware_list = image_market.search_learnware(user_info) l = len(sorted_score_list) - for idx in range(min(l, 10)): + acc_list = [] + for idx in range(l): learnware = single_learnware_list[idx] score = sorted_score_list[idx] pred_y = learnware.predict(user_data) acc = eval_prediction(pred_y, user_label) + acc_list.append(acc) logger.info("search rank: %d, score: %.3f, learnware_id: %s, acc: %.3f" % (idx, score, learnware.id, acc)) + select_list.append(acc_list[0]) + avg_list.append(np.mean(acc_list)) + improve_list.append((acc_list[0] - np.mean(acc_list)) / np.mean(acc_list)) + logger.info( + "Accuracy of selected learnware: %.3f, Average performance: %.3f" % (np.mean(select_list), np.mean(avg_list)) + ) + logger.info("Average performance improvement: %.3f" % (np.mean(improve_list))) + if __name__ == "__main__": # prepare_data() # prepare_model() - test_search(False) + test_search() diff --git a/examples/example_m5/main.py b/examples/example_m5/main.py index 1d3a39b..761582c 100644 --- a/examples/example_m5/main.py +++ b/examples/example_m5/main.py @@ -15,45 +15,17 @@ from m5 import DataLoader semantic_specs = [ { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, - "Device": {"Values": ["GPU"], "Type": "Tag"}, - "Scenario": {"Values": ["Nature"], "Type": "Tag"}, - "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_1", "Type": "String"}, - }, - { - "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, - "Device": {"Values": ["GPU"], "Type": "Tag"}, - "Scenario": {"Values": ["Business", "Nature"], "Type": "Tag"}, - "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_2", "Type": "String"}, - }, - { - "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class"}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_3", "Type": "String"}, - }, + "Name": {"Values": "learnware_1", "Type": "String"}, + } ] user_senmantic = { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class"}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "String"}, @@ -86,7 +58,7 @@ class M5DatasetWorkflow: zip_path_list.append(os.path.join(curr_root, zip_path)) for idx, zip_path in enumerate(zip_path_list): - semantic_spec = semantic_specs[idx % 3] + semantic_spec = semantic_specs[0] semantic_spec["Name"]["Values"] = "learnware_%d" % (idx) semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx) easy_market.add_learnware(zip_path, semantic_spec) diff --git a/examples/example_pfs/main.py b/examples/example_pfs/main.py index c6708db..f9b7bc1 100644 --- a/examples/example_pfs/main.py +++ b/examples/example_pfs/main.py @@ -15,45 +15,17 @@ from pfs import Dataloader semantic_specs = [ { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, - "Device": {"Values": ["GPU"], "Type": "Tag"}, - "Scenario": {"Values": ["Nature"], "Type": "Tag"}, - "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_1", "Type": "String"}, - }, - { - "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, - "Device": {"Values": ["GPU"], "Type": "Tag"}, - "Scenario": {"Values": ["Business", "Nature"], "Type": "Tag"}, - "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_2", "Type": "String"}, - }, - { - "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class"}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_3", "Type": "String"}, - }, + "Name": {"Values": "learnware_1", "Type": "String"}, + } ] user_senmantic = { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class"}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "String"}, @@ -86,7 +58,7 @@ class PFSDatasetWorkflow: zip_path_list.append(os.path.join(curr_root, zip_path)) for idx, zip_path in enumerate(zip_path_list): - semantic_spec = semantic_specs[idx % 3] + semantic_spec = semantic_specs[0] semantic_spec["Name"]["Values"] = "learnware_%d" % (idx) semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx) easy_market.add_learnware(zip_path, semantic_spec) diff --git a/examples/workflow_by_code/main.py b/examples/workflow_by_code/main.py index 2b72e27..d2baedc 100644 --- a/examples/workflow_by_code/main.py +++ b/examples/workflow_by_code/main.py @@ -18,37 +18,12 @@ curr_root = os.path.dirname(os.path.abspath(__file__)) semantic_specs = [ { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, - "Device": {"Values": ["GPU"], "Type": "Tag"}, - "Scenario": {"Values": ["Nature"], "Type": "Tag"}, - "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_1", "Type": "String"}, - }, - { - "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, - "Device": {"Values": ["GPU"], "Type": "Tag"}, - "Scenario": {"Values": ["Business", "Nature"], "Type": "Tag"}, - "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_2", "Type": "String"}, - }, - { - "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class"}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_3", "Type": "String"}, - }, + "Name": {"Values": "learnware_1", "Type": "String"}, + } ] user_senmantic = { @@ -118,7 +93,7 @@ class LearnwareMarketWorkflow: print("Total Item:", len(easy_market)) for idx, zip_path in enumerate(self.zip_path_list): - semantic_spec = semantic_specs[idx % 3] + semantic_spec = semantic_specs[0] semantic_spec["Name"]["Values"] = "learnware_%d" % (idx) semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx) easy_market.add_learnware(zip_path, semantic_spec) diff --git a/learnware/market/easy.py b/learnware/market/easy.py index 0133fa1..0aeaa3c 100644 --- a/learnware/market/easy.py +++ b/learnware/market/easy.py @@ -73,6 +73,7 @@ class EasyMarket(BaseMarket): learnware.instantiate_model() except Exception as e: logger.warning(f"The learnware [{learnware.id}] is instantiated failed! Due to {repr(e)}") + raise return cls.INVALID_LEARNWARE try: @@ -333,7 +334,7 @@ class EasyMarket(BaseMarket): learnware_list: List[Learnware], user_rkme: RKMEStatSpecification, max_search_num: int, - weight_cutoff: float = 0.95, + weight_cutoff: float = 0.98, ) -> Tuple[List[float], List[Learnware]]: """Select learnwares based on a total mixture ratio, then recalculate their mixture weights @@ -449,7 +450,7 @@ class EasyMarket(BaseMarket): learnware_list: List[Learnware], user_rkme: RKMEStatSpecification, max_search_num: int, - score_cutoff: float = 0.01, + score_cutoff: float = 0.001, ) -> Tuple[List[float], List[Learnware]]: """Greedily match learnwares such that their mixture become more and more closer to user's rkme @@ -581,6 +582,7 @@ class EasyMarket(BaseMarket): user_semantic_spec = user_info.get_semantic_spec() if match_semantic_spec(learnware_semantic_spec, user_semantic_spec): match_learnwares.append(learnware) + logger.info("semantic_spec search: choose %d from %d learnwares" % (len(match_learnwares), len(learnware_list))) return match_learnwares def search_learnware(