From 1584cdaf9599df3bb9560578ddcd187385d4de24 Mon Sep 17 00:00:00 2001 From: xiey Date: Wed, 19 Apr 2023 18:37:20 +0800 Subject: [PATCH] [MNT] change semantic_spec in examples --- examples/example_image/main.py | 38 ++++------------------------- examples/example_m5/main.py | 40 +++++-------------------------- examples/example_pfs/main.py | 38 ++++------------------------- examples/workflow_by_code/main.py | 33 ++++--------------------- learnware/market/easy.py | 12 +++++----- 5 files changed, 26 insertions(+), 135 deletions(-) diff --git a/examples/example_image/main.py b/examples/example_image/main.py index d6d5f7c..048a9e7 100644 --- a/examples/example_image/main.py +++ b/examples/example_image/main.py @@ -38,45 +38,17 @@ os.makedirs(model_save_root, exist_ok=True) semantic_specs = [ { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, - "Device": {"Values": ["GPU"], "Type": "Tag"}, - "Scenario": {"Values": ["Nature"], "Type": "Tag"}, - "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_1", "Type": "String"}, - }, - { - "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, - "Device": {"Values": ["GPU"], "Type": "Tag"}, - "Scenario": {"Values": ["Business", "Nature"], "Type": "Tag"}, - "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_2", "Type": "String"}, - }, - { - "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class"}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_3", "Type": "String"}, - }, + "Name": {"Values": "learnware_1", "Type": "String"}, + } ] user_senmantic = { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class"}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "String"}, @@ -144,7 +116,7 @@ def prepare_market(): new_learnware_path = prepare_learnware( data_path, model_path, init_file_path, yaml_file_path, tmp_dir, "%s_%d" % (dataset, i) ) - semantic_spec = semantic_specs[i % 3] + semantic_spec = semantic_specs[0] semantic_spec["Name"]["Values"] = "learnware_%d" % (i) semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (i) image_market.add_learnware(new_learnware_path, semantic_spec) diff --git a/examples/example_m5/main.py b/examples/example_m5/main.py index 82ff2d3..761582c 100644 --- a/examples/example_m5/main.py +++ b/examples/example_m5/main.py @@ -15,45 +15,17 @@ from m5 import DataLoader semantic_specs = [ { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, - "Device": {"Values": ["GPU"], "Type": "Tag"}, - "Scenario": {"Values": ["Nature"], "Type": "Tag"}, - "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_1", "Type": "String"}, - }, - { - "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, - "Device": {"Values": ["GPU"], "Type": "Tag"}, - "Scenario": {"Values": ["Business", "Nature"], "Type": "Tag"}, - "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_2", "Type": "String"}, - }, - { - "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class"}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_3", "Type": "String"}, - }, + "Name": {"Values": "learnware_1", "Type": "String"}, + } ] user_senmantic = { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class"}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "String"}, @@ -86,7 +58,7 @@ class M5DatasetWorkflow: zip_path_list.append(os.path.join(curr_root, zip_path)) for idx, zip_path in enumerate(zip_path_list): - semantic_spec = semantic_specs[idx % 3] + semantic_spec = semantic_specs[0] semantic_spec["Name"]["Values"] = "learnware_%d" % (idx) semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx) easy_market.add_learnware(zip_path, semantic_spec) @@ -101,7 +73,7 @@ class M5DatasetWorkflow: m5 = DataLoader() idx_list = m5.get_idx_list() - algo_list = ['lgb'] # algo_list = ["ridge", "lgb"] + algo_list = ["lgb"] # algo_list = ["ridge", "lgb"] curr_root = os.path.dirname(os.path.abspath(__file__)) curr_root = os.path.join(curr_root, "learnware_pool") diff --git a/examples/example_pfs/main.py b/examples/example_pfs/main.py index fd943c9..580ce8d 100644 --- a/examples/example_pfs/main.py +++ b/examples/example_pfs/main.py @@ -15,45 +15,17 @@ from pfs import Dataloader semantic_specs = [ { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, - "Device": {"Values": ["GPU"], "Type": "Tag"}, - "Scenario": {"Values": ["Nature"], "Type": "Tag"}, - "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_1", "Type": "String"}, - }, - { - "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, - "Device": {"Values": ["GPU"], "Type": "Tag"}, - "Scenario": {"Values": ["Business", "Nature"], "Type": "Tag"}, - "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_2", "Type": "String"}, - }, - { - "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class"}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_3", "Type": "String"}, - }, + "Name": {"Values": "learnware_1", "Type": "String"}, + } ] user_senmantic = { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class"}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "String"}, @@ -86,7 +58,7 @@ class PFSDatasetWorkflow: zip_path_list.append(os.path.join(curr_root, zip_path)) for idx, zip_path in enumerate(zip_path_list): - semantic_spec = semantic_specs[idx % 3] + semantic_spec = semantic_specs[0] semantic_spec["Name"]["Values"] = "learnware_%d" % (idx) semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx) easy_market.add_learnware(zip_path, semantic_spec) diff --git a/examples/workflow_by_code/main.py b/examples/workflow_by_code/main.py index 2b72e27..d2baedc 100644 --- a/examples/workflow_by_code/main.py +++ b/examples/workflow_by_code/main.py @@ -18,37 +18,12 @@ curr_root = os.path.dirname(os.path.abspath(__file__)) semantic_specs = [ { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, - "Device": {"Values": ["GPU"], "Type": "Tag"}, - "Scenario": {"Values": ["Nature"], "Type": "Tag"}, - "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_1", "Type": "String"}, - }, - { - "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, - "Device": {"Values": ["GPU"], "Type": "Tag"}, - "Scenario": {"Values": ["Business", "Nature"], "Type": "Tag"}, - "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_2", "Type": "String"}, - }, - { - "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class"}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "String"}, - "Name": {"Values": "learnware_3", "Type": "String"}, - }, + "Name": {"Values": "learnware_1", "Type": "String"}, + } ] user_senmantic = { @@ -118,7 +93,7 @@ class LearnwareMarketWorkflow: print("Total Item:", len(easy_market)) for idx, zip_path in enumerate(self.zip_path_list): - semantic_spec = semantic_specs[idx % 3] + semantic_spec = semantic_specs[0] semantic_spec["Name"]["Values"] = "learnware_%d" % (idx) semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx) easy_market.add_learnware(zip_path, semantic_spec) diff --git a/learnware/market/easy.py b/learnware/market/easy.py index 130d918..0133fa1 100644 --- a/learnware/market/easy.py +++ b/learnware/market/easy.py @@ -333,7 +333,7 @@ class EasyMarket(BaseMarket): learnware_list: List[Learnware], user_rkme: RKMEStatSpecification, max_search_num: int, - weight_cutoff: float = 0.95 + weight_cutoff: float = 0.95, ) -> Tuple[List[float], List[Learnware]]: """Select learnwares based on a total mixture ratio, then recalculate their mixture weights @@ -372,15 +372,15 @@ class EasyMarket(BaseMarket): mixture_list.append(learnware_list[idx]) else: break - + if len(mixture_list) <= 1: mixture_list = [learnware_list[sort_by_weight_idx_list[0]]] mixture_weight = [1] else: if len(mixture_list) > max_search_num: - mixture_list = mixture_list[:max_search_num] + mixture_list = mixture_list[:max_search_num] mixture_weight, _ = self._calculate_rkme_spec_mixture_weight(mixture_list, user_rkme) - + return mixture_weight, mixture_list def _filter_by_rkme_spec_single( @@ -618,11 +618,11 @@ class EasyMarket(BaseMarket): sorted_score_list, single_learnware_list = self._filter_by_rkme_spec_single( sorted_score_list, single_learnware_list ) - if search_method == 'auto': + if search_method == "auto": weight_list, mixture_learnware_list = self._search_by_rkme_spec_mixture_auto( learnware_list, user_rkme, max_search_num ) - elif search_method == 'greedy': + elif search_method == "greedy": weight_list, mixture_learnware_list = self._search_by_rkme_spec_mixture_greedy( learnware_list, user_rkme, max_search_num )