diff --git a/examples/dataset_table_workflow/base.py b/examples/dataset_table_workflow/base.py index a10c69a..7e619ef 100644 --- a/examples/dataset_table_workflow/base.py +++ b/examples/dataset_table_workflow/base.py @@ -1,6 +1,5 @@ import os import time -import torch import random import requests import tempfile @@ -12,8 +11,8 @@ from learnware.market import instantiate_learnware_market from learnware.reuse.utils import fill_data_with_mean from learnware.tests.benchmarks import LearnwareBenchmark -from config import * -from methods import * +from config import market_mapping_params +from methods import loss_func_rmse, test_methods from utils import set_seed logger = get_module_logger("base_table", level="INFO") @@ -26,14 +25,14 @@ class TableWorkflow: self.curves_result_path = os.path.join(self.result_path, "curves") os.makedirs(self.result_path, exist_ok=True) os.makedirs(self.curves_result_path, exist_ok=True) - self._prepare_market(benchmark_config, name, rebuild, retrain) - + self._prepare_market(benchmark_config, name, rebuild, retrain) + @staticmethod def _limited_data(method, test_info, loss_func): def subset_generator(): for subset in test_info["train_subsets"]: yield subset - + all_scores = [] for subset in subset_generator(): subset_scores = [] @@ -43,7 +42,7 @@ class TableWorkflow: subset_scores.append(loss_func(model.predict(test_info["test_x"]), test_info["test_y"])) all_scores.append(subset_scores) return all_scores - + @staticmethod def get_train_subsets(n_labeled_list, n_repeat_list, train_x, train_y): np.random.seed(1) @@ -56,9 +55,11 @@ class TableWorkflow: n_label = len(train_x) for _ in range(repeated): subset_idxs = np.random.choice(len(train_x), n_label, replace=False) - train_subsets[-1].append({"x_train": np.array(train_x[subset_idxs]), "y_train": np.array(train_y[subset_idxs])}) + train_subsets[-1].append( + {"x_train": np.array(train_x[subset_idxs]), "y_train": np.array(train_y[subset_idxs])} + ) return train_subsets - + def _prepare_market(self, benchmark_config, name, rebuild, retrain): client = LearnwareClient() self.benchmark = LearnwareBenchmark().get_benchmark(benchmark_config) @@ -69,14 +70,17 @@ class TableWorkflow: organizer_kwargs={ "auto_update": True, "auto_update_limit": len(self.benchmark.learnware_ids), - **market_mapping_params - } if retrain else None + **market_mapping_params, + } + if retrain + else None, ) self.user_semantic = client.get_semantic_specification(self.benchmark.learnware_ids[0]) self.user_semantic["Name"]["Values"] = "" - - if len(self.market) == 0 or rebuild == True: - if retrain: set_seed(0) + + if len(self.market) == 0 or rebuild is True: + if retrain: + set_seed(0) for learnware_id in self.benchmark.learnware_ids: with tempfile.TemporaryDirectory(prefix="table_benchmark_") as tempdir: zip_path = os.path.join(tempdir, f"{learnware_id}.zip") @@ -87,24 +91,28 @@ class TableWorkflow: self.market.add_learnware(zip_path, semantic_spec) break except (requests.exceptions.RequestException, IOError, Exception) as e: - logger.info(f"An error occurred when downloading {learnware_id}: {e}\n{traceback.format_exc()}, retrying...") + logger.info( + f"An error occurred when downloading {learnware_id}: {e}\n{traceback.format_exc()}, retrying..." + ) time.sleep(1) continue - + def test_method(self, test_info, recorders, loss_func=loss_func_rmse): method_name_full = test_info["method_name"] - method_name = method_name_full if method_name_full == "user_model" else "_".join(method_name_full.split("_")[1:]) + method_name = ( + method_name_full if method_name_full == "user_model" else "_".join(method_name_full.split("_")[1:]) + ) method = test_methods[method_name_full] user, idx = test_info["user"], test_info["idx"] recorder = recorders[method_name_full] - + save_root_path = os.path.join(self.curves_result_path, f"{user}/{user}_{idx}") os.makedirs(save_root_path, exist_ok=True) save_path = os.path.join(save_root_path, f"{method_name}.json") - + if recorder.should_test_method(user, idx, save_path): scores = self._limited_data(method, test_info, loss_func) recorder.record(user, scores) recorder.save(save_path) - - logger.info(f"Method {method_name} on {user}_{idx} finished") \ No newline at end of file + + logger.info(f"Method {method_name} on {user}_{idx} finished") diff --git a/examples/dataset_table_workflow/config.py b/examples/dataset_table_workflow/config.py index 7f0d5de..5c27d33 100644 --- a/examples/dataset_table_workflow/config.py +++ b/examples/dataset_table_workflow/config.py @@ -17,23 +17,23 @@ user_semantic = { } styles = { - 'user_model': {"color": "navy", "marker": "o", "linestyle": "-"}, - 'select_score': {'color': 'gold', 'marker': 's', 'linestyle': '--'}, - 'oracle_score': {'color': 'darkorange', 'marker': '^', 'linestyle': '-.'}, - 'mean_score': {'color': 'gray', 'marker': 'x', 'linestyle': ':'}, - 'single_aug': {'color': 'gold', 'marker': 's', 'linestyle': '--'}, - 'multiple_avg': {'color': 'blue', 'marker': '*', 'linestyle': '-'}, - 'multiple_aug': {'color': 'purple', 'marker': 'd', 'linestyle': '--'}, - 'ensemble_pruning': {"color": "magenta", "marker": "d", "linestyle": "-."} + "user_model": {"color": "navy", "marker": "o", "linestyle": "-"}, + "select_score": {"color": "gold", "marker": "s", "linestyle": "--"}, + "oracle_score": {"color": "darkorange", "marker": "^", "linestyle": "-."}, + "mean_score": {"color": "gray", "marker": "x", "linestyle": ":"}, + "single_aug": {"color": "gold", "marker": "s", "linestyle": "--"}, + "multiple_avg": {"color": "blue", "marker": "*", "linestyle": "-"}, + "multiple_aug": {"color": "purple", "marker": "d", "linestyle": "--"}, + "ensemble_pruning": {"color": "magenta", "marker": "d", "linestyle": "-."}, } labels = { - 'user_model': "User Model", - 'single_aug': "Single Learnware Reuse (FeatAug)", + "user_model": "User Model", + "single_aug": "Single Learnware Reuse (FeatAug)", "select_score": "Single Learnware Reuse (FeatAug)", - 'multiple_aug': "Multiple Learnware Reuse (FeatAug)", - 'ensemble_pruning': "Multiple Learnware Reuse (EnsemblePrune)", - 'multiple_avg': "Multiple Learnware Reuse (Averaging)" + "multiple_aug": "Multiple Learnware Reuse (FeatAug)", + "ensemble_pruning": "Multiple Learnware Reuse (EnsemblePrune)", + "multiple_avg": "Multiple Learnware Reuse (Averaging)", } align_model_params = { @@ -47,9 +47,9 @@ align_model_params = { } market_mapping_params = { - "lr": 1e-4, + "lr": 1e-4, "num_epoch": 50, - "batch_size": 64, + "batch_size": 64, "num_partition": 2, # num of column partitions for pos/neg sampling "overlap_ratio": 0.7, # specify the overlap ratio of column partitions during the CL "hidden_dim": 256, # the dimension of hidden embeddings @@ -73,10 +73,10 @@ user_model_params = { "n_estimators": 100000, "boost_from_average": False, "num_threads": 32, - "verbose": -1 + "verbose": -1, }, "MAX_ROUNDS": 1000, - "early_stopping_rounds": 1000 + "early_stopping_rounds": 1000, } }, "PFS": { @@ -95,7 +95,7 @@ user_model_params = { "n_estimators": 100000, }, "MAX_ROUNDS": 1000, - "early_stopping_rounds": 1000 + "early_stopping_rounds": 1000, } }, "PFS_HOMO": { @@ -114,185 +114,620 @@ user_model_params = { "n_estimators": 100000, }, "MAX_ROUNDS": 1000, - "early_stopping_rounds": 1000 + "early_stopping_rounds": 1000, } - } + }, } homo_table_benchmark_config = BenchmarkConfig( name="PFS_HOMO", user_num=53, learnware_ids=[ - "00002265", "00002266", "00002267", "00002268", - "00002269", "00002270", "00002271", "00002272", - "00002273", "00002274", "00002275", "00002276", - "00002277", "00002278", "00002279", "00002280", - "00002281", "00002282", "00002283", "00002284", - "00002285", "00002286", "00002287", "00002288", - "00002289", "00002290", "00002291", "00002292", - "00002293", "00002294", "00002295", "00002296", - "00002297", "00002298", "00002299", "00002300", - "00002301", "00002302", "00002303", "00002304", - "00002305", "00002306", "00002307", "00002308", - "00002309", "00002310", "00002311", "00002312", - "00002313", "00002314", "00002315", "00002316", - "00002317" + "00002265", + "00002266", + "00002267", + "00002268", + "00002269", + "00002270", + "00002271", + "00002272", + "00002273", + "00002274", + "00002275", + "00002276", + "00002277", + "00002278", + "00002279", + "00002280", + "00002281", + "00002282", + "00002283", + "00002284", + "00002285", + "00002286", + "00002287", + "00002288", + "00002289", + "00002290", + "00002291", + "00002292", + "00002293", + "00002294", + "00002295", + "00002296", + "00002297", + "00002298", + "00002299", + "00002300", + "00002301", + "00002302", + "00002303", + "00002304", + "00002305", + "00002306", + "00002307", + "00002308", + "00002309", + "00002310", + "00002311", + "00002312", + "00002313", + "00002314", + "00002315", + "00002316", + "00002317", ], test_data_path="PFS_HOMO/test_data.zip", train_data_path="PFS_HOMO/train_data.zip", - extra_info_path="PFS_HOMO/extra_info.zip" + extra_info_path="PFS_HOMO/extra_info.zip", ) -hetero_cross_feat_eng_benchmark_config = BenchmarkConfig( +hetero_cross_feat_eng_benchmark_config = BenchmarkConfig( name="PFS", user_num=41, - learnware_ids = [ - "00000342", "00000343", "00000344", "00000345", - "00000346", "00000347", "00000348", "00000349", - "00000350", "00000351", "00000352", "00000353", - "00000354", "00000355", "00000356", "00000357", - "00000358", "00000359", "00000360", "00000361", - "00000362", "00000363", "00000364", "00000365", - "00000366", "00000367", "00000368", "00000369", - "00000370", "00000371", "00000372", "00000373", - "00000374", "00000375", "00000376", "00000377", - "00000378", "00000379", "00000380", "00000381", - "00000382", "00000383", "00000384", "00000385", - "00000386", "00000387", "00000388", "00000389", - "00000390", "00000391", "00000392", "00000393", - "00000394", "00000395", "00000396", "00000397", - "00000398", "00000399", "00000400", "00000401", - "00000402", "00000403", "00000404", "00000405", - "00000406", "00000407", "00000408", "00000409", - "00000410", "00000411", "00000412", "00000413", - "00000414", "00000415", "00000416", "00000417", - "00000418", "00000419", "00000420", "00000421", - "00000422", "00000423", "00000424", "00000425", - "00000426", "00000427", "00000428", "00000429", - "00000430", "00000431", "00000432", "00000433", - "00000434", "00000435", "00000436", "00000437", - "00000438", "00000439", "00000440", "00000441", - "00000442", "00000443", "00000444", "00000730", - "00000731", "00000732", "00000733", "00000734", - "00000735", "00000736", "00000737", "00000738", - "00000739", "00000740", "00000741", "00000742", - "00000743", "00000744", "00000745", "00000746", - "00000747", "00000748", "00000749", "00000750", - "00000751", "00000752", "00000753", "00000754", - "00000755", "00000756", "00000757", "00000758", - "00000759", "00000760", "00000761", "00000762", - "00000763", "00000764", "00000765", "00000766", - "00000767", "00000768", "00000769", "00000770", - "00000771", "00000772", "00000773", "00000774", - "00000775", "00000776", "00000777", "00000778", - "00000779", "00000780", "00000781", "00000782", - "00000783", "00000786", "00000787", "00000788", - "00000789", "00000790", "00000791", "00000792", - "00000793", "00000794", "00000795", "00000796", - "00000797", "00000798", "00000799", "00000800", - "00000801", "00000802", "00000803", "00000804", - "00000805", "00000806", "00000807", "00000808", - "00000809", "00000810", "00000811", "00000812", - "00000813", "00000814", "00000815", "00000816", - "00000817", "00000818", "00000819", "00000820", - "00000821", "00000822", "00000823", "00000824", - "00000825", "00000826", "00000827", "00000828", - "00000829", "00000830", "00000831", "00000832", - "00000833", "00000834", "00000835", "00000836", - "00000837", "00000838", "00000839", "00000859", - "00000860", "00000861", "00000862", "00000863", - "00000864", "00000865", "00000866", "00000867", - "00000868", "00000869", "00000870", "00000871", - "00000872", "00000873", "00000874", "00000875", - "00000876", "00000877", "00000878", "00000879", - "00000880", "00000881", "00000882", "00000883", - "00000884", "00000885", "00000886", "00000887", - "00000888", "00000889", "00000890", "00000891", - "00000892", "00000893", "00000894", "00000895", - "00000896", "00000897", "00000898", "00000899", - "00000900", "00000901", "00000902", "00000903", - "00000904", "00000905", "00000906", "00000907", - "00000908", "00000909", "00000910", "00000911", - "00000912" + learnware_ids=[ + "00000342", + "00000343", + "00000344", + "00000345", + "00000346", + "00000347", + "00000348", + "00000349", + "00000350", + "00000351", + "00000352", + "00000353", + "00000354", + "00000355", + "00000356", + "00000357", + "00000358", + "00000359", + "00000360", + "00000361", + "00000362", + "00000363", + "00000364", + "00000365", + "00000366", + "00000367", + "00000368", + "00000369", + "00000370", + "00000371", + "00000372", + "00000373", + "00000374", + "00000375", + "00000376", + "00000377", + "00000378", + "00000379", + "00000380", + "00000381", + "00000382", + "00000383", + "00000384", + "00000385", + "00000386", + "00000387", + "00000388", + "00000389", + "00000390", + "00000391", + "00000392", + "00000393", + "00000394", + "00000395", + "00000396", + "00000397", + "00000398", + "00000399", + "00000400", + "00000401", + "00000402", + "00000403", + "00000404", + "00000405", + "00000406", + "00000407", + "00000408", + "00000409", + "00000410", + "00000411", + "00000412", + "00000413", + "00000414", + "00000415", + "00000416", + "00000417", + "00000418", + "00000419", + "00000420", + "00000421", + "00000422", + "00000423", + "00000424", + "00000425", + "00000426", + "00000427", + "00000428", + "00000429", + "00000430", + "00000431", + "00000432", + "00000433", + "00000434", + "00000435", + "00000436", + "00000437", + "00000438", + "00000439", + "00000440", + "00000441", + "00000442", + "00000443", + "00000444", + "00000730", + "00000731", + "00000732", + "00000733", + "00000734", + "00000735", + "00000736", + "00000737", + "00000738", + "00000739", + "00000740", + "00000741", + "00000742", + "00000743", + "00000744", + "00000745", + "00000746", + "00000747", + "00000748", + "00000749", + "00000750", + "00000751", + "00000752", + "00000753", + "00000754", + "00000755", + "00000756", + "00000757", + "00000758", + "00000759", + "00000760", + "00000761", + "00000762", + "00000763", + "00000764", + "00000765", + "00000766", + "00000767", + "00000768", + "00000769", + "00000770", + "00000771", + "00000772", + "00000773", + "00000774", + "00000775", + "00000776", + "00000777", + "00000778", + "00000779", + "00000780", + "00000781", + "00000782", + "00000783", + "00000786", + "00000787", + "00000788", + "00000789", + "00000790", + "00000791", + "00000792", + "00000793", + "00000794", + "00000795", + "00000796", + "00000797", + "00000798", + "00000799", + "00000800", + "00000801", + "00000802", + "00000803", + "00000804", + "00000805", + "00000806", + "00000807", + "00000808", + "00000809", + "00000810", + "00000811", + "00000812", + "00000813", + "00000814", + "00000815", + "00000816", + "00000817", + "00000818", + "00000819", + "00000820", + "00000821", + "00000822", + "00000823", + "00000824", + "00000825", + "00000826", + "00000827", + "00000828", + "00000829", + "00000830", + "00000831", + "00000832", + "00000833", + "00000834", + "00000835", + "00000836", + "00000837", + "00000838", + "00000839", + "00000859", + "00000860", + "00000861", + "00000862", + "00000863", + "00000864", + "00000865", + "00000866", + "00000867", + "00000868", + "00000869", + "00000870", + "00000871", + "00000872", + "00000873", + "00000874", + "00000875", + "00000876", + "00000877", + "00000878", + "00000879", + "00000880", + "00000881", + "00000882", + "00000883", + "00000884", + "00000885", + "00000886", + "00000887", + "00000888", + "00000889", + "00000890", + "00000891", + "00000892", + "00000893", + "00000894", + "00000895", + "00000896", + "00000897", + "00000898", + "00000899", + "00000900", + "00000901", + "00000902", + "00000903", + "00000904", + "00000905", + "00000906", + "00000907", + "00000908", + "00000909", + "00000910", + "00000911", + "00000912", ], test_data_path="PFS/test_data.zip", train_data_path="PFS/train_data.zip", - extra_info_path="PFS/extra_info.zip" + extra_info_path="PFS/extra_info.zip", ) hetero_cross_task_benchmark_config = BenchmarkConfig( name="M5", user_num=30, - learnware_ids = [ - "00000342", "00000343", "00000344", "00000345", - "00000346", "00000347", "00000348", "00000349", - "00000350", "00000351", "00000352", "00000353", - "00000354", "00000355", "00000356", "00000357", - "00000358", "00000359", "00000360", "00000361", - "00000362", "00000363", "00000364", "00000365", - "00000366", "00000367", "00000368", "00000369", - "00000370", "00000371", "00000372", "00000373", - "00000374", "00000375", "00000376", "00000377", - "00000378", "00000379", "00000380", "00000381", - "00000382", "00000383", "00000384", "00000385", - "00000386", "00000387", "00000388", "00000389", - "00000390", "00000391", "00000392", "00000393", - "00000394", "00000395", "00000396", "00000397", - "00000398", "00000399", "00000400", "00000401", - "00000402", "00000403", "00000404", "00000405", - "00000406", "00000407", "00000408", "00000409", - "00000410", "00000411", "00000412", "00000413", - "00000414", "00000415", "00000416", "00000417", - "00000418", "00000419", "00000420", "00000421", - "00000422", "00000423", "00000424", "00000425", - "00000426", "00000427", "00000428", "00000429", - "00000430", "00000431", "00000432", "00000433", - "00000434", "00000435", "00000436", "00000437", - "00000438", "00000439", "00000440", "00000441", - "00000442", "00000443", "00000444", "00000730", - "00000731", "00000732", "00000733", "00000734", - "00000735", "00000736", "00000737", "00000738", - "00000739", "00000740", "00000741", "00000742", - "00000743", "00000744", "00000745", "00000746", - "00000747", "00000748", "00000749", "00000750", - "00000751", "00000752", "00000753", "00000754", - "00000755", "00000756", "00000757", "00000758", - "00000759", "00000760", "00000761", "00000762", - "00000763", "00000764", "00000765", "00000766", - "00000767", "00000768", "00000769", "00000770", - "00000771", "00000772", "00000773", "00000774", - "00000775", "00000776", "00000777", "00000778", - "00000779", "00000780", "00000781", "00000782", - "00000783", "00000786", "00000787", "00000788", - "00000789", "00000790", "00000791", "00000792", - "00000793", "00000794", "00000795", "00000796", - "00000797", "00000798", "00000799", "00000800", - "00000801", "00000802", "00000803", "00000804", - "00000805", "00000806", "00000807", "00000808", - "00000809", "00000810", "00000811", "00000812", - "00000813", "00000814", "00000815", "00000816", - "00000817", "00000818", "00000819", "00000820", - "00000821", "00000822", "00000823", "00000824", - "00000825", "00000826", "00000827", "00000828", - "00000829", "00000830", "00000831", "00000832", - "00000833", "00000834", "00000835", "00000836", - "00000837", "00000838", "00000839", "00000859", - "00000860", "00000861", "00000862", "00000863", - "00000864", "00000865", "00000866", "00000867", - "00000868", "00000869", "00000870", "00000871", - "00000872", "00000873", "00000874", "00000875", - "00000876", "00000877", "00000878", "00000879", - "00000880", "00000881", "00000882", "00000883", - "00000884", "00000885", "00000886", "00000887", - "00000888", "00000889", "00000890", "00000891", - "00000892", "00000893", "00000894", "00000895", - "00000896", "00000897", "00000898", "00000899", - "00000900", "00000901", "00000902", "00000903", - "00000904", "00000905", "00000906", "00000907", - "00000908", "00000909", "00000910", "00000911", - "00000912" + learnware_ids=[ + "00000342", + "00000343", + "00000344", + "00000345", + "00000346", + "00000347", + "00000348", + "00000349", + "00000350", + "00000351", + "00000352", + "00000353", + "00000354", + "00000355", + "00000356", + "00000357", + "00000358", + "00000359", + "00000360", + "00000361", + "00000362", + "00000363", + "00000364", + "00000365", + "00000366", + "00000367", + "00000368", + "00000369", + "00000370", + "00000371", + "00000372", + "00000373", + "00000374", + "00000375", + "00000376", + "00000377", + "00000378", + "00000379", + "00000380", + "00000381", + "00000382", + "00000383", + "00000384", + "00000385", + "00000386", + "00000387", + "00000388", + "00000389", + "00000390", + "00000391", + "00000392", + "00000393", + "00000394", + "00000395", + "00000396", + "00000397", + "00000398", + "00000399", + "00000400", + "00000401", + "00000402", + "00000403", + "00000404", + "00000405", + "00000406", + "00000407", + "00000408", + "00000409", + "00000410", + "00000411", + "00000412", + "00000413", + "00000414", + "00000415", + "00000416", + "00000417", + "00000418", + "00000419", + "00000420", + "00000421", + "00000422", + "00000423", + "00000424", + "00000425", + "00000426", + "00000427", + "00000428", + "00000429", + "00000430", + "00000431", + "00000432", + "00000433", + "00000434", + "00000435", + "00000436", + "00000437", + "00000438", + "00000439", + "00000440", + "00000441", + "00000442", + "00000443", + "00000444", + "00000730", + "00000731", + "00000732", + "00000733", + "00000734", + "00000735", + "00000736", + "00000737", + "00000738", + "00000739", + "00000740", + "00000741", + "00000742", + "00000743", + "00000744", + "00000745", + "00000746", + "00000747", + "00000748", + "00000749", + "00000750", + "00000751", + "00000752", + "00000753", + "00000754", + "00000755", + "00000756", + "00000757", + "00000758", + "00000759", + "00000760", + "00000761", + "00000762", + "00000763", + "00000764", + "00000765", + "00000766", + "00000767", + "00000768", + "00000769", + "00000770", + "00000771", + "00000772", + "00000773", + "00000774", + "00000775", + "00000776", + "00000777", + "00000778", + "00000779", + "00000780", + "00000781", + "00000782", + "00000783", + "00000786", + "00000787", + "00000788", + "00000789", + "00000790", + "00000791", + "00000792", + "00000793", + "00000794", + "00000795", + "00000796", + "00000797", + "00000798", + "00000799", + "00000800", + "00000801", + "00000802", + "00000803", + "00000804", + "00000805", + "00000806", + "00000807", + "00000808", + "00000809", + "00000810", + "00000811", + "00000812", + "00000813", + "00000814", + "00000815", + "00000816", + "00000817", + "00000818", + "00000819", + "00000820", + "00000821", + "00000822", + "00000823", + "00000824", + "00000825", + "00000826", + "00000827", + "00000828", + "00000829", + "00000830", + "00000831", + "00000832", + "00000833", + "00000834", + "00000835", + "00000836", + "00000837", + "00000838", + "00000839", + "00000859", + "00000860", + "00000861", + "00000862", + "00000863", + "00000864", + "00000865", + "00000866", + "00000867", + "00000868", + "00000869", + "00000870", + "00000871", + "00000872", + "00000873", + "00000874", + "00000875", + "00000876", + "00000877", + "00000878", + "00000879", + "00000880", + "00000881", + "00000882", + "00000883", + "00000884", + "00000885", + "00000886", + "00000887", + "00000888", + "00000889", + "00000890", + "00000891", + "00000892", + "00000893", + "00000894", + "00000895", + "00000896", + "00000897", + "00000898", + "00000899", + "00000900", + "00000901", + "00000902", + "00000903", + "00000904", + "00000905", + "00000906", + "00000907", + "00000908", + "00000909", + "00000910", + "00000911", + "00000912", ], test_data_path="M5/test_data.zip", train_data_path="M5/train_data.zip", - extra_info_path="M5/extra_info.zip" -) \ No newline at end of file + extra_info_path="M5/extra_info.zip", +) diff --git a/examples/dataset_table_workflow/hetero.py b/examples/dataset_table_workflow/hetero.py index 68842fc..423f7b9 100644 --- a/examples/dataset_table_workflow/hetero.py +++ b/examples/dataset_table_workflow/hetero.py @@ -1,6 +1,5 @@ import os import warnings -warnings.filterwarnings("ignore") import numpy as np from learnware.logger import get_module_logger @@ -8,11 +7,12 @@ from learnware.specification import generate_stat_spec from learnware.market import BaseUserInfo from learnware.reuse import AveragingReuser, FeatureAlignLearnware -from methods import * +from methods import loss_func_rmse from base import TableWorkflow from config import align_model_params, user_semantic, hetero_n_labeled_list, hetero_n_repeat_list from utils import Recorder, plot_performance_curves, set_seed +warnings.filterwarnings("ignore") logger = get_module_logger("hetero_test", level="INFO") @@ -24,7 +24,7 @@ class HeterogeneousDatasetWorkflow(TableWorkflow): single_score_list = [] ensemble_score_list = [] all_learnwares = self.market.get_learnwares() - + user = self.benchmark.name for idx in range(self.benchmark.user_num): test_x, test_y = self.benchmark.get_test_data(user_ids=idx) @@ -32,23 +32,21 @@ class HeterogeneousDatasetWorkflow(TableWorkflow): user_stat_spec = generate_stat_spec(type="table", X=test_x) input_description = { "Dimension": len(feature_descriptions), - "Description": {str(i): feature_descriptions[i] for i in range(len(feature_descriptions))} + "Description": {str(i): feature_descriptions[i] for i in range(len(feature_descriptions))}, } user_semantic["Input"] = input_description - user_info = BaseUserInfo( - semantic_spec=user_semantic, stat_info={user_stat_spec.type: user_stat_spec} - ) + user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={user_stat_spec.type: user_stat_spec}) logger.info(f"Searching Market for user: {user}_{idx}") - + search_result = self.market.search_learnware(user_info, search_method="auto") single_result = search_result.get_single_results() multiple_result = search_result.get_multiple_results() - + logger.info(f"hetero search result of user {user}_{idx}: {single_result[0].learnware.id}") logger.info( f"single model num: {len(single_result)}, max_score: {single_result[0].score}, min_score: {single_result[-1].score}" ) - + single_hetero_learnware = FeatureAlignLearnware(single_result[0].learnware, **align_model_params) single_hetero_learnware.align(user_rkme=user_stat_spec) pred_y = single_hetero_learnware.predict(test_x) @@ -59,11 +57,11 @@ class HeterogeneousDatasetWorkflow(TableWorkflow): hetero_learnware = FeatureAlignLearnware(learnware, **align_model_params) hetero_learnware.align(user_rkme=user_stat_spec) pred_y = hetero_learnware.predict(test_x) - rmse_list.append(loss_func_rmse(pred_y, test_y)) + rmse_list.append(loss_func_rmse(pred_y, test_y)) logger.info( f"Top1-score: {single_result[0].score}, learnware_id: {single_result[0].learnware.id}, rmse: {single_score_list[0]}" ) - + if len(multiple_result) > 0: mixture_id = " ".join([learnware.id for learnware in multiple_result[0].learnwares]) logger.info(f"mixture_score: {multiple_result[0].score}, mixture_learnware: {mixture_id}") @@ -85,11 +83,11 @@ class HeterogeneousDatasetWorkflow(TableWorkflow): logger.info(f"mixture reuse rmse (ensemble): {ensemble_score}") learnware_rmse_list.append(rmse_list) - + single_list = np.array(learnware_rmse_list) avg_score_list = [np.mean(lst, axis=0) for lst in single_list] oracle_score_list = [np.min(lst, axis=0) for lst in single_list] - + logger.info( "RMSE of selected learnware: %.3f +/- %.3f, Average performance: %.3f +/- %.3f, Oracle performace: %.3f +/- %.3f" % ( @@ -98,7 +96,7 @@ class HeterogeneousDatasetWorkflow(TableWorkflow): np.mean(avg_score_list), np.std(avg_score_list), np.mean(oracle_score_list), - np.std(oracle_score_list) + np.std(oracle_score_list), ) ) logger.info( @@ -106,33 +104,30 @@ class HeterogeneousDatasetWorkflow(TableWorkflow): % (np.mean(ensemble_score_list), np.std(ensemble_score_list)) ) - def labeled_hetero_table_example(self, skip_test): set_seed(0) logger.info("Total Items: %d" % len(self.market)) methods = ["user_model", "hetero_single_aug", "hetero_multiple_avg", "hetero_ensemble_pruning"] recorders = {method: Recorder() for method in methods} user = self.benchmark.name - + if not skip_test: for idx in range(self.benchmark.user_num): test_x, test_y = self.benchmark.get_test_data(user_ids=idx) test_x, test_y = test_x.values, test_y.values - + train_x, train_y = self.benchmark.get_train_data(user_ids=idx) train_x, train_y, feature_descriptions = train_x.values, train_y.values, train_x.columns train_subsets = self.get_train_subsets(hetero_n_labeled_list, hetero_n_repeat_list, train_x, train_y) - + user_stat_spec = generate_stat_spec(type="table", X=test_x) input_description = { "Dimension": len(feature_descriptions), - "Description": {str(i): feature_descriptions[i] for i in range(len(feature_descriptions))} + "Description": {str(i): feature_descriptions[i] for i in range(len(feature_descriptions))}, } user_semantic["Input"] = input_description - user_info = BaseUserInfo( - semantic_spec=user_semantic, stat_info={user_stat_spec.type: user_stat_spec} - ) - + user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={user_stat_spec.type: user_stat_spec}) + logger.info(f"Searching Market for user: {user}_{idx}") search_result = self.market.search_learnware(user_info) single_result = search_result.get_single_results() @@ -144,16 +139,25 @@ class HeterogeneousDatasetWorkflow(TableWorkflow): mixture_learnware_list = multiple_result[0].learnwares else: mixture_learnware_list = [single_result[0].learnware] - - logger.info(f"Hetero search result of user {user}_{idx}: mixture learnware num: {len(mixture_learnware_list)}") - test_info = {"user": user, "idx": idx, "train_subsets": train_subsets, "test_x": test_x, "test_y": test_y, "n_labeled_list": hetero_n_labeled_list} + logger.info( + f"Hetero search result of user {user}_{idx}: mixture learnware num: {len(mixture_learnware_list)}" + ) + + test_info = { + "user": user, + "idx": idx, + "train_subsets": train_subsets, + "test_x": test_x, + "test_y": test_y, + "n_labeled_list": hetero_n_labeled_list, + } common_config = {"user_rkme": user_stat_spec, "learnwares": mixture_learnware_list} method_configs = { "user_model": {"dataset": self.benchmark.name, "model_type": "lgb"}, "hetero_single_aug": {"user_rkme": user_stat_spec, "single_learnware": single_result[0].learnware}, "hetero_multiple_avg": common_config, - "hetero_ensemble_pruning": common_config + "hetero_ensemble_pruning": common_config, } for method_name in methods: @@ -161,8 +165,10 @@ class HeterogeneousDatasetWorkflow(TableWorkflow): test_info["method_name"] = method_name test_info.update(method_configs[method_name]) self.test_method(test_info, recorders, loss_func=loss_func_rmse) - + for method, recorder in recorders.items(): recorder.save(os.path.join(self.curves_result_path, f"{user}/{user}_{method}_performance.json")) - - plot_performance_curves(self.curves_result_path, user, recorders, task="Hetero", n_labeled_list=hetero_n_labeled_list) \ No newline at end of file + + plot_performance_curves( + self.curves_result_path, user, recorders, task="Hetero", n_labeled_list=hetero_n_labeled_list + ) diff --git a/examples/dataset_table_workflow/homo.py b/examples/dataset_table_workflow/homo.py index b22f4e0..e23c7be 100644 --- a/examples/dataset_table_workflow/homo.py +++ b/examples/dataset_table_workflow/homo.py @@ -1,18 +1,18 @@ import os import warnings import numpy as np -warnings.filterwarnings("ignore") from learnware.market import BaseUserInfo from learnware.logger import get_module_logger from learnware.specification import generate_stat_spec -from learnware.reuse import AveragingReuser, JobSelectorReuser, EnsemblePruningReuser +from learnware.reuse import AveragingReuser, JobSelectorReuser -from methods import * +from methods import loss_func_rmse from base import TableWorkflow from config import homo_n_labeled_list, homo_n_repeat_list from utils import Recorder, plot_performance_curves +warnings.filterwarnings("ignore") logger = get_module_logger("homo_table", level="INFO") @@ -30,9 +30,7 @@ class HomogeneousDatasetWorkflow(TableWorkflow): test_x, test_y = self.benchmark.get_test_data(user_ids=idx) test_x, test_y = test_x.values, test_y.values user_stat_spec = generate_stat_spec(type="table", X=test_x) - user_info = BaseUserInfo( - semantic_spec=self.user_semantic, stat_info={user_stat_spec.type: user_stat_spec} - ) + user_info = BaseUserInfo(semantic_spec=self.user_semantic, stat_info={user_stat_spec.type: user_stat_spec}) logger.info(f"Searching Market for user: {user}_{idx}") search_result = self.market.search_learnware(user_info, max_search_num=2) @@ -43,7 +41,7 @@ class HomogeneousDatasetWorkflow(TableWorkflow): logger.info( f"single model num: {len(single_result)}, max_score: {single_result[0].score}, min_score: {single_result[-1].score}" ) - + pred_y = single_result[0].learnware.predict(test_x) single_score_list.append(loss_func_rmse(pred_y, test_y)) @@ -77,7 +75,7 @@ class HomogeneousDatasetWorkflow(TableWorkflow): ensemble_score = loss_func_rmse(ensemble_predict_y, test_y) ensemble_score_list.append(ensemble_score) logger.info(f"mixture reuse rmse (ensemble): {ensemble_score}") - + learnware_rmse_list.append(rmse_list) single_list = np.array(learnware_rmse_list) @@ -103,19 +101,18 @@ class HomogeneousDatasetWorkflow(TableWorkflow): "Averaging Ensemble Reuse Performance: %.3f +/- %.3f" % (np.mean(ensemble_score_list), np.std(ensemble_score_list)) ) - - + def labeled_homo_table_example(self, skip_test): logger.info("Total Item: %d" % (len(self.market))) methods = ["user_model", "homo_single_aug", "homo_ensemble_pruning"] recorders = {method: Recorder() for method in methods} user = self.benchmark.name - + if not skip_test: for idx in range(self.benchmark.user_num): test_x, test_y = self.benchmark.get_test_data(user_ids=idx) test_x, test_y = test_x.values, test_y.values - + train_x, train_y = self.benchmark.get_train_data(user_ids=idx) train_x, train_y = train_x.values, train_y.values train_subsets = self.get_train_subsets(homo_n_labeled_list, homo_n_repeat_list, train_x, train_y) @@ -124,7 +121,7 @@ class HomogeneousDatasetWorkflow(TableWorkflow): user_info = BaseUserInfo( semantic_spec=self.user_semantic, stat_info={"RKMETableSpecification": user_stat_spec} ) - + logger.info(f"Searching Market for user: {user}_{idx}") search_result = self.market.search_learnware(user_info) single_result = search_result.get_single_results() @@ -142,12 +139,18 @@ class HomogeneousDatasetWorkflow(TableWorkflow): else: mixture_learnware_list = [single_result[0].learnware] - test_info = {"user": user, "idx": idx, "train_subsets": train_subsets, "test_x": test_x, "test_y": test_y} + test_info = { + "user": user, + "idx": idx, + "train_subsets": train_subsets, + "test_x": test_x, + "test_y": test_y, + } common_config = {"learnwares": mixture_learnware_list} method_configs = { "user_model": {"dataset": self.benchmark.name, "model_type": "lgb"}, "homo_single_aug": {"single_learnware": [single_result[0].learnware]}, - "homo_ensemble_pruning": common_config + "homo_ensemble_pruning": common_config, } for method_name in methods: @@ -155,8 +158,10 @@ class HomogeneousDatasetWorkflow(TableWorkflow): test_info["method_name"] = method_name test_info.update(method_configs[method_name]) self.test_method(test_info, recorders, loss_func=loss_func_rmse) - + for method, recorder in recorders.items(): recorder.save(os.path.join(self.curves_result_path, f"{user}/{user}_{method}_performance.json")) - plot_performance_curves(self.curves_result_path, user, recorders, task="Homo", n_labeled_list=homo_n_labeled_list) \ No newline at end of file + plot_performance_curves( + self.curves_result_path, user, recorders, task="Homo", n_labeled_list=homo_n_labeled_list + ) diff --git a/examples/dataset_table_workflow/methods.py b/examples/dataset_table_workflow/methods.py index fe40eef..b682919 100644 --- a/examples/dataset_table_workflow/methods.py +++ b/examples/dataset_table_workflow/methods.py @@ -31,7 +31,7 @@ class HomoScoringMethods: reuse_multiple_augment = FeatureAugmentReuser(multiple_learnwares, mode="regression") reuse_multiple_augment.fit(x_train=x_train, y_train=y_train) return reuse_multiple_augment - + @staticmethod def multiple_avg_score(x_train, y_train, test_info): multiple_learnwares = test_info["learnwares"] @@ -64,29 +64,35 @@ class HeteroMethods: reuse_single_augment = HeteroMapAlignLearnware(single_learnware, mode="regression", **align_model_params) reuse_single_augment.align(user_rkme=user_rkme, x_train=x_train, y_train=y_train) return reuse_single_augment - + @staticmethod def multiple_aug_score(x_train, y_train, test_info): user_rkme, multiple_learnwares = test_info["user_rkme"], test_info["learnwares"] - hetero_learnware_list = HeteroMethods.create_hetero_learnware_list(multiple_learnwares, user_rkme, x_train, y_train) + hetero_learnware_list = HeteroMethods.create_hetero_learnware_list( + multiple_learnwares, user_rkme, x_train, y_train + ) reuse_multiple_augment = FeatureAugmentReuser(hetero_learnware_list, mode="regression") reuse_multiple_augment.fit(x_train=x_train, y_train=y_train) return reuse_multiple_augment - + @staticmethod def multiple_ensemble_pruning_score(x_train, y_train, test_info): user_rkme, multiple_learnwares = test_info["user_rkme"], test_info["learnwares"] - hetero_learnware_list = HeteroMethods.create_hetero_learnware_list(multiple_learnwares, user_rkme, x_train, y_train) + hetero_learnware_list = HeteroMethods.create_hetero_learnware_list( + multiple_learnwares, user_rkme, x_train, y_train + ) if len(hetero_learnware_list) == 1: return hetero_learnware_list[0] reuse_pruning = EnsemblePruningReuser(hetero_learnware_list, mode="regression") reuse_pruning.fit(val_X=x_train, val_y=y_train) return reuse_pruning - + @staticmethod def multiple_avg_score(x_train, y_train, test_info): user_rkme, multiple_learnwares = test_info["user_rkme"], test_info["learnwares"] - hetero_learnware_list = HeteroMethods.create_hetero_learnware_list(multiple_learnwares, user_rkme, x_train, y_train) + hetero_learnware_list = HeteroMethods.create_hetero_learnware_list( + multiple_learnwares, user_rkme, x_train, y_train + ) reuse_multiple_avg = AveragingReuser(hetero_learnware_list, mode="mean") return reuse_multiple_avg @@ -100,5 +106,5 @@ test_methods = { "homo_single_aug": HomoScoringMethods.single_aug_score, "homo_multiple_aug": HomoScoringMethods.multiple_aug_score, "homo_multiple_avg": HomoScoringMethods.multiple_avg_score, - "homo_ensemble_pruning": HomoScoringMethods.multiple_ensemble_pruning_score -} \ No newline at end of file + "homo_ensemble_pruning": HomoScoringMethods.multiple_ensemble_pruning_score, +} diff --git a/examples/dataset_table_workflow/train.py b/examples/dataset_table_workflow/train.py index b4f1b6c..d8cf66b 100644 --- a/examples/dataset_table_workflow/train.py +++ b/examples/dataset_table_workflow/train.py @@ -1,7 +1,5 @@ -import numpy as np import lightgbm as lgb from lightgbm import early_stopping -from sklearn.metrics import mean_squared_error from learnware.logger import get_module_logger from config import user_model_params @@ -24,7 +22,7 @@ def train_lgb(X_train, y_train, X_val, y_val, dataset): dtrain, num_boost_round=MAX_ROUNDS, valid_sets=[dtrain, dval] if dataset == "Corporacion" else [dval], - callbacks=[early_stopping(model_param["early_stopping_rounds"], verbose=False)] + callbacks=[early_stopping(model_param["early_stopping_rounds"], verbose=False)], ) val_pred.append(bst.predict(X_val, num_iteration=bst.best_iteration or MAX_ROUNDS)) return bst @@ -38,6 +36,6 @@ def train_model(X_train, y_train, X_val, y_val, test_info): dataset = test_info["dataset"] model_type = test_info["model_type"] assert model_type in ["lgb", "ridge"] - + if model_type == "lgb": - return train_lgb(X_train, y_train, X_val, y_val, dataset) \ No newline at end of file + return train_lgb(X_train, y_train, X_val, y_val, dataset) diff --git a/examples/dataset_table_workflow/utils.py b/examples/dataset_table_workflow/utils.py index f176a6a..08e21e6 100644 --- a/examples/dataset_table_workflow/utils.py +++ b/examples/dataset_table_workflow/utils.py @@ -8,7 +8,7 @@ import matplotlib.pyplot as plt from collections import defaultdict from learnware.logger import get_module_logger -from config import * +from config import styles, labels logger = get_module_logger("base_table", level="INFO") @@ -29,46 +29,46 @@ class Recorder: def save(self, path): with open(path, "w") as f: json.dump(self.data, f, indent=4, default=list) - + def load(self, path): with open(path, "r") as f: self.data = json.load(f, object_hook=lambda x: defaultdict(list, x)) - + def should_test_method(self, user, idx, path): if os.path.exists(path): self.load(path) return user not in self.data or idx > len(self.data[user]) - 1 return True - + def plot_performance_curves(path, user, recorders, task, n_labeled_list): plt.figure(figsize=(10, 6)) plt.xticks(range(len(n_labeled_list)), n_labeled_list) - for method, recorder in recorders.items(): + for method, recorder in recorders.items(): data_path = os.path.join(path, f"{user}/{user}_{method}_performance.json") recorder.load(data_path) scores_array = recorder.get_performance_data(user) - + mean_curve, std_curve = [], [] for i in range(len(n_labeled_list)): sub_scores_array = np.vstack([lst[i] for lst in scores_array]) - sub_scores_mean = np.squeeze(np.mean(sub_scores_array, axis=0)) + sub_scores_mean = np.squeeze(np.mean(sub_scores_array, axis=0)) mean_curve.append(np.mean(sub_scores_mean)) std_curve.append(np.std(sub_scores_mean)) - + mean_curve = np.array(mean_curve) std_curve = np.array(std_curve) - method_plot = '_'.join(method.split('_')[1:]) if method not in ['user_model', 'oracle_score', 'select_score', 'mean_score'] else method + method_plot = ( + "_".join(method.split("_")[1:]) + if method not in ["user_model", "oracle_score", "select_score", "mean_score"] + else method + ) style = styles.get(method_plot, {"color": "black", "linestyle": "-"}) plt.plot(mean_curve, label=labels.get(method_plot), **style) plt.fill_between( - range(len(mean_curve)), - mean_curve - std_curve, - mean_curve + std_curve, - color=style["color"], - alpha=0.2 + range(len(mean_curve)), mean_curve - std_curve, mean_curve + std_curve, color=style["color"], alpha=0.2 ) plt.xlabel("Amount of Labeled User Data", fontsize=14) @@ -76,7 +76,7 @@ def plot_performance_curves(path, user, recorders, task, n_labeled_list): plt.title(f"Results on {task} Table Experimental Scenario", fontsize=16) plt.legend(fontsize=12) plt.tight_layout() - + root_path = os.path.abspath(os.path.join(__file__, "..")) fig_path = os.path.join(root_path, "results", "figs") os.makedirs(fig_path, exist_ok=True) @@ -91,4 +91,4 @@ def set_seed(seed): torch.cuda.manual_seed(seed) torch.cuda.manual_seed_all(seed) torch.backends.cudnn.benchmark = False - torch.backends.cudnn.deterministic = True \ No newline at end of file + torch.backends.cudnn.deterministic = True