From d00e3094d0b019e278119167054d6bc7d0e746bd Mon Sep 17 00:00:00 2001 From: liuht Date: Fri, 21 Apr 2023 16:21:12 +0800 Subject: [PATCH 1/3] [FIX] add default value for use_herding --- learnware/learnware/reuse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/learnware/learnware/reuse.py b/learnware/learnware/reuse.py index 8d1d0fe..6bb075f 100644 --- a/learnware/learnware/reuse.py +++ b/learnware/learnware/reuse.py @@ -54,7 +54,7 @@ class JobSelectorReuser(BaseReuser): return selector_pred_y - def job_selector(self, user_data: np.ndarray, use_herding: bool): + def job_selector(self, user_data: np.ndarray, use_herding: bool = True): """Train job selector based on user's data, which predicts which learnware in the pool should be selected Parameters From 4e249efab0e2efd337dfaa400177ca88070d7ee5 Mon Sep 17 00:00:00 2001 From: Gene Date: Fri, 21 Apr 2023 16:55:20 +0800 Subject: [PATCH 2/3] [FIX] Fix bugs in M5 --- examples/example_image/main.py | 6 +++--- examples/example_m5/example_init.py | 3 ++- examples/example_pfs/example_init.py | 1 + 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/examples/example_image/main.py b/examples/example_image/main.py index 70589dc..90a34e5 100644 --- a/examples/example_image/main.py +++ b/examples/example_image/main.py @@ -4,7 +4,7 @@ from get_data import * import os import random from utils import generate_uploader, generate_user, ImageDataLoader, train, eval_prediction -from learnware.learnware import Learnware, JobSelectorReuser, EnsembleReuser +from learnware.learnware import Learnware, JobSelectorReuser, AveragingReuser import time from learnware.market import EasyMarket, BaseUserInfo @@ -173,7 +173,7 @@ def test_search(gamma=0.1, load_market=True): job_selector_score_list.append(reuse_score) print(f"mixture reuse loss: {reuse_score}\n") - reuse_ensemble = EnsembleReuser(learnware_list=mixture_learnware_list, mode="vote") + reuse_ensemble = AveragingReuser(learnware_list=mixture_learnware_list, mode="vote") ensemble_predict_y = reuse_ensemble.predict(user_data=user_data) ensemble_score = eval_prediction(ensemble_predict_y, user_label) ensemble_score_list.append(ensemble_score) @@ -198,4 +198,4 @@ def test_search(gamma=0.1, load_market=True): if __name__ == "__main__": # prepare_data() # prepare_model() - test_search(load_market=True) + test_search(load_market=False) diff --git a/examples/example_m5/example_init.py b/examples/example_m5/example_init.py index d875d96..70f366d 100644 --- a/examples/example_m5/example_init.py +++ b/examples/example_m5/example_init.py @@ -1,13 +1,14 @@ import os import joblib import numpy as np +import lightgbm as lgb from learnware.model import BaseModel class Model(BaseModel): def __init__(self): dir_path = os.path.dirname(os.path.abspath(__file__)) - self.model = joblib.load(os.path.join(dir_path, "model.out")) + self.model = lgb.Booster(model_file=os.path.join(dir_path, "model.out")) def fit(self, X: np.ndarray, y: np.ndarray): pass diff --git a/examples/example_pfs/example_init.py b/examples/example_pfs/example_init.py index d875d96..88b788a 100644 --- a/examples/example_pfs/example_init.py +++ b/examples/example_pfs/example_init.py @@ -6,6 +6,7 @@ from learnware.model import BaseModel class Model(BaseModel): def __init__(self): + super(Model, self).__init__(input_shape=(31,), output_shape=()) dir_path = os.path.dirname(os.path.abspath(__file__)) self.model = joblib.load(os.path.join(dir_path, "model.out")) From 1289b6d8632d29a77f7bfc8fae1566d2b3d3dcab Mon Sep 17 00:00:00 2001 From: liuht Date: Fri, 21 Apr 2023 16:56:50 +0800 Subject: [PATCH 3/3] [MNT] Add input_shape and output_shape --- examples/example_m5/example_init.py | 1 + examples/example_m5/main.py | 46 ++++++++++++++++++++++------- 2 files changed, 37 insertions(+), 10 deletions(-) diff --git a/examples/example_m5/example_init.py b/examples/example_m5/example_init.py index d875d96..c5d26d1 100644 --- a/examples/example_m5/example_init.py +++ b/examples/example_m5/example_init.py @@ -6,6 +6,7 @@ from learnware.model import BaseModel class Model(BaseModel): def __init__(self): + super(Model, self).__init__(input_shape=(82,), output_shape=()) dir_path = os.path.dirname(os.path.abspath(__file__)) self.model = joblib.load(os.path.join(dir_path, "model.out")) diff --git a/examples/example_m5/main.py b/examples/example_m5/main.py index 6b7544e..a0853c4 100644 --- a/examples/example_m5/main.py +++ b/examples/example_m5/main.py @@ -1,13 +1,14 @@ import os import fire import zipfile +import numpy as np from tqdm import tqdm from shutil import copyfile, rmtree import learnware from learnware.market import EasyMarket, BaseUserInfo from learnware.market import database_ops -from learnware.learnware import Learnware, JobSelectorReuser +from learnware.learnware import Learnware, JobSelectorReuser, AveragingReuser import learnware.specification as specification from m5 import DataLoader @@ -114,7 +115,7 @@ class M5DatasetWorkflow: rmtree(dir_path) def test(self, regenerate_flag=False): - self.prepare_learnware(regenerate_flag) + #self.prepare_learnware(regenerate_flag) self._init_learnware_market() easy_market = EasyMarket() @@ -122,10 +123,17 @@ class M5DatasetWorkflow: m5 = DataLoader() idx_list = m5.get_idx_list() + os.makedirs("./user_spec", exist_ok=True) + sinle_score_list = [] + random_score_list = [] + job_selector_score_list = [] + ensemble_score_list = [] for idx in idx_list: train_x, train_y, test_x, test_y = m5.get_idx_data(idx) user_spec = specification.utils.generate_rkme_spec(X=test_x, gamma=0.1, cuda_idx=0) + user_spec_path = f"./user_spec/user_{idx}.json" + user_spec.save(user_spec_path) user_info = BaseUserInfo( id=f"user_{idx}", semantic_spec=user_senmantic, stat_info={"RKMEStatSpecification": user_spec} @@ -141,18 +149,36 @@ class M5DatasetWorkflow: print( f"single model num: {len(sorted_score_list)}, max_score: {sorted_score_list[0]}, min_score: {sorted_score_list[-1]}" ) + loss_list = [] for score, learnware in zip(sorted_score_list, single_learnware_list): pred_y = learnware.predict(test_x) - loss = m5.score(test_y, pred_y) - print(f"score: {score}, learnware_id: {learnware.id}, loss: {loss}") + loss_list.append(m5.score(test_y, pred_y)) + print( + f"Top1-score: {sorted_score_list[0]}, learnware_id: {single_learnware_list[0].id}, loss: {loss_list[-1]}" + ) mixture_id = " ".join([learnware.id for learnware in mixture_learnware_list]) - print(f"mixture_learnware: {mixture_id}\n") - - reuse_baseline = JobSelectorReuser(learnware_list=mixture_learnware_list) - reuse_predict = reuse_baseline.predict(user_data=test_x) - reuse_score = m5.score(test_y, reuse_predict) - print(f"mixture reuse loss: {reuse_score}\n") + print(f"mixture_score: {mixture_score}, mixture_learnware: {mixture_id}") + + reuse_job_selector = JobSelectorReuser(learnware_list=mixture_learnware_list) + job_selector_predict_y = reuse_job_selector.predict(user_data=test_x) + job_selector_score = m5.score(test_y, job_selector_predict_y) + print(f"mixture reuse loss (job selector): {job_selector_score}") + + reuse_ensemble = AveragingReuser(learnware_list=mixture_learnware_list) + ensemble_predict_y = reuse_ensemble.predict(user_data=test_x) + ensemble_score = m5.score(test_y, ensemble_predict_y) + print(f"mixture reuse loss (ensemble): {ensemble_score}\n") + + sinle_score_list.append(loss_list[0]) + random_score_list.append(np.mean(loss_list)) + job_selector_score_list.append(job_selector_score) + ensemble_score_list.append(ensemble_score) + + print(f"Single search score: {np.mean(sinle_score_list)}") + print(f"Job selector score: {np.mean(job_selector_score_list)}") + print(f"Average ensemble score: {np.mean(ensemble_score_list)}") + print(f"Random search score: {np.mean(random_score_list)}") if __name__ == "__main__":