From f41d0645db504cc7d3ea4b0ba9fc0bc739a26dd5 Mon Sep 17 00:00:00 2001 From: chenzx Date: Fri, 21 Apr 2023 15:33:08 +0800 Subject: [PATCH] [MNT] Update Image Ensemble --- examples/example_image/main.py | 14 ++++++++++++-- examples/example_image/utils.py | 10 +++++++--- learnware/learnware/reuse.py | 22 +++++++++++++++++----- 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/examples/example_image/main.py b/examples/example_image/main.py index 097da01..82ec117 100644 --- a/examples/example_image/main.py +++ b/examples/example_image/main.py @@ -4,7 +4,7 @@ from get_data import * import os import random from utils import generate_uploader, generate_user, ImageDataLoader, train, eval_prediction -from learnware.learnware import Learnware, JobSelectorReuser +from learnware.learnware import Learnware, JobSelectorReuser, EnsembleReuser import time from learnware.market import EasyMarket, BaseUserInfo @@ -143,6 +143,7 @@ def test_search(gamma=0.1, load_market=True): avg_list = [] improve_list = [] job_selector_score_list = [] + ensemble_score_list = [] for i in range(n_users): user_data_path = os.path.join(user_save_root, "user_%d_X.npy" % (i)) user_label_path = os.path.join(user_save_root, "user_%d_y.npy" % (i)) @@ -171,8 +172,14 @@ def test_search(gamma=0.1, load_market=True): reuse_predict = reuse_baseline.predict(user_data=user_data) reuse_score = eval_prediction(reuse_predict, user_label) job_selector_score_list.append(reuse_score) - """ print(f"mixture reuse loss: {reuse_score}\n") + """ + + reuse_ensemble = EnsembleReuser(learnware_list=mixture_learnware_list, mode="vote") + ensemble_predict_y = reuse_ensemble.predict(user_data=user_data) + ensemble_score = eval_prediction(ensemble_predict_y, user_label) + ensemble_score_list.append(ensemble_score) + print(f"mixture reuse accuracy (ensemble): {ensemble_score}\n") select_list.append(acc_list[0]) avg_list.append(np.mean(acc_list)) improve_list.append((acc_list[0] - np.mean(acc_list)) / np.mean(acc_list)) @@ -182,6 +189,9 @@ def test_search(gamma=0.1, load_market=True): ) logger.info("Average performance improvement: %.3f" % (np.mean(improve_list))) # logger.info("Average Job Selector Reuse Performance: %.3f +/- %.3f"%(np.mean(job_selector_score_list), np.std(job_selector_score_list))) + logger.info( + "Ensemble Reuse Performance: %.3f +/- %.3f" % (np.mean(ensemble_score_list), np.std(ensemble_score_list)) + ) if __name__ == "__main__": diff --git a/examples/example_image/utils.py b/examples/example_image/utils.py index 392dac9..b7f5056 100644 --- a/examples/example_image/utils.py +++ b/examples/example_image/utils.py @@ -162,9 +162,13 @@ def test(test_X, test_y, model, batch_size=128): def eval_prediction(pred_y, target_y): device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - _, predicted = torch.max(pred_y.data, 1) - annos = torch.from_numpy(target_y).to(device) - total = annos.size(0) + if not isinstance(pred_y, np.ndarray): + pred_y = pred_y.detach().cpu().numpy() + predicted = np.argmax(pred_y, 1) + # print(predicted) + # annos = torch.from_numpy(target_y).to(device) + annos = target_y + total = annos.shape[0] correct = (predicted == annos).sum().item() criterion = nn.CrossEntropyLoss() return correct / total diff --git a/learnware/learnware/reuse.py b/learnware/learnware/reuse.py index 0b49f10..8207b21 100644 --- a/learnware/learnware/reuse.py +++ b/learnware/learnware/reuse.py @@ -9,6 +9,7 @@ import learnware.specification as specification from .base import BaseReuser from ..specification import RKMEStatSpecification from ..logger import get_module_logger +from scipy.special import softmax logger = get_module_logger("Reuser") @@ -228,7 +229,7 @@ class JobSelectorReuser(BaseReuser): class EnsembleReuser(BaseReuser): """Baseline Multiple Learnware Reuser uing Ensemble Method""" - def __init__(self, learnware_list: List[Learnware]): + def __init__(self, learnware_list: List[Learnware], mode="mean"): """The initialization method for ensemble reuser Parameters @@ -237,6 +238,7 @@ class EnsembleReuser(BaseReuser): The learnware list, which should have RKME Specification for each learnweare """ super(EnsembleReuser, self).__init__(learnware_list) + self.mode = mode def predict(self, user_data: np.ndarray) -> np.ndarray: """Give prediction for user data using baseline ensemble method @@ -256,10 +258,20 @@ class EnsembleReuser(BaseReuser): for idx in range(len(self.learnware_list)): pred_y = self.learnware_list[idx].predict(user_data) - if mean_pred_y is None: - mean_pred_y = pred_y - else: - mean_pred_y += pred_y + if self.mode == "mean": + if mean_pred_y is None: + mean_pred_y = pred_y + else: + mean_pred_y += pred_y + elif self.mode == "vote": + # print(pred_y.shape) + if not isinstance(pred_y, np.ndarray): + pred_y = pred_y.detach().cpu().numpy() + softmax_pred = softmax(pred_y, axis=1) + if mean_pred_y is None: + mean_pred_y = softmax_pred + else: + mean_pred_y += softmax_pred mean_pred_y /= len(self.learnware_list)