[MNT] Update Image Ensemble

examples/example_image/main.py

@@ -4,7 +4,7 @@ from get_data import *
 import os
 import random
 from utils import generate_uploader, generate_user, ImageDataLoader, train, eval_prediction
-from learnware.learnware import Learnware, JobSelectorReuser
+from learnware.learnware import Learnware, JobSelectorReuser, EnsembleReuser
 import time
 
 from learnware.market import EasyMarket, BaseUserInfo
@@ -143,6 +143,7 @@ def test_search(gamma=0.1, load_market=True):
     avg_list = []
     improve_list = []
     job_selector_score_list = []
+    ensemble_score_list = []
     for i in range(n_users):
         user_data_path = os.path.join(user_save_root, "user_%d_X.npy" % (i))
         user_label_path = os.path.join(user_save_root, "user_%d_y.npy" % (i))
@@ -171,8 +172,14 @@ def test_search(gamma=0.1, load_market=True):
         reuse_predict = reuse_baseline.predict(user_data=user_data)
         reuse_score = eval_prediction(reuse_predict, user_label)
         job_selector_score_list.append(reuse_score)
-        """
         print(f"mixture reuse loss: {reuse_score}\n")
+        """
+
+        reuse_ensemble = EnsembleReuser(learnware_list=mixture_learnware_list, mode="vote")
+        ensemble_predict_y = reuse_ensemble.predict(user_data=user_data)
+        ensemble_score = eval_prediction(ensemble_predict_y, user_label)
+        ensemble_score_list.append(ensemble_score)
+        print(f"mixture reuse accuracy (ensemble): {ensemble_score}\n")
         select_list.append(acc_list[0])
         avg_list.append(np.mean(acc_list))
         improve_list.append((acc_list[0] - np.mean(acc_list)) / np.mean(acc_list))
@@ -182,6 +189,9 @@ def test_search(gamma=0.1, load_market=True):
     )
     logger.info("Average performance improvement: %.3f" % (np.mean(improve_list)))
     # logger.info("Average Job Selector Reuse Performance: %.3f +/- %.3f"%(np.mean(job_selector_score_list), np.std(job_selector_score_list)))
+    logger.info(
+        "Ensemble Reuse Performance: %.3f +/- %.3f" % (np.mean(ensemble_score_list), np.std(ensemble_score_list))
+    )
 
 
 if __name__ == "__main__":

examples/example_image/utils.py

@@ -162,9 +162,13 @@ def test(test_X, test_y, model, batch_size=128):
 
 def eval_prediction(pred_y, target_y):
     device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-    _, predicted = torch.max(pred_y.data, 1)
-    annos = torch.from_numpy(target_y).to(device)
-    total = annos.size(0)
+    if not isinstance(pred_y, np.ndarray):
+        pred_y = pred_y.detach().cpu().numpy()
+    predicted = np.argmax(pred_y, 1)
+    # print(predicted)
+    # annos = torch.from_numpy(target_y).to(device)
+    annos = target_y
+    total = annos.shape[0]
     correct = (predicted == annos).sum().item()
     criterion = nn.CrossEntropyLoss()
     return correct / total

learnware/learnware/reuse.py

@@ -9,6 +9,7 @@ import learnware.specification as specification
 from .base import BaseReuser
 from ..specification import RKMEStatSpecification
 from ..logger import get_module_logger
+from scipy.special import softmax
 
 logger = get_module_logger("Reuser")
 
@@ -228,7 +229,7 @@ class JobSelectorReuser(BaseReuser):
 class EnsembleReuser(BaseReuser):
     """Baseline Multiple Learnware Reuser uing Ensemble Method"""
 
-    def __init__(self, learnware_list: List[Learnware]):
+    def __init__(self, learnware_list: List[Learnware], mode="mean"):
         """The initialization method for ensemble reuser
 
         Parameters
@@ -237,6 +238,7 @@ class EnsembleReuser(BaseReuser):
             The learnware list, which should have RKME Specification for each learnweare
         """
         super(EnsembleReuser, self).__init__(learnware_list)
+        self.mode = mode
 
     def predict(self, user_data: np.ndarray) -> np.ndarray:
         """Give prediction for user data using baseline ensemble method
@@ -256,10 +258,20 @@ class EnsembleReuser(BaseReuser):
         for idx in range(len(self.learnware_list)):
             pred_y = self.learnware_list[idx].predict(user_data)
 
-            if mean_pred_y is None:
-                mean_pred_y = pred_y
-            else:
-                mean_pred_y += pred_y
+            if self.mode == "mean":
+                if mean_pred_y is None:
+                    mean_pred_y = pred_y
+                else:
+                    mean_pred_y += pred_y
+            elif self.mode == "vote":
+                # print(pred_y.shape)
+                if not isinstance(pred_y, np.ndarray):
+                    pred_y = pred_y.detach().cpu().numpy()
+                softmax_pred = softmax(pred_y, axis=1)
+                if mean_pred_y is None:
+                    mean_pred_y = softmax_pred
+                else:
+                    mean_pred_y += softmax_pred
 
         mean_pred_y /= len(self.learnware_list)