[MNT]Update easy market

examples/example_image/main.py

@@ -169,24 +169,25 @@ def test_search(gamma=0.1, load_market=True):
             acc = eval_prediction(pred_y, user_label)
             acc_list.append(acc)
             logger.info("search rank: %d, score: %.3f, learnware_id: %s, acc: %.3f" % (idx, score, learnware.id, acc))
-        # test reuse
 
-        """
-        reuse_baseline = JobSelectorReuser(learnware_list=mixture_learnware_list)
+        # test reuse (job selector)
+        reuse_baseline = JobSelectorReuser(learnware_list=mixture_learnware_list, herding_num=100)
         reuse_predict = reuse_baseline.predict(user_data=user_data)
         reuse_score = eval_prediction(reuse_predict, user_label)
         job_selector_score_list.append(reuse_score)
-        print(f"mixture reuse loss: {reuse_score}\n")
-        """
+        print(f"mixture reuse loss: {reuse_score}")
 
+        # test reuse (ensemble)
         reuse_ensemble = AveragingReuser(learnware_list=mixture_learnware_list, mode="vote")
         ensemble_predict_y = reuse_ensemble.predict(user_data=user_data)
         ensemble_score = eval_prediction(ensemble_predict_y, user_label)
         ensemble_score_list.append(ensemble_score)
         print(f"mixture reuse accuracy (ensemble): {ensemble_score}\n")
+
         select_list.append(acc_list[0])
         avg_list.append(np.mean(acc_list))
         improve_list.append((acc_list[0] - np.mean(acc_list)) / np.mean(acc_list))
+
     logger.info(
         "Accuracy of selected learnware: %.3f +/- %.3f, Average performance: %.3f +/- %.3f"
         % (np.mean(select_list), np.std(select_list), np.mean(avg_list), np.std(avg_list))

examples/example_pfs/main.py

@@ -47,7 +47,7 @@ class PFSDatasetWorkflow:
     def _init_learnware_market(self):
         """initialize learnware market"""
         learnware.init()
-        easy_market = EasyMarket(rebuild=True)
+        easy_market = EasyMarket(market_id="pfs")
         print("Total Item:", len(easy_market))
 
         zip_path_list = []
@@ -116,7 +116,7 @@ class PFSDatasetWorkflow:
         self.prepare_learnware(regenerate_flag)
         self._init_learnware_market()
 
-        easy_market = EasyMarket()
+        easy_market = EasyMarket(market_id="pfs")
         print("Total Item:", len(easy_market))
 
         pfs = Dataloader()

examples/example_pfs/upload.py

@@ -6,8 +6,8 @@ import json
 import time
 from tqdm import tqdm
 
-email = "tanzh@lamda.nju.edu.cn"
-password = hashlib.md5(b"Qwerty123").hexdigest()
+email = "liujd@lamda.nju.edu.cn"
+password = hashlib.md5(b"liujdlamda").hexdigest()
 login_url = "http://210.28.134.201:8089/auth/login"
 submit_url = "http://210.28.134.201:8089/user/add_learnware"
 all_data_type = ["Table", "Image", "Video", "Text", "Audio"]
@@ -61,8 +61,8 @@ def main():
         scenario = list(set(random.choices(all_scenario, k=5)))
         semantic_specification = {
             "Data": {"Values": ["Table"], "Type": "Class"},
+            "Library": {"Values": ["Scikit-learn"], "Type": "Class"},
             "Task": {"Values": ["Regression"], "Type": "Class"},
-            "Device": {"Values": ["CPU"], "Type": "Tag"},
             "Scenario": {"Values": ["Business"], "Type": "Tag"},
             "Description": {
                 "Values": "A sales-forecasting model from Predict Future Sales Competition on Kaggle",

learnware/learnware/reuse.py

@@ -1,4 +1,6 @@
+import torch
 import numpy as np
+import tensorflow as tf
 from typing import Tuple, Any, List, Union, Dict
 from cvxopt import matrix, solvers
 from lightgbm import LGBMClassifier
@@ -45,24 +47,37 @@ class JobSelectorReuser(BaseReuser):
             Prediction given by job-selector method
         """
         select_result = self.job_selector(user_data)
-        selector_pred_y = np.zeros(user_data.shape[0])
+        pred_y_list = []
+        data_idxs_list = []
 
         for idx in range(len(self.learnware_list)):
             data_idx_list = np.where(select_result == idx)[0]
             if len(data_idx_list) > 0:
-                selector_pred_y[data_idx_list] = self.learnware_list[idx].predict(user_data[data_idx_list])
+                pred_y = self.learnware_list[idx].predict(user_data[data_idx_list])
+                if isinstance(pred_y, torch.Tensor):
+                    pred_y = pred_y.detach().cpu().numpy()
+                elif isinstance(pred_y, tf.Tensor):
+                    pred_y = pred_y.numpy()
+
+                pred_y_list.append(pred_y)
+                data_idxs_list.append(data_idx_list)
+
+        if pred_y_list[0].ndim == 1:
+            selector_pred_y = np.zeros(user_data.shape[0])
+        else:
+            selector_pred_y = np.zeros((user_data.shape[0], pred_y_list[0].shape[1]))
+        for pred_y, data_idx_list in zip(pred_y_list, data_idxs_list):
+            selector_pred_y[data_idx_list] = pred_y
 
         return selector_pred_y
 
-    def job_selector(self, user_data: np.ndarray, use_herding: bool = True):
+    def job_selector(self, user_data: np.ndarray):
         """Train job selector based on user's data, which predicts which learnware in the pool should be selected
 
         Parameters
         ----------
         user_data : np.ndarray
             User's labeled raw data.
-        use_herding: bool
-            Whether create job selector training samples by herding
         """
         if len(self.learnware_list) == 1:
             user_data_num = user_data.shape[0]
@@ -116,6 +131,13 @@ class JobSelectorReuser(BaseReuser):
             val_herding_y = np.array(val_herding_y)
 
             # use herding samples to train a job selector
+            herding_X = herding_X.reshape(herding_X.shape[0], -1)
+            train_herding_X = train_herding_X.reshape(train_herding_X.shape[0], -1)
+            val_herding_X = val_herding_X.reshape(val_herding_X.shape[0], -1)
+            herding_y = herding_y.astype(int)
+            train_herding_y = train_herding_y.astype(int)
+            val_herding_y = val_herding_y.astype(int)
+
             job_selector = self._selector_grid_search(
                 herding_X,
                 herding_y,
@@ -125,7 +147,7 @@ class JobSelectorReuser(BaseReuser):
                 val_herding_y,
                 len(self.learnware_list),
             )
-            job_select_result = np.array(job_selector.predict(user_data))
+            job_select_result = np.array(job_selector.predict(user_data.reshape(user_data.shape[0], -1)))
 
             return job_select_result
 
@@ -155,7 +177,8 @@ class JobSelectorReuser(BaseReuser):
         A = matrix(np.ones((1, task_num)))
         b = matrix(np.ones((1, 1)))
         solvers.options["show_progress"] = False
-        sol = solvers.qp(P, q, G, h, A, b)
+
+        sol = solvers.qp(P, q, G, h, A, b, kktsolver="ldl")
         task_mixture_weight = np.array(sol["x"]).reshape(-1)
 
         return task_mixture_weight
@@ -266,6 +289,10 @@ class AveragingReuser(BaseReuser):
 
         for idx in range(len(self.learnware_list)):
             pred_y = self.learnware_list[idx].predict(user_data)
+            if isinstance(pred_y, torch.Tensor):
+                pred_y = pred_y.detach().cpu().numpy()
+            elif isinstance(pred_y, tf.Tensor):
+                pred_y = pred_y.numpy()
 
             if self.mode == "mean":
                 if mean_pred_y is None:
@@ -273,10 +300,7 @@ class AveragingReuser(BaseReuser):
                 else:
                     mean_pred_y += pred_y
             elif self.mode == "vote":
-                # print(pred_y.shape)
-                if not isinstance(pred_y, np.ndarray):
-                    pred_y = pred_y.detach().cpu().numpy()
-                softmax_pred = softmax(pred_y, axis=0)
+                softmax_pred = softmax(pred_y, axis=1)
                 if mean_pred_y is None:
                     mean_pred_y = softmax_pred
                 else:

learnware/specification/rkme.py

@@ -262,7 +262,9 @@ class RKMEStatSpecification(BaseStatSpecification):
         sample_list = []
         for i, n in Counter(np.array(sample_assign.cpu())).items():
             for _ in range(n):
-                sample_list.append(torch.normal(mean=self.z[i], std=0.25).reshape(1, -1))
+                sample_list.append(
+                    torch.normal(mean=self.z[i].reshape(self.z[i].shape[0], -1), std=0.25).reshape(1, -1)
+                )
         if len(sample_list) > 1:
             return torch.cat(sample_list, axis=0)
         elif len(sample_list) == 1: