From 891c83d607d720164ee111f171c82b4aac3a18e3 Mon Sep 17 00:00:00 2001 From: Gene Date: Sat, 22 Apr 2023 14:43:09 +0800 Subject: [PATCH] [FIX] Fix bugs in Reuser --- examples/example_image/main.py | 13 ++++++---- examples/example_pfs/main.py | 4 +-- examples/example_pfs/upload.py | 6 ++--- learnware/learnware/reuse.py | 46 +++++++++++++++++++++++++-------- learnware/specification/rkme.py | 4 ++- 5 files changed, 51 insertions(+), 22 deletions(-) diff --git a/examples/example_image/main.py b/examples/example_image/main.py index 757fe44..52f125b 100644 --- a/examples/example_image/main.py +++ b/examples/example_image/main.py @@ -110,7 +110,7 @@ def prepare_learnware(data_path, model_path, init_file_path, yaml_path, save_roo def prepare_market(): - image_market = EasyMarket(rebuild=True) + image_market = EasyMarket(market_id="image") rmtree(learnware_pool_dir) os.makedirs(learnware_pool_dir, exist_ok=True) for i in range(n_uploaders): @@ -166,22 +166,25 @@ def test_search(gamma=0.1, load_market=True): acc = eval_prediction(pred_y, user_label) acc_list.append(acc) logger.info("search rank: %d, score: %.3f, learnware_id: %s, acc: %.3f" % (idx, score, learnware.id, acc)) - # test reuse - reuse_baseline = JobSelectorReuser(learnware_list=mixture_learnware_list) + # test reuse (job selector) + reuse_baseline = JobSelectorReuser(learnware_list=mixture_learnware_list, herding_num=100) reuse_predict = reuse_baseline.predict(user_data=user_data) reuse_score = eval_prediction(reuse_predict, user_label) job_selector_score_list.append(reuse_score) - print(f"mixture reuse loss: {reuse_score}\n") + print(f"mixture reuse loss: {reuse_score}") + # test reuse (ensemble) reuse_ensemble = AveragingReuser(learnware_list=mixture_learnware_list, mode="vote") ensemble_predict_y = reuse_ensemble.predict(user_data=user_data) ensemble_score = eval_prediction(ensemble_predict_y, user_label) ensemble_score_list.append(ensemble_score) print(f"mixture reuse accuracy (ensemble): {ensemble_score}\n") + select_list.append(acc_list[0]) avg_list.append(np.mean(acc_list)) improve_list.append((acc_list[0] - np.mean(acc_list)) / np.mean(acc_list)) + logger.info( "Accuracy of selected learnware: %.3f +/- %.3f, Average performance: %.3f +/- %.3f" % (np.mean(select_list), np.std(select_list), np.mean(avg_list), np.std(avg_list)) @@ -199,4 +202,4 @@ def test_search(gamma=0.1, load_market=True): if __name__ == "__main__": # prepare_data() # prepare_model() - test_search(load_market=False) + test_search(load_market=True) diff --git a/examples/example_pfs/main.py b/examples/example_pfs/main.py index 95e2662..22a7a7a 100644 --- a/examples/example_pfs/main.py +++ b/examples/example_pfs/main.py @@ -47,7 +47,7 @@ class PFSDatasetWorkflow: def _init_learnware_market(self): """initialize learnware market""" learnware.init() - easy_market = EasyMarket(rebuild=True) + easy_market = EasyMarket(market_id="pfs") print("Total Item:", len(easy_market)) zip_path_list = [] @@ -116,7 +116,7 @@ class PFSDatasetWorkflow: self.prepare_learnware(regenerate_flag) self._init_learnware_market() - easy_market = EasyMarket() + easy_market = EasyMarket(market_id="pfs") print("Total Item:", len(easy_market)) pfs = Dataloader() diff --git a/examples/example_pfs/upload.py b/examples/example_pfs/upload.py index ed8449f..9719230 100644 --- a/examples/example_pfs/upload.py +++ b/examples/example_pfs/upload.py @@ -6,8 +6,8 @@ import json import time from tqdm import tqdm -email = "tanzh@lamda.nju.edu.cn" -password = hashlib.md5(b"Qwerty123").hexdigest() +email = "liujd@lamda.nju.edu.cn" +password = hashlib.md5(b"liujdlamda").hexdigest() login_url = "http://210.28.134.201:8089/auth/login" submit_url = "http://210.28.134.201:8089/user/add_learnware" all_data_type = ["Table", "Image", "Video", "Text", "Audio"] @@ -61,8 +61,8 @@ def main(): scenario = list(set(random.choices(all_scenario, k=5))) semantic_specification = { "Data": {"Values": ["Table"], "Type": "Class"}, + "Library": {"Values": ["Scikit-learn"], "Type": "Class"}, "Task": {"Values": ["Regression"], "Type": "Class"}, - "Device": {"Values": ["CPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": { "Values": "A sales-forecasting model from Predict Future Sales Competition on Kaggle", diff --git a/learnware/learnware/reuse.py b/learnware/learnware/reuse.py index 97ee44e..adc6ace 100644 --- a/learnware/learnware/reuse.py +++ b/learnware/learnware/reuse.py @@ -1,4 +1,6 @@ +import torch import numpy as np +import tensorflow as tf from typing import Tuple, Any, List, Union, Dict from cvxopt import matrix, solvers from lightgbm import LGBMClassifier @@ -45,24 +47,37 @@ class JobSelectorReuser(BaseReuser): Prediction given by job-selector method """ select_result = self.job_selector(user_data) - selector_pred_y = np.zeros(user_data.shape[0]) + pred_y_list = [] + data_idxs_list = [] for idx in range(len(self.learnware_list)): data_idx_list = np.where(select_result == idx)[0] if len(data_idx_list) > 0: - selector_pred_y[data_idx_list] = self.learnware_list[idx].predict(user_data[data_idx_list]) + pred_y = self.learnware_list[idx].predict(user_data[data_idx_list]) + if isinstance(pred_y, torch.Tensor): + pred_y = pred_y.detach().cpu().numpy() + elif isinstance(pred_y, tf.Tensor): + pred_y = pred_y.numpy() + + pred_y_list.append(pred_y) + data_idxs_list.append(data_idx_list) + + if pred_y_list[0].ndim == 1: + selector_pred_y = np.zeros(user_data.shape[0]) + else: + selector_pred_y = np.zeros((user_data.shape[0], pred_y_list[0].shape[1])) + for pred_y, data_idx_list in zip(pred_y_list, data_idxs_list): + selector_pred_y[data_idx_list] = pred_y return selector_pred_y - def job_selector(self, user_data: np.ndarray, use_herding: bool = True): + def job_selector(self, user_data: np.ndarray): """Train job selector based on user's data, which predicts which learnware in the pool should be selected Parameters ---------- user_data : np.ndarray User's labeled raw data. - use_herding: bool - Whether create job selector training samples by herding """ if len(self.learnware_list) == 1: user_data_num = user_data.shape[0] @@ -116,6 +131,13 @@ class JobSelectorReuser(BaseReuser): val_herding_y = np.array(val_herding_y) # use herding samples to train a job selector + herding_X = herding_X.reshape(herding_X.shape[0], -1) + train_herding_X = train_herding_X.reshape(train_herding_X.shape[0], -1) + val_herding_X = val_herding_X.reshape(val_herding_X.shape[0], -1) + herding_y = herding_y.astype(int) + train_herding_y = train_herding_y.astype(int) + val_herding_y = val_herding_y.astype(int) + job_selector = self._selector_grid_search( herding_X, herding_y, @@ -125,7 +147,7 @@ class JobSelectorReuser(BaseReuser): val_herding_y, len(self.learnware_list), ) - job_select_result = np.array(job_selector.predict(user_data)) + job_select_result = np.array(job_selector.predict(user_data.reshape(user_data.shape[0], -1))) return job_select_result @@ -155,7 +177,8 @@ class JobSelectorReuser(BaseReuser): A = matrix(np.ones((1, task_num))) b = matrix(np.ones((1, 1))) solvers.options["show_progress"] = False - sol = solvers.qp(P, q, G, h, A, b) + + sol = solvers.qp(P, q, G, h, A, b, kktsolver="ldl") task_mixture_weight = np.array(sol["x"]).reshape(-1) return task_mixture_weight @@ -266,6 +289,10 @@ class AveragingReuser(BaseReuser): for idx in range(len(self.learnware_list)): pred_y = self.learnware_list[idx].predict(user_data) + if isinstance(pred_y, torch.Tensor): + pred_y = pred_y.detach().cpu().numpy() + elif isinstance(pred_y, tf.Tensor): + pred_y = pred_y.numpy() if self.mode == "mean": if mean_pred_y is None: @@ -273,10 +300,7 @@ class AveragingReuser(BaseReuser): else: mean_pred_y += pred_y elif self.mode == "vote": - # print(pred_y.shape) - if not isinstance(pred_y, np.ndarray): - pred_y = pred_y.detach().cpu().numpy() - softmax_pred = softmax(pred_y, axis=0) + softmax_pred = softmax(pred_y, axis=1) if mean_pred_y is None: mean_pred_y = softmax_pred else: diff --git a/learnware/specification/rkme.py b/learnware/specification/rkme.py index a7a34b3..cdf84f7 100644 --- a/learnware/specification/rkme.py +++ b/learnware/specification/rkme.py @@ -262,7 +262,9 @@ class RKMEStatSpecification(BaseStatSpecification): sample_list = [] for i, n in Counter(np.array(sample_assign.cpu())).items(): for _ in range(n): - sample_list.append(torch.normal(mean=self.z[i], std=0.25).reshape(1, -1)) + sample_list.append( + torch.normal(mean=self.z[i].reshape(self.z[i].shape[0], -1), std=0.25).reshape(1, -1) + ) if len(sample_list) > 1: return torch.cat(sample_list, axis=0) elif len(sample_list) == 1: