diff --git a/examples/example_image/main.py b/examples/example_image/main.py index a5a4a2e..432572d 100644 --- a/examples/example_image/main.py +++ b/examples/example_image/main.py @@ -169,24 +169,25 @@ def test_search(gamma=0.1, load_market=True): acc = eval_prediction(pred_y, user_label) acc_list.append(acc) logger.info("search rank: %d, score: %.3f, learnware_id: %s, acc: %.3f" % (idx, score, learnware.id, acc)) - # test reuse - """ - reuse_baseline = JobSelectorReuser(learnware_list=mixture_learnware_list) + # test reuse (job selector) + reuse_baseline = JobSelectorReuser(learnware_list=mixture_learnware_list, herding_num=100) reuse_predict = reuse_baseline.predict(user_data=user_data) reuse_score = eval_prediction(reuse_predict, user_label) job_selector_score_list.append(reuse_score) - print(f"mixture reuse loss: {reuse_score}\n") - """ + print(f"mixture reuse loss: {reuse_score}") + # test reuse (ensemble) reuse_ensemble = AveragingReuser(learnware_list=mixture_learnware_list, mode="vote") ensemble_predict_y = reuse_ensemble.predict(user_data=user_data) ensemble_score = eval_prediction(ensemble_predict_y, user_label) ensemble_score_list.append(ensemble_score) print(f"mixture reuse accuracy (ensemble): {ensemble_score}\n") + select_list.append(acc_list[0]) avg_list.append(np.mean(acc_list)) improve_list.append((acc_list[0] - np.mean(acc_list)) / np.mean(acc_list)) + logger.info( "Accuracy of selected learnware: %.3f +/- %.3f, Average performance: %.3f +/- %.3f" % (np.mean(select_list), np.std(select_list), np.mean(avg_list), np.std(avg_list)) diff --git a/examples/example_pfs/main.py b/examples/example_pfs/main.py index 95e2662..22a7a7a 100644 --- a/examples/example_pfs/main.py +++ b/examples/example_pfs/main.py @@ -47,7 +47,7 @@ class PFSDatasetWorkflow: def _init_learnware_market(self): """initialize learnware market""" learnware.init() - easy_market = EasyMarket(rebuild=True) + easy_market = EasyMarket(market_id="pfs") print("Total Item:", len(easy_market)) zip_path_list = [] @@ -116,7 +116,7 @@ class PFSDatasetWorkflow: self.prepare_learnware(regenerate_flag) self._init_learnware_market() - easy_market = EasyMarket() + easy_market = EasyMarket(market_id="pfs") print("Total Item:", len(easy_market)) pfs = Dataloader() diff --git a/examples/example_pfs/upload.py b/examples/example_pfs/upload.py index ed8449f..9719230 100644 --- a/examples/example_pfs/upload.py +++ b/examples/example_pfs/upload.py @@ -6,8 +6,8 @@ import json import time from tqdm import tqdm -email = "tanzh@lamda.nju.edu.cn" -password = hashlib.md5(b"Qwerty123").hexdigest() +email = "liujd@lamda.nju.edu.cn" +password = hashlib.md5(b"liujdlamda").hexdigest() login_url = "http://210.28.134.201:8089/auth/login" submit_url = "http://210.28.134.201:8089/user/add_learnware" all_data_type = ["Table", "Image", "Video", "Text", "Audio"] @@ -61,8 +61,8 @@ def main(): scenario = list(set(random.choices(all_scenario, k=5))) semantic_specification = { "Data": {"Values": ["Table"], "Type": "Class"}, + "Library": {"Values": ["Scikit-learn"], "Type": "Class"}, "Task": {"Values": ["Regression"], "Type": "Class"}, - "Device": {"Values": ["CPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": { "Values": "A sales-forecasting model from Predict Future Sales Competition on Kaggle", diff --git a/learnware/learnware/reuse.py b/learnware/learnware/reuse.py index 97ee44e..adc6ace 100644 --- a/learnware/learnware/reuse.py +++ b/learnware/learnware/reuse.py @@ -1,4 +1,6 @@ +import torch import numpy as np +import tensorflow as tf from typing import Tuple, Any, List, Union, Dict from cvxopt import matrix, solvers from lightgbm import LGBMClassifier @@ -45,24 +47,37 @@ class JobSelectorReuser(BaseReuser): Prediction given by job-selector method """ select_result = self.job_selector(user_data) - selector_pred_y = np.zeros(user_data.shape[0]) + pred_y_list = [] + data_idxs_list = [] for idx in range(len(self.learnware_list)): data_idx_list = np.where(select_result == idx)[0] if len(data_idx_list) > 0: - selector_pred_y[data_idx_list] = self.learnware_list[idx].predict(user_data[data_idx_list]) + pred_y = self.learnware_list[idx].predict(user_data[data_idx_list]) + if isinstance(pred_y, torch.Tensor): + pred_y = pred_y.detach().cpu().numpy() + elif isinstance(pred_y, tf.Tensor): + pred_y = pred_y.numpy() + + pred_y_list.append(pred_y) + data_idxs_list.append(data_idx_list) + + if pred_y_list[0].ndim == 1: + selector_pred_y = np.zeros(user_data.shape[0]) + else: + selector_pred_y = np.zeros((user_data.shape[0], pred_y_list[0].shape[1])) + for pred_y, data_idx_list in zip(pred_y_list, data_idxs_list): + selector_pred_y[data_idx_list] = pred_y return selector_pred_y - def job_selector(self, user_data: np.ndarray, use_herding: bool = True): + def job_selector(self, user_data: np.ndarray): """Train job selector based on user's data, which predicts which learnware in the pool should be selected Parameters ---------- user_data : np.ndarray User's labeled raw data. - use_herding: bool - Whether create job selector training samples by herding """ if len(self.learnware_list) == 1: user_data_num = user_data.shape[0] @@ -116,6 +131,13 @@ class JobSelectorReuser(BaseReuser): val_herding_y = np.array(val_herding_y) # use herding samples to train a job selector + herding_X = herding_X.reshape(herding_X.shape[0], -1) + train_herding_X = train_herding_X.reshape(train_herding_X.shape[0], -1) + val_herding_X = val_herding_X.reshape(val_herding_X.shape[0], -1) + herding_y = herding_y.astype(int) + train_herding_y = train_herding_y.astype(int) + val_herding_y = val_herding_y.astype(int) + job_selector = self._selector_grid_search( herding_X, herding_y, @@ -125,7 +147,7 @@ class JobSelectorReuser(BaseReuser): val_herding_y, len(self.learnware_list), ) - job_select_result = np.array(job_selector.predict(user_data)) + job_select_result = np.array(job_selector.predict(user_data.reshape(user_data.shape[0], -1))) return job_select_result @@ -155,7 +177,8 @@ class JobSelectorReuser(BaseReuser): A = matrix(np.ones((1, task_num))) b = matrix(np.ones((1, 1))) solvers.options["show_progress"] = False - sol = solvers.qp(P, q, G, h, A, b) + + sol = solvers.qp(P, q, G, h, A, b, kktsolver="ldl") task_mixture_weight = np.array(sol["x"]).reshape(-1) return task_mixture_weight @@ -266,6 +289,10 @@ class AveragingReuser(BaseReuser): for idx in range(len(self.learnware_list)): pred_y = self.learnware_list[idx].predict(user_data) + if isinstance(pred_y, torch.Tensor): + pred_y = pred_y.detach().cpu().numpy() + elif isinstance(pred_y, tf.Tensor): + pred_y = pred_y.numpy() if self.mode == "mean": if mean_pred_y is None: @@ -273,10 +300,7 @@ class AveragingReuser(BaseReuser): else: mean_pred_y += pred_y elif self.mode == "vote": - # print(pred_y.shape) - if not isinstance(pred_y, np.ndarray): - pred_y = pred_y.detach().cpu().numpy() - softmax_pred = softmax(pred_y, axis=0) + softmax_pred = softmax(pred_y, axis=1) if mean_pred_y is None: mean_pred_y = softmax_pred else: diff --git a/learnware/specification/rkme.py b/learnware/specification/rkme.py index a7a34b3..cdf84f7 100644 --- a/learnware/specification/rkme.py +++ b/learnware/specification/rkme.py @@ -262,7 +262,9 @@ class RKMEStatSpecification(BaseStatSpecification): sample_list = [] for i, n in Counter(np.array(sample_assign.cpu())).items(): for _ in range(n): - sample_list.append(torch.normal(mean=self.z[i], std=0.25).reshape(1, -1)) + sample_list.append( + torch.normal(mean=self.z[i].reshape(self.z[i].shape[0], -1), std=0.25).reshape(1, -1) + ) if len(sample_list) > 1: return torch.cat(sample_list, axis=0) elif len(sample_list) == 1: