diff --git a/examples/dataset_cifar_workflow/benchmarks/dataset/utils.py b/examples/dataset_cifar_workflow/benchmarks/dataset/utils.py index 1708cea..2c6231a 100644 --- a/examples/dataset_cifar_workflow/benchmarks/dataset/utils.py +++ b/examples/dataset_cifar_workflow/benchmarks/dataset/utils.py @@ -6,6 +6,8 @@ import torch import torchvision from torch.utils.data import TensorDataset, Dataset, DataLoader +from learnware.utils import choose_device + torchvision.disable_beta_transforms_warning() from torchvision.transforms import transforms, v2 @@ -49,10 +51,11 @@ def split_dataset(labels, size, split="uploader", order=None): def build_zca_matrix(X, reg_coef=0.1): X = (X - torch.mean(X, [0, 2, 3], keepdim=True)) / (torch.std(X, [0, 2, 3], keepdim=True)) + device = choose_device(0) X_flat = X.reshape(X.shape[0], -1) cov = (X_flat.T @ X_flat) / X_flat.shape[0] reg_amount = reg_coef * torch.trace(cov) / cov.shape[0] - u, s, _ = torch.svd(cov.cuda() + reg_amount * torch.eye(cov.shape[0]).cuda()) + u, s, _ = torch.svd(cov.to(device) + reg_amount * torch.eye(cov.shape[0]).to(device)) inv_sqrt_zca_eigs = s ** (-0.5) whitening_transform = torch.einsum( 'ij,j,kj->ik', u, inv_sqrt_zca_eigs, u) diff --git a/examples/dataset_cifar_workflow/benchmarks/utils.py b/examples/dataset_cifar_workflow/benchmarks/utils.py index faedd04..085258d 100644 --- a/examples/dataset_cifar_workflow/benchmarks/utils.py +++ b/examples/dataset_cifar_workflow/benchmarks/utils.py @@ -3,6 +3,8 @@ import os import zipfile from collections import defaultdict from shutil import rmtree + +from matplotlib import pyplot as plt from tabulate import tabulate import numpy as np @@ -127,7 +129,8 @@ def build_learnware(name: str, market: LearnwareMarket, order, model_name="conv" return model def train_model(model: nn.Module, train_set: Dataset, valid_set: Dataset, - save_path: str, epochs=35, batch_size=128, device=None): + save_path: str, epochs=35, batch_size=128, + device=None, verbose=True): device = choose_device(0) if device is None else device model.train() @@ -160,12 +163,14 @@ def train_model(model: nn.Module, train_set: Dataset, valid_set: Dataset, best_loss = valid_loss torch.save(model.state_dict(), save_path) - print("Epoch: {}, Valid Best Accuracy: {:.3f}% ({:.3f})".format(epoch+1, valid_acc, valid_loss)) + if verbose: + print("Epoch: {}, Valid Best Accuracy: {:.3f}% ({:.3f})".format(epoch+1, valid_acc, valid_loss)) if valid_acc > 99.0: - print("Early Stopping at 99% !") + if verbose: + print("Early Stopping at 99% !") break - if (epoch + 1) % 5 == 0: + if verbose and (epoch + 1) % 5 == 0: print('Epoch: {}, Train Average Loss: {:.3f}, Accuracy {:.3f}%, Valid Average Loss: {:.3f}'.format( epoch+1, np.mean(running_loss), train_acc, valid_loss)) @@ -214,10 +219,53 @@ class Recorder: return str(tabulate(table, headers=["Case"] + self.headers, tablefmt='orgtbl')) + def __getitem__(self, item): + return [[x[item] for x in v] for k, v in self.data.items()] + def save(self, path): with open(path, "w") as f: json.dump(self.data, f) def load(self, path): with open(path, "r") as f: - self.data = json.load(f) \ No newline at end of file + self.data = json.load(f) + + +def plot_labeled_performance_curves(name, user_mat, pruning_mat, n_labeled_list, save_path=None): + plt.figure(figsize=(10, 6)) + plt.xticks(range(len(n_labeled_list)), n_labeled_list) + + mats = [user_mat, pruning_mat] + + styles = [ + {"color": "navy", "linestyle": "-", "marker": "o"}, + {"color": "magenta", "linestyle": "-.", "marker": "d"}, + ] + + labels = [ + "User Model", + "Multiple Learnware Reuse (EnsemblePrune)", + ] + + for mat, style, label in zip(mats, styles, labels): + array_mat = 1 - np.asarray(mat) / 100 + mean_curve, std_curve = np.mean(array_mat, axis=1), np.std(array_mat, axis=1) + plt.plot(mean_curve, **style, label=label) + plt.fill_between( + range(len(n_labeled_list)), + mean_curve - 0.5 * std_curve, + mean_curve + 0.5 * std_curve, + color=style["color"], + alpha=0.2, + ) + + plt.xlabel("Labeled Data Size") + plt.ylabel("1 - Accuracy") + plt.title(f"{name} Homo Limited Labeled Data") + plt.legend() + plt.tight_layout() + if save_path: + plt.savefig( + save_path, bbox_inches="tight", dpi=600 + ) + plt.show() diff --git a/examples/dataset_cifar_workflow/main.py b/examples/dataset_cifar_workflow/main.py index 6fcbc0f..1de6796 100644 --- a/examples/dataset_cifar_workflow/main.py +++ b/examples/dataset_cifar_workflow/main.py @@ -1,24 +1,30 @@ import os +from datetime import datetime import fire import numpy as np +import tqdm from numpy import mean -from torch.utils.data import DataLoader +import torch +from torch.utils.data import DataLoader, TensorDataset import learnware -from benchmarks.utils import build_learnware, build_specification, evaluate, Recorder +from benchmarks.utils import * +from benchmarks.dataset.data import faster_train, uploader_data +from benchmarks.models.conv import ConvModel from learnware.client import LearnwareClient from learnware.market import instantiate_learnware_market, BaseUserInfo -from learnware.reuse import JobSelectorReuser, AveragingReuser -from learnware.specification import generate_rkme_image_spec +from learnware.reuse import JobSelectorReuser, AveragingReuser, EnsemblePruningReuser +from learnware.utils import choose_device + +PROXY_IP = "172.27.138.61" +os.environ["HTTP_PROXY"] = "http://" + PROXY_IP + ":7890" +os.environ["HTTPS_PROXY"] = "http://" + PROXY_IP + ":7890" -PROXY_IP = "172.24.57.111" -os.environ["HTTP_PROXY"] = "http://"+PROXY_IP+":7890" -os.environ["HTTPS_PROXY"] = "http://"+PROXY_IP+":7890" class CifarDatasetWorkflow: - def prepare_learnware(self, market_size=50, market_id=None, rebuild=False): + def prepare(self, market_size=50, market_id=None, rebuild=False, faster=True): """initialize learnware market""" learnware.init() assert not rebuild @@ -29,14 +35,17 @@ class CifarDatasetWorkflow: print("Using market_id", market_id) market = instantiate_learnware_market(name="easy", market_id=market_id, rebuild=rebuild) + device = choose_device(0) + if faster: + faster_train(device) for i, order in enumerate(orders[len(market):]): - print("=" * 20 + "learnware {}".format(i) + "=" * 20) + print("=" * 20 + "learnware {}".format(len(market)) + "=" * 20) print("order:", order) - build_learnware("cifar10", market, order) + build_learnware("cifar10", market, order, device=device) print("Total Item:", len(market)) - def evaluate_unlabeled(self, user_size=100, market_id=None): + def evaluate(self, user_size=100, market_id=None, faster=True): learnware.init() market_id = "dataset_cifar_workflow" if market_id is None else market_id @@ -45,23 +54,24 @@ class CifarDatasetWorkflow: print("Using market_id", market_id) market = instantiate_learnware_market(name="easy", market_id=market_id, rebuild=False) - top_1_acc_record, ensemble_acc_record, best_acc_record, mean_acc_record = [], [], [], [] - top_1_loss_record, ensemble_loss_record, best_loss_record, mean_loss_record = [], [], [], [] - - recorder = Recorder() + device = choose_device(0) + if faster: + faster_train(device) + unlabeled = Recorder(["Accuracy", "Loss"], ["{:.3f}% ± {:.3f}%", "{:.3f} ± {:.3f}"]) + labeled = Recorder(["Training", "Pruning"], ["{:.3f}% ± {:.3f}%", "{:.3f}% ± {:.3f}%"]) for i, order in enumerate(orders): print("=" * 20 + "user {}".format(i) + "=" * 20) print("order:", order) user_spec, dataset = build_specification("cifar10", i, order) user_info = BaseUserInfo(semantic_spec=LearnwareClient.create_semantic_specification( - self=None, - description="For Cifar Dataset Workflow", - data_type="Image", - task_type="Classification", - library_type="PyTorch", - scenarios=["Computer"], - output_description={"Dimension": 10, "Description": {str(i): "i" for i in range(10)}}), + self=None, + description="For Cifar Dataset Workflow", + data_type="Image", + task_type="Classification", + library_type="PyTorch", + scenarios=["Computer"], + output_description={"Dimension": 10, "Description": {str(i): "i" for i in range(10)}}), stat_info={"RKMEImageSpecification": user_spec}) search_result = market.search_learnware(user_info) @@ -73,22 +83,61 @@ class CifarDatasetWorkflow: loss, acc = evaluate(item, dataset) loss_list.append(loss) acc_list.append(acc) - recorder.record("Best", accuracy=max(acc_list), loss=min(loss_list)) - recorder.record("Average", accuracy=mean(acc_list), loss=mean(loss_list)) + unlabeled.record("Best", max(acc_list), min(loss_list)) + unlabeled.record("Average", mean(acc_list), mean(loss_list)) - top_1_loss, top_1_acc = evaluate(single_result[0].learnware, dataset) - recorder.record("Top-1 Learnware", accuracy=top_1_acc, loss=top_1_loss) + top_1_loss, top_1_acc = evaluate(single_result[0].learnware, dataset) + unlabeled.record("Top-1 Learnware", top_1_acc, top_1_loss) reuse_ensemble = AveragingReuser(learnware_list=multiple_result[0].learnwares, mode="vote_by_prob") - # reuse_ensemble = AveragingReuser(learnware_list=[item.learnware for item in single_result[:3]], mode="vote_by_prob") ensemble_loss, ensemble_acc = evaluate(reuse_ensemble, dataset) - recorder.record("Voting Reuse", accuracy=ensemble_acc, loss=ensemble_loss) + unlabeled.record("Voting Reuse", ensemble_acc, ensemble_loss) reuse_job_selector = JobSelectorReuser(learnware_list=multiple_result[0].learnwares, use_herding=False) job_loss, job_acc = evaluate(reuse_job_selector, dataset) - recorder.record("Job Selector", accuracy=job_acc, loss=job_loss) - - print(recorder.summary()) + unlabeled.record("Job Selector", job_acc, job_loss) + + train_set, valid_set, spec_set, order = uploader_data(order=order) + for labeled_size in tqdm.tqdm([100, 200, 500, 1000, 2000, 4000, 6000, 8000, 10000]): + loader = DataLoader(train_set, batch_size=labeled_size, shuffle=True) + X, y = next(iter(loader)) + + sampled_dataset = TensorDataset(X, y) + mode_save_path = os.path.abspath(os.path.join(__file__, "..", "cache", "model.pth")) + model = ConvModel(channel=X.shape[1], im_size=(X.shape[2], X.shape[3]), + n_random_features=10).to(device) + train_model(model, sampled_dataset, sampled_dataset, mode_save_path, + epochs=35, batch_size=128, device=device, verbose=False) + model.load_state_dict(torch.load(mode_save_path)) + _, train_acc = evaluate(model, dataset, distribution=True) + + ensemble_pruning = EnsemblePruningReuser(learnware_list=multiple_result[0].learnwares) + ensemble_pruning.fit(val_X=X, val_y=y) + _, pruning_acc = evaluate(ensemble_pruning, dataset, distribution=False) + + labeled.record("{:d}".format(labeled_size), train_acc, pruning_acc) + + print(unlabeled.summary()) + print(labeled.summary()) + + # Save recorder + current_time = datetime.now() + formatted_time = current_time.strftime("%Y-%m-%d_%H-%M-%S") + log_dir = os.path.abspath(os.path.join(__file__, "..", "log", formatted_time)) + os.makedirs(log_dir, exist_ok=True) + unlabeled.save(os.path.join(log_dir, "unlabeled.json")) + labeled.save(os.path.join(log_dir, "labeled.json")) + + def plot(self, record_dir): + unlabeled = Recorder(["Accuracy", "Loss"], ["{:.3f}% ± {:.3f}%", "{:.3f} ± {:.3f}"]) + labeled = Recorder(["Training", "Pruning"], ["{:.3f}% ± {:.3f}%", "{:.3f}% ± {:.3f}%"]) + + unlabeled.load(os.path.join(record_dir, "unlabeled.json")) + labeled.load(os.path.join(record_dir, "labeled.json")) + + plot_labeled_performance_curves("Image", labeled[0], labeled[1], + [100, 200, 500, 1000, 2000, 4000, 6000, 8000, 10000], + save_path=os.path.abspath(os.path.join(__file__, "..", "labeled.png"))) if __name__ == "__main__": diff --git a/examples/dataset_cifar_workflow/mock.py b/examples/dataset_cifar_workflow/mock.py index 554718f..e08199b 100644 --- a/examples/dataset_cifar_workflow/mock.py +++ b/examples/dataset_cifar_workflow/mock.py @@ -64,7 +64,7 @@ def get_cifar10(output_channels=3, image_size=32, z_score=True, order=None): train_X = transform_data(X_train, whitening_mat) test_X = transform_data(X_train, whitening_mat) - selected_data_indexes, order = split_dataset(y_test, 3000, split="user", order=order) + selected_data_indexes, order = split_dataset(y_test, 10000, split="user", order=order) return TensorDataset(test_X[selected_data_indexes], y_test[selected_data_indexes]), order @@ -72,35 +72,12 @@ def get_cifar10(output_channels=3, image_size=32, z_score=True, order=None): if __name__ == "__main__": - # 3 5 - # learnware.init(deterministic=False) - # - # userset1, order = get_cifar10() - # print(order) - # loader = DataLoader(userset1, batch_size=3000, shuffle=True) - # sampled_X, _ = next(iter(loader)) - # spec = generate_rkme_image_spec(sampled_X, whitening=False) - # spec.msg = order - # spec.save("old1.json") - # old1 = spec - # - # # userset2 = userset1 - # userset2, order = get_cifar10() - # print(order) - # loader = DataLoader(userset2, batch_size=3000, shuffle=True) - # sampled_X, _ = next(iter(loader)) - # spec = generate_rkme_image_spec(sampled_X, whitening=False) - # spec.msg = order - # spec.save("old2.json") - # old2 = spec - # - # old1, order1 = get_spec("hope1.json", order=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) - # old2, order2 = get_spec("hope2.json", order=[2, 3, 4, 5, 0, 1, 6, 7, 8, 9]) - # np.random.seed(0) - # random.seed(0) - old1, order1 = get_spec(None, order=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) - - old2, order2 = get_spec(None, order=[2, 3, 4, 5, 6, 7, 0, 1, 8, 9]) - print(order1, order2) - print(f(old1.dist(old2))) + # old1, order1 = get_spec("spec_1_V100.json", order=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + # old2, order2 = get_spec("spec_2_A100.json", order=[2, 3, 4, 5, 6, 7, 0, 1, 8, 9]) + + old3, order3 = get_spec("spec_3_A100.json", order=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + old4, order4 = get_spec("spec_6_A100.json", order=[2, 3, 4, 5, 6, 7, 0, 1, 8, 9]) + + print(order3, order4) + print(f(old3.dist(old4))) diff --git a/learnware/specification/regular/image/rkme.py b/learnware/specification/regular/image/rkme.py index 20838dc..f957fee 100644 --- a/learnware/specification/regular/image/rkme.py +++ b/learnware/specification/regular/image/rkme.py @@ -459,7 +459,7 @@ def deterministic(cross_platform, device): torch.cuda.manual_seed_all(0) deterministic_state = torch.backends.cudnn.deterministic torch.backends.cudnn.deterministic = True - if cross_platform: + if cross_platform and torch.cuda.is_available(): torch.cuda.set_rng_state( new_state=torch.cuda.get_rng_state(device.index), device="cpu") @@ -467,7 +467,7 @@ def deterministic(cross_platform, device): yield RandomGenerator(seed=0, cross_platform=cross_platform) torch.backends.cudnn.deterministic = deterministic_state - if cross_platform: + if cross_platform and torch.cuda.is_available(): torch.cuda.set_rng_state( new_state=torch.cuda.get_rng_state(device.index), device="cuda")