From 908f90918d0ea7564c1c2f573be2f09e66cafe2d Mon Sep 17 00:00:00 2001 From: shihy Date: Mon, 4 Dec 2023 20:34:57 +0800 Subject: [PATCH] [ENH] New Recorder for Cifar Workflow --- .../benchmarks/utils.py | 36 +++++++++++++++++-- examples/dataset_cifar_workflow/main.py | 35 +++++++++++++----- examples/dataset_cifar_workflow/prepare.bash | 2 +- learnware/specification/regular/image/rkme.py | 17 ++++++++- 4 files changed, 77 insertions(+), 13 deletions(-) diff --git a/examples/dataset_cifar_workflow/benchmarks/utils.py b/examples/dataset_cifar_workflow/benchmarks/utils.py index 6458400..b1f42fd 100644 --- a/examples/dataset_cifar_workflow/benchmarks/utils.py +++ b/examples/dataset_cifar_workflow/benchmarks/utils.py @@ -1,7 +1,9 @@ import json import os import zipfile +from collections import defaultdict from shutil import rmtree +from tabulate import tabulate import numpy as np import torch @@ -46,7 +48,7 @@ def evaluate(model, evaluate_set: Dataset, device=None): if isinstance(model, nn.Module): model.train() - return loss, acc + return loss.item(), acc def build_learnware(name: str, market: LearnwareMarket, order, model_name="conv", @@ -177,4 +179,34 @@ def build_specification(name: str, cache_id, order, sampled_size=3000): spec.msg = indices.tolist() spec.save(cache_path) - return spec, test_dataset \ No newline at end of file + return spec, test_dataset + + +class Recorder: + + def __init__(self): + self.data = defaultdict(list) + + def record(self, name, accuracy, loss): + self.data[name].append((accuracy, loss)) + + def latest(self): + table = [] + + for name, values in self.data.items(): + value = values[-1] + table.append([name, "{:.3f}%".format(value[0]), "{:.3f}".format(value[1])]) + + return str(tabulate(table, headers=["Case", "Accuracy", "Loss"], tablefmt='orgtbl')) + + def accumulated(self): + table = [] + + for name, values in self.data.items(): + value_mean = [np.mean(v) for v in zip(*values)] + value_std = [np.std(v) for v in zip(*values)] + table.append([name, + "{:.3f}% ± {:.3f}%".format(value_mean[0], value_std[0]), + "{:.3f} ± {:.3f}" .format(value_mean[1], value_std[1])]) + + return str(tabulate(table, headers=["Case", "Accuracy", "Loss"], tablefmt='orgtbl')) \ No newline at end of file diff --git a/examples/dataset_cifar_workflow/main.py b/examples/dataset_cifar_workflow/main.py index 6550efd..3842de2 100644 --- a/examples/dataset_cifar_workflow/main.py +++ b/examples/dataset_cifar_workflow/main.py @@ -6,7 +6,7 @@ from numpy import mean from torch.utils.data import DataLoader import learnware -from benchmarks.utils import build_learnware, build_specification, evaluate +from benchmarks.utils import build_learnware, build_specification, evaluate, Recorder from learnware.client import LearnwareClient from learnware.market import instantiate_learnware_market, BaseUserInfo from learnware.reuse import AveragingReuser @@ -21,6 +21,7 @@ class CifarDatasetWorkflow: def prepare_learnware(self, market_size=50, market_id=None, rebuild=False): """initialize learnware market""" learnware.init() + assert not rebuild market_id = "dataset_cifar_workflow" if market_id is None else market_id orders = np.stack([np.random.permutation(10) for _ in range(market_size)]) @@ -46,6 +47,8 @@ class CifarDatasetWorkflow: top_1_acc_record, ensemble_acc_record, best_acc_record, mean_acc_record = [], [], [], [] top_1_loss_record, ensemble_loss_record, best_loss_record, mean_loss_record = [], [], [], [] + + recorder = Recorder() for i, order in enumerate(orders): print("=" * 20 + "user {}".format(i) + "=" * 20) print("order:", order) @@ -72,25 +75,39 @@ class CifarDatasetWorkflow: acc_list.append(acc) best_acc_record.append(max(acc_list)) + best_loss_record.append(min(loss_list)) + print("Best Accuracy: {:.3f}% ({:.3f}%), Best Loss: {:.3f} ({:.3f})".format( + max(acc_list), mean(best_acc_record), min(loss_list), mean(best_loss_record))) + recorder.record("Best", accuracy=max(acc_list), loss=min(loss_list)) + mean_acc_record.append(mean(acc_list)) - print("Best Accuracy: {:.3f}% ({:.3f}%), Avg Accuracy: {:.3f}% ({:.3f}%)".format( - max(acc_list), mean(best_acc_record), mean(acc_list), mean(mean_acc_record))) + mean_loss_record.append(mean(loss_list)) + print("Avg Accuracy: {:.3f}% ({:.3f}%), Avg Loss: {:.3f} ({:.3f})".format( + mean(acc_list), mean(mean_acc_record), mean(loss_list), mean(mean_loss_record))) + recorder.record("Average", accuracy=mean(acc_list), loss=mean(loss_list)) top_1_loss, top_1_acc = evaluate(single_result[0].learnware, dataset) top_1_acc_record.append(top_1_acc) + top_1_loss_record.append(top_1_loss) print( - "Top-1\tAccuracy: {:.3f}% ({:.3f}%), Loss: {:.3f}".format( - top_1_acc, mean(top_1_acc_record), loss_list[0]) + "Top-1\tAccuracy: {:.3f}% ({:.3f}%), Loss: {:.3f}({:.3f})".format( + top_1_acc, mean(top_1_acc_record), top_1_loss, mean(top_1_loss_record)) ) + recorder.record("Top-1", accuracy=top_1_acc, loss=top_1_loss) - # multiple_result[0].learnwares - reuse_ensemble = AveragingReuser(learnware_list=[item.learnware for item in single_result[:3]], mode="vote_by_prob") + reuse_ensemble = AveragingReuser(learnware_list=multiple_result[0].learnwares, mode="vote_by_prob") + # reuse_ensemble = AveragingReuser(learnware_list=[item.learnware for item in single_result[:3]], mode="vote_by_prob") ensemble_loss, ensemble_acc = evaluate(reuse_ensemble, dataset) ensemble_acc_record.append(ensemble_acc) + ensemble_loss_record.append(ensemble_loss) print( - "Averaging Reuse\tAccuracy: {:.3f} ({:.3f}%), Loss: {:.3f}".format( - ensemble_acc, mean(ensemble_acc_record), ensemble_loss) + "Averaging Reuse\tAccuracy: {:.3f}% ({:.3f}%), Loss: {:.3f} ({:.3f})".format( + ensemble_acc, mean(ensemble_acc_record), ensemble_loss, mean(ensemble_loss_record)) ) + recorder.record("Voting Reuse", accuracy=ensemble_acc, loss=ensemble_loss) + + print(recorder.latest()) + print(recorder.accumulated()) if __name__ == "__main__": diff --git a/examples/dataset_cifar_workflow/prepare.bash b/examples/dataset_cifar_workflow/prepare.bash index 9d48246..f967115 100644 --- a/examples/dataset_cifar_workflow/prepare.bash +++ b/examples/dataset_cifar_workflow/prepare.bash @@ -12,5 +12,5 @@ echo "The output is redirected to log/${token}.log with token ${token}" export CUDA_VISIBLE_DEVICES=1 # shellcheck disable=SC2086 -nohup python -u main.py prepare_learnware --market_id="momo" --rebuild > "./log/${token}.log" 2>&1 & +nohup python -u main.py prepare_learnware --market_id="momo" > "./log/${token}.log" 2>&1 & echo "With PID = $!" \ No newline at end of file diff --git a/learnware/specification/regular/image/rkme.py b/learnware/specification/regular/image/rkme.py index 0d115b8..757aaf7 100644 --- a/learnware/specification/regular/image/rkme.py +++ b/learnware/specification/regular/image/rkme.py @@ -335,7 +335,22 @@ class RKMEImageSpecification(RegularStatSpecification): return K_12 def herding(self, T: int) -> np.ndarray: - raise NotImplementedError("The function herding hasn't been supported in Image RKME Specification.") + """Iteratively sample examples from an unknown distribution with the help of its RKME specification + + Parameters + ---------- + T : int + Total iteration number for sampling. + + Returns + ------- + np.ndarray + A collection of examples which approximate the unknown distribution. + """ + indices = torch.multinomial(self.beta, T, replacement=True) + mock = self.z[indices] + torch.randn_like(self.z[indices]) * 0.01 + + return mock.numpy() def _sampling_candidates(self, N: int) -> np.ndarray: raise NotImplementedError()