[ENH] New Recorder for Cifar Workflow

examples/dataset_cifar_workflow/benchmarks/utils.py

@@ -1,7 +1,9 @@
 import json
 import os
 import zipfile
+from collections import defaultdict
 from shutil import rmtree
+from tabulate import tabulate
 
 import numpy as np
 import torch
@@ -46,7 +48,7 @@ def evaluate(model, evaluate_set: Dataset, device=None):
     if isinstance(model, nn.Module):
         model.train()
 
-    return loss, acc
+    return loss.item(), acc
 
 
 def build_learnware(name: str, market: LearnwareMarket, order, model_name="conv",
@@ -177,4 +179,34 @@ def build_specification(name: str, cache_id, order, sampled_size=3000):
         spec.msg = indices.tolist()
         spec.save(cache_path)
 
-    return spec, test_dataset
+    return spec, test_dataset
+
+
+class Recorder:
+
+    def __init__(self):
+        self.data = defaultdict(list)
+
+    def record(self, name, accuracy, loss):
+        self.data[name].append((accuracy, loss))
+
+    def latest(self):
+        table = []
+
+        for name, values in self.data.items():
+            value = values[-1]
+            table.append([name, "{:.3f}%".format(value[0]), "{:.3f}".format(value[1])])
+
+        return str(tabulate(table, headers=["Case", "Accuracy", "Loss"], tablefmt='orgtbl'))
+
+    def accumulated(self):
+        table = []
+
+        for name, values in self.data.items():
+            value_mean = [np.mean(v) for v in zip(*values)]
+            value_std = [np.std(v) for v in zip(*values)]
+            table.append([name,
+                          "{:.3f}% ± {:.3f}%".format(value_mean[0], value_std[0]),
+                          "{:.3f} ± {:.3f}" .format(value_mean[1], value_std[1])])
+
+        return str(tabulate(table, headers=["Case", "Accuracy", "Loss"], tablefmt='orgtbl'))

examples/dataset_cifar_workflow/main.py

@@ -6,7 +6,7 @@ from numpy import mean
 from torch.utils.data import DataLoader
 
 import learnware
-from benchmarks.utils import build_learnware, build_specification, evaluate
+from benchmarks.utils import build_learnware, build_specification, evaluate, Recorder
 from learnware.client import LearnwareClient
 from learnware.market import instantiate_learnware_market, BaseUserInfo
 from learnware.reuse import AveragingReuser
@@ -21,6 +21,7 @@ class CifarDatasetWorkflow:
     def prepare_learnware(self, market_size=50, market_id=None, rebuild=False):
         """initialize learnware market"""
         learnware.init()
+        assert not rebuild
 
         market_id = "dataset_cifar_workflow" if market_id is None else market_id
         orders = np.stack([np.random.permutation(10) for _ in range(market_size)])
@@ -46,6 +47,8 @@ class CifarDatasetWorkflow:
 
         top_1_acc_record, ensemble_acc_record, best_acc_record, mean_acc_record = [], [], [], []
         top_1_loss_record, ensemble_loss_record, best_loss_record, mean_loss_record = [], [], [], []
+
+        recorder = Recorder()
         for i, order in enumerate(orders):
             print("=" * 20 + "user {}".format(i) + "=" * 20)
             print("order:", order)
@@ -72,25 +75,39 @@ class CifarDatasetWorkflow:
                 acc_list.append(acc)
 
             best_acc_record.append(max(acc_list))
+            best_loss_record.append(min(loss_list))
+            print("Best Accuracy: {:.3f}% ({:.3f}%), Best Loss: {:.3f} ({:.3f})".format(
+                max(acc_list), mean(best_acc_record), min(loss_list), mean(best_loss_record)))
+            recorder.record("Best", accuracy=max(acc_list), loss=min(loss_list))
+
             mean_acc_record.append(mean(acc_list))
-            print("Best Accuracy: {:.3f}% ({:.3f}%), Avg Accuracy: {:.3f}% ({:.3f}%)".format(
-                max(acc_list), mean(best_acc_record), mean(acc_list), mean(mean_acc_record)))
+            mean_loss_record.append(mean(loss_list))
+            print("Avg Accuracy: {:.3f}% ({:.3f}%), Avg Loss: {:.3f} ({:.3f})".format(
+                mean(acc_list), mean(mean_acc_record), mean(loss_list), mean(mean_loss_record)))
+            recorder.record("Average", accuracy=mean(acc_list), loss=mean(loss_list))
 
             top_1_loss, top_1_acc  = evaluate(single_result[0].learnware, dataset)
             top_1_acc_record.append(top_1_acc)
+            top_1_loss_record.append(top_1_loss)
             print(
-                "Top-1\tAccuracy: {:.3f}% ({:.3f}%), Loss: {:.3f}".format(
-                    top_1_acc, mean(top_1_acc_record), loss_list[0])
+                "Top-1\tAccuracy: {:.3f}% ({:.3f}%), Loss: {:.3f}({:.3f})".format(
+                    top_1_acc, mean(top_1_acc_record), top_1_loss, mean(top_1_loss_record))
             )
+            recorder.record("Top-1", accuracy=top_1_acc, loss=top_1_loss)
 
-            # multiple_result[0].learnwares
-            reuse_ensemble = AveragingReuser(learnware_list=[item.learnware for item in single_result[:3]], mode="vote_by_prob")
+            reuse_ensemble = AveragingReuser(learnware_list=multiple_result[0].learnwares, mode="vote_by_prob")
+            # reuse_ensemble = AveragingReuser(learnware_list=[item.learnware for item in single_result[:3]], mode="vote_by_prob")
             ensemble_loss, ensemble_acc = evaluate(reuse_ensemble, dataset)
             ensemble_acc_record.append(ensemble_acc)
+            ensemble_loss_record.append(ensemble_loss)
             print(
-                "Averaging Reuse\tAccuracy: {:.3f} ({:.3f}%), Loss: {:.3f}".format(
-                    ensemble_acc, mean(ensemble_acc_record), ensemble_loss)
+                "Averaging Reuse\tAccuracy: {:.3f}% ({:.3f}%), Loss: {:.3f} ({:.3f})".format(
+                    ensemble_acc, mean(ensemble_acc_record), ensemble_loss, mean(ensemble_loss_record))
             )
+            recorder.record("Voting Reuse", accuracy=ensemble_acc, loss=ensemble_loss)
+
+            print(recorder.latest())
+            print(recorder.accumulated())
 
 
 if __name__ == "__main__":

examples/dataset_cifar_workflow/prepare.bash

@@ -12,5 +12,5 @@ echo "The output is redirected to log/${token}.log with token ${token}"
 
 export CUDA_VISIBLE_DEVICES=1
 # shellcheck disable=SC2086
-nohup python -u main.py prepare_learnware --market_id="momo" --rebuild > "./log/${token}.log" 2>&1 &
+nohup python -u main.py prepare_learnware --market_id="momo" > "./log/${token}.log" 2>&1 &
 echo "With PID = $!"

learnware/specification/regular/image/rkme.py

@@ -335,7 +335,22 @@ class RKMEImageSpecification(RegularStatSpecification):
         return K_12
 
     def herding(self, T: int) -> np.ndarray:
-        raise NotImplementedError("The function herding hasn't been supported in Image RKME Specification.")
+        """Iteratively sample examples from an unknown distribution with the help of its RKME specification
+
+        Parameters
+        ----------
+        T : int
+            Total iteration number for sampling.
+
+        Returns
+        -------
+        np.ndarray
+            A collection of examples which approximate the unknown distribution.
+        """
+        indices = torch.multinomial(self.beta, T, replacement=True)
+        mock = self.z[indices] + torch.randn_like(self.z[indices]) * 0.01
+
+        return mock.numpy()
 
     def _sampling_candidates(self, N: int) -> np.ndarray:
         raise NotImplementedError()