[ENH] Finish Image Workflow

examples/dataset_cifar_workflow/benchmarks/dataset/utils.py

@@ -6,6 +6,8 @@ import torch
 import torchvision
 from torch.utils.data import TensorDataset, Dataset, DataLoader
 
+from learnware.utils import choose_device
+
 torchvision.disable_beta_transforms_warning()
 from torchvision.transforms import transforms, v2
 
@@ -49,10 +51,11 @@ def split_dataset(labels, size, split="uploader", order=None):
 def build_zca_matrix(X, reg_coef=0.1):
     X = (X - torch.mean(X, [0, 2, 3], keepdim=True)) / (torch.std(X, [0, 2, 3], keepdim=True))
 
+    device = choose_device(0)
     X_flat = X.reshape(X.shape[0], -1)
     cov = (X_flat.T @ X_flat) / X_flat.shape[0]
     reg_amount = reg_coef * torch.trace(cov) / cov.shape[0]
-    u, s, _ = torch.svd(cov.cuda() + reg_amount * torch.eye(cov.shape[0]).cuda())
+    u, s, _ = torch.svd(cov.to(device) + reg_amount * torch.eye(cov.shape[0]).to(device))
     inv_sqrt_zca_eigs = s ** (-0.5)
     whitening_transform = torch.einsum(
         'ij,j,kj->ik', u, inv_sqrt_zca_eigs, u)

examples/dataset_cifar_workflow/benchmarks/utils.py

@@ -3,6 +3,8 @@ import os
 import zipfile
 from collections import defaultdict
 from shutil import rmtree
+
+from matplotlib import pyplot as plt
 from tabulate import tabulate
 
 import numpy as np
@@ -127,7 +129,8 @@ def build_learnware(name: str, market: LearnwareMarket, order, model_name="conv"
     return model
 
 def train_model(model: nn.Module, train_set: Dataset, valid_set: Dataset,
-                save_path: str, epochs=35, batch_size=128, device=None):
+                save_path: str, epochs=35, batch_size=128,
+                device=None, verbose=True):
     device = choose_device(0) if device is None else device
 
     model.train()
@@ -160,12 +163,14 @@ def train_model(model: nn.Module, train_set: Dataset, valid_set: Dataset,
             best_loss = valid_loss
 
             torch.save(model.state_dict(), save_path)
-            print("Epoch: {}, Valid Best Accuracy: {:.3f}% ({:.3f})".format(epoch+1, valid_acc, valid_loss))
+            if verbose:
+                print("Epoch: {}, Valid Best Accuracy: {:.3f}% ({:.3f})".format(epoch+1, valid_acc, valid_loss))
         if valid_acc > 99.0:
-            print("Early Stopping at 99% !")
+            if verbose:
+                print("Early Stopping at 99% !")
             break
 
-        if (epoch + 1) % 5 == 0:
+        if verbose and (epoch + 1) % 5 == 0:
             print('Epoch: {}, Train Average Loss: {:.3f}, Accuracy {:.3f}%, Valid Average Loss: {:.3f}'.format(
                 epoch+1, np.mean(running_loss), train_acc, valid_loss))
 
@@ -214,10 +219,53 @@ class Recorder:
 
         return str(tabulate(table, headers=["Case"] + self.headers, tablefmt='orgtbl'))
 
+    def __getitem__(self, item):
+        return [[x[item] for x in v] for k, v in self.data.items()]
+
     def save(self, path):
         with open(path, "w") as f:
             json.dump(self.data, f)
 
     def load(self, path):
         with open(path, "r") as f:
-            self.data = json.load(f)
+            self.data = json.load(f)
+
+
+def plot_labeled_performance_curves(name, user_mat, pruning_mat, n_labeled_list, save_path=None):
+    plt.figure(figsize=(10, 6))
+    plt.xticks(range(len(n_labeled_list)), n_labeled_list)
+
+    mats = [user_mat, pruning_mat]
+
+    styles = [
+        {"color": "navy", "linestyle": "-", "marker": "o"},
+        {"color": "magenta", "linestyle": "-.", "marker": "d"},
+    ]
+
+    labels = [
+        "User Model",
+        "Multiple Learnware Reuse (EnsemblePrune)",
+    ]
+
+    for mat, style, label in zip(mats, styles, labels):
+        array_mat = 1 - np.asarray(mat) / 100
+        mean_curve, std_curve = np.mean(array_mat, axis=1), np.std(array_mat, axis=1)
+        plt.plot(mean_curve, **style, label=label)
+        plt.fill_between(
+            range(len(n_labeled_list)),
+            mean_curve - 0.5 * std_curve,
+            mean_curve + 0.5 * std_curve,
+            color=style["color"],
+            alpha=0.2,
+        )
+
+    plt.xlabel("Labeled Data Size")
+    plt.ylabel("1 - Accuracy")
+    plt.title(f"{name} Homo Limited Labeled Data")
+    plt.legend()
+    plt.tight_layout()
+    if save_path:
+        plt.savefig(
+            save_path, bbox_inches="tight", dpi=600
+        )
+    plt.show()

examples/dataset_cifar_workflow/main.py

@@ -1,24 +1,30 @@
 import os
+from datetime import datetime
 
 import fire
 import numpy as np
+import tqdm
 from numpy import mean
-from torch.utils.data import DataLoader
+import torch
+from torch.utils.data import DataLoader, TensorDataset
 
 import learnware
-from benchmarks.utils import build_learnware, build_specification, evaluate, Recorder
+from benchmarks.utils import *
+from benchmarks.dataset.data import faster_train, uploader_data
+from benchmarks.models.conv import ConvModel
 from learnware.client import LearnwareClient
 from learnware.market import instantiate_learnware_market, BaseUserInfo
-from learnware.reuse import JobSelectorReuser, AveragingReuser
-from learnware.specification import generate_rkme_image_spec
+from learnware.reuse import JobSelectorReuser, AveragingReuser, EnsemblePruningReuser
+from learnware.utils import choose_device
+
+PROXY_IP = "172.27.138.61"
+os.environ["HTTP_PROXY"] = "http://" + PROXY_IP + ":7890"
+os.environ["HTTPS_PROXY"] = "http://" + PROXY_IP + ":7890"
 
-PROXY_IP = "172.24.57.111"
-os.environ["HTTP_PROXY"] = "http://"+PROXY_IP+":7890"
-os.environ["HTTPS_PROXY"] = "http://"+PROXY_IP+":7890"
 
 class CifarDatasetWorkflow:
 
-    def prepare_learnware(self, market_size=50, market_id=None, rebuild=False):
+    def prepare(self, market_size=50, market_id=None, rebuild=False, faster=True):
         """initialize learnware market"""
         learnware.init()
         assert not rebuild
@@ -29,14 +35,17 @@ class CifarDatasetWorkflow:
         print("Using market_id", market_id)
         market = instantiate_learnware_market(name="easy", market_id=market_id, rebuild=rebuild)
 
+        device = choose_device(0)
+        if faster:
+            faster_train(device)
         for i, order in enumerate(orders[len(market):]):
-            print("=" * 20 + "learnware {}".format(i) + "=" * 20)
+            print("=" * 20 + "learnware {}".format(len(market)) + "=" * 20)
             print("order:", order)
-            build_learnware("cifar10", market, order)
+            build_learnware("cifar10", market, order, device=device)
 
         print("Total Item:", len(market))
 
-    def evaluate_unlabeled(self, user_size=100, market_id=None):
+    def evaluate(self, user_size=100, market_id=None, faster=True):
         learnware.init()
 
         market_id = "dataset_cifar_workflow" if market_id is None else market_id
@@ -45,23 +54,24 @@ class CifarDatasetWorkflow:
         print("Using market_id", market_id)
         market = instantiate_learnware_market(name="easy", market_id=market_id, rebuild=False)
 
-        top_1_acc_record, ensemble_acc_record, best_acc_record, mean_acc_record = [], [], [], []
-        top_1_loss_record, ensemble_loss_record, best_loss_record, mean_loss_record = [], [], [], []
-
-        recorder = Recorder()
+        device = choose_device(0)
+        if faster:
+            faster_train(device)
+        unlabeled = Recorder(["Accuracy", "Loss"], ["{:.3f}% ± {:.3f}%", "{:.3f} ± {:.3f}"])
+        labeled = Recorder(["Training", "Pruning"], ["{:.3f}% ± {:.3f}%", "{:.3f}% ± {:.3f}%"])
         for i, order in enumerate(orders):
             print("=" * 20 + "user {}".format(i) + "=" * 20)
             print("order:", order)
             user_spec, dataset = build_specification("cifar10", i, order)
 
             user_info = BaseUserInfo(semantic_spec=LearnwareClient.create_semantic_specification(
-                    self=None,
-                    description="For Cifar Dataset Workflow",
-                    data_type="Image",
-                    task_type="Classification",
-                    library_type="PyTorch",
-                    scenarios=["Computer"],
-                    output_description={"Dimension": 10, "Description": {str(i): "i" for i in range(10)}}),
+                self=None,
+                description="For Cifar Dataset Workflow",
+                data_type="Image",
+                task_type="Classification",
+                library_type="PyTorch",
+                scenarios=["Computer"],
+                output_description={"Dimension": 10, "Description": {str(i): "i" for i in range(10)}}),
                 stat_info={"RKMEImageSpecification": user_spec})
 
             search_result = market.search_learnware(user_info)
@@ -73,22 +83,61 @@ class CifarDatasetWorkflow:
                 loss, acc = evaluate(item, dataset)
                 loss_list.append(loss)
                 acc_list.append(acc)
-            recorder.record("Best", accuracy=max(acc_list), loss=min(loss_list))
-            recorder.record("Average", accuracy=mean(acc_list), loss=mean(loss_list))
+            unlabeled.record("Best", max(acc_list), min(loss_list))
+            unlabeled.record("Average", mean(acc_list), mean(loss_list))
 
-            top_1_loss, top_1_acc  = evaluate(single_result[0].learnware, dataset)
-            recorder.record("Top-1 Learnware", accuracy=top_1_acc, loss=top_1_loss)
+            top_1_loss, top_1_acc = evaluate(single_result[0].learnware, dataset)
+            unlabeled.record("Top-1 Learnware", top_1_acc, top_1_loss)
 
             reuse_ensemble = AveragingReuser(learnware_list=multiple_result[0].learnwares, mode="vote_by_prob")
-            # reuse_ensemble = AveragingReuser(learnware_list=[item.learnware for item in single_result[:3]], mode="vote_by_prob")
             ensemble_loss, ensemble_acc = evaluate(reuse_ensemble, dataset)
-            recorder.record("Voting Reuse", accuracy=ensemble_acc, loss=ensemble_loss)
+            unlabeled.record("Voting Reuse", ensemble_acc, ensemble_loss)
 
             reuse_job_selector = JobSelectorReuser(learnware_list=multiple_result[0].learnwares, use_herding=False)
             job_loss, job_acc = evaluate(reuse_job_selector, dataset)
-            recorder.record("Job Selector", accuracy=job_acc, loss=job_loss)
-
-            print(recorder.summary())
+            unlabeled.record("Job Selector", job_acc, job_loss)
+
+            train_set, valid_set, spec_set, order = uploader_data(order=order)
+            for labeled_size in tqdm.tqdm([100, 200, 500, 1000, 2000, 4000, 6000, 8000, 10000]):
+                loader = DataLoader(train_set, batch_size=labeled_size, shuffle=True)
+                X, y = next(iter(loader))
+
+                sampled_dataset = TensorDataset(X, y)
+                mode_save_path = os.path.abspath(os.path.join(__file__, "..", "cache", "model.pth"))
+                model = ConvModel(channel=X.shape[1], im_size=(X.shape[2], X.shape[3]),
+                                  n_random_features=10).to(device)
+                train_model(model, sampled_dataset, sampled_dataset, mode_save_path,
+                            epochs=35, batch_size=128, device=device, verbose=False)
+                model.load_state_dict(torch.load(mode_save_path))
+                _, train_acc = evaluate(model, dataset, distribution=True)
+
+                ensemble_pruning = EnsemblePruningReuser(learnware_list=multiple_result[0].learnwares)
+                ensemble_pruning.fit(val_X=X, val_y=y)
+                _, pruning_acc = evaluate(ensemble_pruning, dataset, distribution=False)
+
+                labeled.record("{:d}".format(labeled_size), train_acc, pruning_acc)
+
+            print(unlabeled.summary())
+            print(labeled.summary())
+
+        # Save recorder
+        current_time = datetime.now()
+        formatted_time = current_time.strftime("%Y-%m-%d_%H-%M-%S")
+        log_dir = os.path.abspath(os.path.join(__file__, "..", "log", formatted_time))
+        os.makedirs(log_dir, exist_ok=True)
+        unlabeled.save(os.path.join(log_dir, "unlabeled.json"))
+        labeled.save(os.path.join(log_dir, "labeled.json"))
+
+    def plot(self, record_dir):
+        unlabeled = Recorder(["Accuracy", "Loss"], ["{:.3f}% ± {:.3f}%", "{:.3f} ± {:.3f}"])
+        labeled = Recorder(["Training", "Pruning"], ["{:.3f}% ± {:.3f}%", "{:.3f}% ± {:.3f}%"])
+
+        unlabeled.load(os.path.join(record_dir, "unlabeled.json"))
+        labeled.load(os.path.join(record_dir, "labeled.json"))
+
+        plot_labeled_performance_curves("Image", labeled[0], labeled[1],
+                                        [100, 200, 500, 1000, 2000, 4000, 6000, 8000, 10000],
+                                        save_path=os.path.abspath(os.path.join(__file__, "..", "labeled.png")))
 
 
 if __name__ == "__main__":

examples/dataset_cifar_workflow/mock.py

@@ -64,7 +64,7 @@ def get_cifar10(output_channels=3, image_size=32, z_score=True, order=None):
     train_X = transform_data(X_train, whitening_mat)
     test_X = transform_data(X_train, whitening_mat)
 
-    selected_data_indexes, order = split_dataset(y_test, 3000, split="user", order=order)
+    selected_data_indexes, order = split_dataset(y_test, 10000, split="user", order=order)
 
     return TensorDataset(test_X[selected_data_indexes], y_test[selected_data_indexes]), order
 
@@ -72,35 +72,12 @@ def get_cifar10(output_channels=3, image_size=32, z_score=True, order=None):
 
 
 if __name__ == "__main__":
-    # 3 5
-    # learnware.init(deterministic=False)
-    #
-    # userset1, order = get_cifar10()
-    # print(order)
-    # loader = DataLoader(userset1, batch_size=3000, shuffle=True)
-    # sampled_X, _ = next(iter(loader))
-    # spec = generate_rkme_image_spec(sampled_X, whitening=False)
-    # spec.msg = order
-    # spec.save("old1.json")
-    # old1 = spec
-    #
-    # # userset2 = userset1
-    # userset2, order = get_cifar10()
-    # print(order)
-    # loader = DataLoader(userset2, batch_size=3000, shuffle=True)
-    # sampled_X, _ = next(iter(loader))
-    # spec = generate_rkme_image_spec(sampled_X, whitening=False)
-    # spec.msg = order
-    # spec.save("old2.json")
-    # old2 = spec
-    #
-    # old1, order1 = get_spec("hope1.json", order=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
-    # old2, order2 = get_spec("hope2.json", order=[2, 3, 4, 5, 0, 1, 6, 7, 8, 9])
-    # np.random.seed(0)
-    # random.seed(0)
-    old1, order1 = get_spec(None, order=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
-
-    old2, order2 = get_spec(None, order=[2, 3, 4, 5, 6, 7, 0, 1, 8, 9])
-    print(order1, order2)
-    print(f(old1.dist(old2)))
+    # old1, order1 = get_spec("spec_1_V100.json", order=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
+    # old2, order2 = get_spec("spec_2_A100.json", order=[2, 3, 4, 5, 6, 7, 0, 1, 8, 9])
+
+    old3, order3 = get_spec("spec_3_A100.json", order=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
+    old4, order4 = get_spec("spec_6_A100.json", order=[2, 3, 4, 5, 6, 7, 0, 1, 8, 9])
+
+    print(order3, order4)
+    print(f(old3.dist(old4)))
 

learnware/specification/regular/image/rkme.py

@@ -459,7 +459,7 @@ def deterministic(cross_platform, device):
     torch.cuda.manual_seed_all(0)
     deterministic_state = torch.backends.cudnn.deterministic
     torch.backends.cudnn.deterministic = True
-    if cross_platform:
+    if cross_platform and torch.cuda.is_available():
         torch.cuda.set_rng_state(
             new_state=torch.cuda.get_rng_state(device.index),
             device="cpu")
@@ -467,7 +467,7 @@ def deterministic(cross_platform, device):
     yield RandomGenerator(seed=0, cross_platform=cross_platform)
 
     torch.backends.cudnn.deterministic = deterministic_state
-    if cross_platform:
+    if cross_platform and torch.cuda.is_available():
         torch.cuda.set_rng_state(
             new_state=torch.cuda.get_rng_state(device.index),
             device="cuda")