[MNT] unify example format

examples/dataset_image_workflow/README.md

@@ -0,0 +1,31 @@
+# Image Dataset Workflow Example
+
+## Introduction
+
+For the CIFAR-10 dataset, we sampled the training set unevenly by category and constructed unbalanced training datasets for the 50 learnwares that contained only some of the categories. This makes it unlikely that there exists any learnware in the learnware market that can accurately handle all categories of data; only the learnware whose training data is closest to the data distribution of the target task is likely to perform well on the target task. Specifically, the probability of each category being sampled obeys a random multinomial distribution, with a non-zero probability of sampling on only 4 categories, and the sampling ratio is 0.4: 0.4: 0.1: 0.1. Ultimately, the training set for each learnware contains 12,000 samples covering the data of 4 categories in CIFAR-10.
+
+We constructed 50 target tasks using data from the test set of CIFAR-10. Similar to constructing the training set for the learnwares, in order to allow for some variation between tasks, we sampled the test set unevenly. Specifically, the probability of each category being sampled obeys a random multinomial distribution, with non-zero sampling probability on 6 categories, and the sampling ratio is 0.3: 0.3: 0.1: 0.1: 0.1: 0.1. Ultimately, each target task contains 3000 samples covering the data of 6 categories in CIFAR-10.
+
+With this experimental setup, we evaluated the performance of RKME Image by calculating the mean accuracy across all users.
+
+| Metric                               | Value               |
+|--------------------------------------|---------------------|
+| Mean in Market (Single)              | 0.346               |
+| Best in Market (Single)              | 0.688               |
+| Top-1 Reuse (Single)                 | 0.534               |
+| Job Selector Reuse (Multiple)        | 0.534               |
+| Average Ensemble Reuse (Multiple)    | 0.676               |
+
+In some specific settings, the user will have a small number of labeled samples. In such settings, learning the weight of selected learnwares on a limited number of labeled samples can result in a better performance than training directly on a limited number of labeled samples.
+
+<div align=center>
+  <img src="../../docs/_static/img/image_labeled.png" alt="Image Limited Labeled Data" style="width:50%;" />
+</div>
+
+## Run the code
+
+Run the following command to start the ``image_example``.
+
+```bash
+python workflow.py image_example
+```

examples/dataset_image_workflow/benchmarks/dataset/__init__.py

@@ -1 +0,0 @@
-from .data import *

examples/dataset_image_workflow/benchmarks/dataset/data.py

@@ -1,49 +0,0 @@
-import os
-
-import numpy as np
-import torch
-from torch.utils.data import random_split, Subset
-from torchvision import datasets
-from torchvision.transforms import transforms
-from torch.utils.data import TensorDataset
-
-from .utils import cached
-from examples.dataset_image_workflow.benchmarks.dataset.utils import split_dataset, build_transforms
-
-cache_root = os.path.abspath(os.path.join(os.path.dirname( __file__ ), '..', '..', 'cache'))
-
-cifar_train = datasets.CIFAR10(root=cache_root, download=True, train=True, transform=transforms.ToTensor())
-cifar_train_X = torch.stack([u[0] for u in cifar_train])
-augment_transform, regular_transform, whiten_transform = build_transforms(cifar_train_X)
-
-cifar_train_set_augment = datasets.CIFAR10(root=cache_root, download=True, train=True, transform=whiten_transform)
-cifar_test_set = datasets.CIFAR10(root=cache_root, download=True, train=False, transform=whiten_transform)
-cifar_spec_train_set = datasets.CIFAR10(root=cache_root, download=True, train=True, transform=whiten_transform)
-cifar_spec_test_set = datasets.CIFAR10(root=cache_root, download=True, train=False, transform=whiten_transform)
-train_targets = cifar_train_set_augment.targets
-test_targets = cifar_test_set.targets
-
-def faster_train(device):
-    global cifar_train_set_augment
-    global cifar_test_set
-    global cifar_spec_train_set
-    global cifar_spec_test_set
-    cifar_train_set_augment = cached(cifar_train_set_augment, device=device)
-    cifar_test_set = cached(cifar_test_set, device=device)
-    cifar_spec_train_set = cached(cifar_spec_train_set, device=device)
-    cifar_spec_test_set = cached(cifar_spec_test_set, device=device)
-
-def uploader_data(order=None):
-    train_indices, order = split_dataset(torch.asarray(train_targets), 12500, split="uploader", order=order)
-    valid_indices, _ = split_dataset(torch.asarray(test_targets), 2000, split="uploader", order=order)
-
-    return (Subset(cifar_train_set_augment, train_indices),
-            Subset(cifar_test_set, valid_indices),
-            Subset(cifar_spec_train_set, train_indices),
-            order)
-
-def user_data(indices=None, order=None):
-    if indices is None:
-        indices, order = split_dataset(torch.asarray(test_targets), 3000, split="user", order=order)
-
-    return Subset(cifar_test_set, indices), Subset(cifar_spec_test_set, indices), indices, order

examples/dataset_image_workflow/benchmarks/dataset/utils.py

@@ -1,96 +0,0 @@
-import random
-from functools import reduce
-
-import numpy as np
-import torch
-import torchvision
-from torch.utils.data import TensorDataset, Dataset, DataLoader
-
-from learnware.utils import choose_device
-
-torchvision.disable_beta_transforms_warning()
-from torchvision.transforms import transforms, v2
-
-
-def sample_by_labels(labels: torch.Tensor, weights, total_num):
-    weights = np.asarray(weights)
-
-    norm_factor = np.sum(weights)
-    last_non_zero = np.argwhere(weights > 0)[-1].item()
-    category_nums = [int(w * total_num / norm_factor) for w in weights[:last_non_zero]]
-    category_nums += [total_num - sum(category_nums)]
-    category_nums += [0] * (weights.shape[0] - last_non_zero - 1)
-
-    selected_cls_indexes = [
-        random.sample(list(torch.where(labels == c)[0]), k=n)
-            for c, n in enumerate(category_nums)
-    ]
-
-    return selected_cls_indexes
-
-
-USER_WEIGHTS = [3, 3, 1, 1, 1, 1, 0, 0, 0, 0]
-UPLOADER_WEIGHTS = [4, 4, 1, 1, 0, 0, 0, 0, 0, 0]
-def split_dataset(labels, size, split="uploader", order=None):
-    if split == "uploader":
-        weights = np.asarray(UPLOADER_WEIGHTS)
-    elif split == "user":
-        weights = np.asarray(USER_WEIGHTS)
-    else:
-        raise Exception(split)
-
-    if order is None:
-        order = list(range(len(weights)))
-        random.shuffle(order)
-
-    selected_data_indexes = reduce(lambda x, y: x+y, sample_by_labels(labels, weights[order], size))
-    selected_data_indexes = torch.stack(selected_data_indexes)
-
-    return selected_data_indexes, order
-
-def build_zca_matrix(X, reg_coef=0.1):
-    X = (X - torch.mean(X, [0, 2, 3], keepdim=True)) / (torch.std(X, [0, 2, 3], keepdim=True))
-
-    device = choose_device(0)
-    X_flat = X.reshape(X.shape[0], -1)
-    cov = (X_flat.T @ X_flat) / X_flat.shape[0]
-    reg_amount = reg_coef * torch.trace(cov) / cov.shape[0]
-    u, s, _ = torch.svd(cov.to(device) + reg_amount * torch.eye(cov.shape[0]).to(device))
-    inv_sqrt_zca_eigs = s ** (-0.5)
-    whitening_transform = torch.einsum(
-        'ij,j,kj->ik', u, inv_sqrt_zca_eigs, u)
-
-    return whitening_transform.cpu()
-
-def build_transforms(train_X):
-    size = train_X.shape[2], train_X.shape[3]
-    whitening_matrix = build_zca_matrix(train_X)
-
-    mean_vector = torch.mean(train_X, [0, 2, 3], keepdim=True).squeeze(0)
-    std_vector = torch.std(train_X, [0, 2, 3], keepdim=True).squeeze(0)
-
-    augment_transform = transforms.Compose([
-        transforms.Resize(size),
-        transforms.ToTensor(),
-        transforms.Normalize(mean=mean_vector, std=std_vector),
-    ])
-
-    regular_transform = transforms.Compose([
-        transforms.Resize(size),
-        transforms.ToTensor(),
-        transforms.Normalize(mean=mean_vector, std=std_vector),
-    ])
-
-    whiten_transform = transforms.Compose([
-        transforms.Resize(size),
-        transforms.ToTensor(),
-        transforms.Normalize(mean=mean_vector, std=std_vector),
-        # transform_data
-        transforms.LinearTransformation(whitening_matrix, torch.zeros_like(train_X[0].reshape(-1)))
-    ])
-
-    return augment_transform, regular_transform, whiten_transform
-
-def cached(data: Dataset, device):
-    X, y = next(iter(DataLoader(data, batch_size=len(data))))
-    return TensorDataset(X.to(device), y.to(device))

examples/dataset_image_workflow/benchmarks/models/conv/__init__.py

@@ -1,29 +0,0 @@
-import os
-
-import torch
-import numpy as np
-from learnware.model import BaseModel
-
-from .model import ConvModel
-
-
-class Model(BaseModel):
-    def __init__(self, device="cuda", input_channel=3):
-        super(Model, self).__init__(input_shape=(input_channel, 32, 32), output_shape=(10,))
-        dir_path = os.path.dirname(os.path.abspath(__file__))
-        self.device =device
-        self.model = ConvModel(channel=input_channel, n_random_features=10)
-        self.model.load_state_dict(torch.load(os.path.join(dir_path, "model.pth")))
-        self.model.to(device).eval()
-
-    def fit(self, X: np.ndarray, y: np.ndarray):
-        raise NotImplementedError()
-
-    def predict(self, X: np.ndarray) -> np.ndarray:
-        return self.model(torch.asarray(X, dtype=torch.float32, device=self.device))
-
-    def __call__(self, *args, **kwargs):
-        self.predict(*args, **kwargs)
-
-    def finetune(self, X: np.ndarray, y: np.ndarray):
-        raise NotImplementedError()

examples/dataset_image_workflow/benchmarks/models/conv/model.py

@@ -1,71 +0,0 @@
-from torch import nn
-
-
-class ConvModel(nn.Module):
-    def __init__(self, channel, n_random_features, net_width = 64, net_depth = 3, net_act = 'relu',
-                 net_norm = 'batchnorm', net_pooling = 'avgpooling', im_size = (32,32)):
-        super().__init__()
-        self.features, shape_feat = self._make_layers(channel, net_width, net_depth, net_norm, net_act, net_pooling, im_size)
-        num_feat = shape_feat[0]*shape_feat[1]*shape_feat[2]
-        self.classifier = nn.Linear(num_feat, n_random_features)
-
-    def forward(self, x):
-        out = self.features(x)
-        out = out.reshape(out.size(0), -1)
-        out = self.classifier(out)
-        return out
-
-    def _get_activation(self, net_act):
-        if net_act == 'sigmoid':
-            return nn.Sigmoid()
-        elif net_act == 'relu':
-            return nn.ReLU(inplace=True)
-        elif net_act == 'leakyrelu':
-            return nn.LeakyReLU(negative_slope=0.01)
-        elif net_act == 'gelu':
-            return nn.SiLU()
-        else:
-            raise Exception('unknown activation function: %s'%net_act)
-
-    def _get_pooling(self, net_pooling):
-        if net_pooling == 'maxpooling':
-            return nn.MaxPool2d(kernel_size=2, stride=2)
-        elif net_pooling == 'avgpooling':
-            return nn.AvgPool2d(kernel_size=2, stride=2)
-        elif net_pooling == 'none':
-            return None
-        else:
-            raise Exception('unknown net_pooling: %s'%net_pooling)
-
-    def _get_normlayer(self, net_norm, shape_feat):
-        if net_norm == 'batchnorm':
-            return nn.BatchNorm2d(shape_feat[0], affine=True)
-        elif net_norm == 'layernorm':
-            return nn.LayerNorm(shape_feat, elementwise_affine=True)
-        elif net_norm == 'instancenorm':
-            return nn.GroupNorm(shape_feat[0], shape_feat[0], affine=True)
-        elif net_norm == 'groupnorm':
-            return nn.GroupNorm(4, shape_feat[0], affine=True)
-        elif net_norm == 'none':
-            return None
-        else:
-            raise Exception('unknown net_norm: %s'%net_norm)
-
-    def _make_layers(self, channel, net_width, net_depth, net_norm, net_act, net_pooling, im_size):
-        layers = []
-        in_channels = channel
-        shape_feat = [in_channels, im_size[0], im_size[1]]
-        for d in range(net_depth):
-            layers += [nn.Conv2d(in_channels, net_width, kernel_size=3, padding='same')]
-
-            shape_feat[0] = net_width
-            if net_norm != 'none':
-                layers += [self._get_normlayer(net_norm, shape_feat)]
-            layers += [self._get_activation(net_act)]
-            in_channels = net_width
-            if net_pooling != 'none':
-                layers += [self._get_pooling(net_pooling)]
-                shape_feat[1] //= 2
-                shape_feat[2] //= 2
-
-        return nn.Sequential(*layers), shape_feat

examples/dataset_image_workflow/benchmarks/models/conv/requirements.txt

@@ -1,3 +0,0 @@
-numpy
-torch>2.0.0
-torchvision

examples/dataset_image_workflow/benchmarks/models/learnware.yaml

@@ -1,8 +0,0 @@
-model:
-  class_name: Model
-  kwargs: {}
-stat_specifications:
-  - module_path: learnware.specification
-    class_name: RKMEImageSpecification
-    file_name: spec.json
-    kwargs: {}

examples/dataset_image_workflow/benchmarks/utils.py

@@ -1,278 +0,0 @@
-import json
-import os
-import pickle
-import zipfile
-from collections import defaultdict
-from shutil import rmtree
-
-from matplotlib import pyplot as plt
-from tabulate import tabulate
-
-import numpy as np
-import torch
-from torch import optim, nn
-from torch.utils.data import DataLoader, Dataset
-
-from learnware.client import LearnwareClient
-from learnware.learnware import Learnware
-from learnware.specification import generate_rkme_image_spec, RKMEImageSpecification
-from .dataset import uploader_data, user_data
-from .dataset.utils import cached
-from .models.conv import ConvModel
-from learnware.market import LearnwareMarket
-from learnware.utils import choose_device
-
-from torch.profiler import profile, record_function, ProfilerActivity
-
-@torch.no_grad()
-def evaluate(model, evaluate_set: Dataset, device=None, distribution=True):
-    device = choose_device(0) if device is None else device
-
-    if isinstance(model, nn.Module):
-        model.eval()
-        mapping = lambda m, x: m(x)
-    else: # For predict interface
-        mapping = lambda m, x: m.predict(x)
-
-    criterion = nn.CrossEntropyLoss(reduction="sum")
-    total, correct, loss = 0, 0, torch.as_tensor(0.0, dtype=torch.float32, device=device)
-    dataloader = DataLoader(evaluate_set, batch_size=1024, shuffle=True)
-    for i, (X, y) in enumerate(dataloader):
-        X, y = X.to(device), y.to(device)
-        out = mapping(model, X)
-        if not torch.is_tensor(out):
-            out = torch.from_numpy(out).to(device)
-
-        if distribution:
-            loss += criterion(out, y)
-            _, predicted = torch.max(out.data, 1)
-        else:
-            predicted = out
-
-        total += y.size(0)
-        correct += (predicted == y).sum().item()
-
-    acc = correct / total * 100
-    loss = loss / total
-
-    if isinstance(model, nn.Module):
-        model.train()
-
-    return loss.item(), acc
-
-
-def build_learnware(name: str, market: LearnwareMarket, order, model_name="conv",
-                    out_classes=10, epochs=35, batch_size=128, device=None):
-    device = choose_device(0) if device is None else device
-
-    if name == "cifar10":
-        train_set, valid_set, spec_set, order = uploader_data(order=order)
-    else:
-        raise Exception("Not support", name)
-
-    cache_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'cache', 'learnware'))
-    if os.path.exists(cache_dir):
-        rmtree(cache_dir)
-    os.makedirs(cache_dir, exist_ok=True)
-
-    channel = train_set[0][0].shape[0]
-    image_size = train_set[0][0].shape[1], train_set[0][0].shape[2]
-    model = ConvModel(channel=channel, im_size=image_size,
-                      n_random_features=out_classes).to(device)
-
-    # train model
-    save_path = os.path.join(cache_dir, "model.pth")
-    train_model(model, train_set, valid_set, save_path, epochs=epochs, batch_size=batch_size, device=device)
-
-    # build specification
-    loader = DataLoader(spec_set, batch_size=3000, shuffle=True)
-    sampled_X, _ = next(iter(loader))
-    spec = generate_rkme_image_spec(sampled_X, whitening=False, experimental=False)
-
-    # add to market
-    model_dir = os.path.abspath(os.path.join(__file__, "..", "models"))
-    spec.save(os.path.join(cache_dir, "spec.json"))
-
-    zip_file = os.path.join(cache_dir, "learnware.zip")
-    # zip -q -r -j zip_file dir_path
-    with zipfile.ZipFile(zip_file, "w") as zip_obj:
-        for foldername, subfolders, filenames in os.walk(os.path.join(model_dir, model_name)):
-            for filename in filenames:
-                if filename.endswith(".pyc"):
-                    continue
-                file_path = os.path.join(foldername, filename)
-                zip_info = zipfile.ZipInfo(filename)
-                zip_info.compress_type = zipfile.ZIP_STORED
-                with open(file_path, "rb") as file:
-                    zip_obj.writestr(zip_info, file.read())
-
-        for filename, file_path in zip(["spec.json", "model.pth", "learnware.yaml"],
-                                      [os.path.join(cache_dir, "spec.json"),
-                                       os.path.join(cache_dir, "model.pth"),
-                                       os.path.join(model_dir, "learnware.yaml")]):
-            zip_info = zipfile.ZipInfo(filename)
-            zip_info.compress_type = zipfile.ZIP_STORED
-            with open(file_path, "rb") as file:
-                zip_obj.writestr(zip_info, file.read())
-
-    print(", ".join([str(o) for o in order]))
-    market.add_learnware(zip_file, semantic_spec=LearnwareClient.create_semantic_specification(
-        self=None,
-        name="learnware",
-        description=", ".join([str(o) for o in order]),
-        data_type="Image",
-        task_type="Classification",
-        library_type="PyTorch",
-        scenarios=["Computer"],
-        output_description={"Dimension": out_classes, "Description": {str(i): "i" for i in range(out_classes)}})
-    )
-
-    return model
-
-def train_model(model: nn.Module, train_set: Dataset, valid_set: Dataset,
-                save_path: str, epochs=35, batch_size=128,
-                device=None, verbose=True):
-    device = choose_device(0) if device is None else device
-
-    model.train()
-    # SGD optimizer with learning rate 1e-2
-    optimizer = optim.SGD(model.parameters(), lr=1e-2, momentum=0.9)
-    # Scheduler
-    # scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=20)
-    # mean-squared error loss
-    criterion = nn.CrossEntropyLoss()
-    # Prepare DataLoader
-    dataloader = DataLoader(train_set, batch_size=batch_size, shuffle=True)
-    # valid loss
-    best_loss = 100000 # initially
-    # Optimizing...
-    for epoch in range(epochs):
-        running_loss = []
-        model.train()
-        for i, (X, y) in enumerate(dataloader):
-            X, y = X.to(device=device), y.to(device=device)
-            optimizer.zero_grad()
-            out = model(X)
-            loss = criterion(out, y)
-            loss.backward()
-            optimizer.step()
-            running_loss.append(loss.item())
-
-        valid_loss, valid_acc = evaluate(model, valid_set, device=device)
-        train_loss, train_acc = evaluate(model, train_set, device=device)
-        if valid_loss < best_loss:
-            best_loss = valid_loss
-
-            torch.save(model.state_dict(), save_path)
-            if verbose:
-                print("Epoch: {}, Valid Best Accuracy: {:.3f}% ({:.3f})".format(epoch+1, valid_acc, valid_loss))
-        if valid_acc > 99.0:
-            if verbose:
-                print("Early Stopping at 99% !")
-            break
-
-        if verbose and (epoch + 1) % 5 == 0:
-            print('Epoch: {}, Train Average Loss: {:.3f}, Accuracy {:.3f}%, Valid Average Loss: {:.3f}'.format(
-                epoch+1, np.mean(running_loss), train_acc, valid_loss))
-
-
-def build_specification(name: str, cache_id, order, sampled_size=3000):
-    cache_dir = os.path.abspath(os.path.join(
-        os.path.dirname(__file__), '..', 'cache'))
-    os.makedirs(cache_dir, exist_ok=True)
-    spec_cache_path = os.path.join(cache_dir, 'spec', "spec_{}.json".format(cache_id))
-
-    if os.path.exists(spec_cache_path):
-        spec = RKMEImageSpecification()
-        spec.load(spec_cache_path)
-
-        test_dataset, spec_dataset, _, _ = user_data(indices=torch.asarray(spec.msg))
-    else:
-        test_dataset, spec_dataset, indices, _ = user_data(order=order)
-        loader = DataLoader(spec_dataset, batch_size=sampled_size, shuffle=True)
-        sampled_X, _ = next(iter(loader))
-        spec = generate_rkme_image_spec(sampled_X, whitening=False, experimental=False)
-
-        spec.msg = indices.tolist()
-        spec.save(spec_cache_path)
-
-    # Save test_dataset to disk, spec_dataset is same as test_dataset for now
-    X, y = next(iter(DataLoader(test_dataset, batch_size=len(test_dataset))))
-    with open(os.path.join(cache_dir, 'test_data', "user{}_X.pkl".format(cache_id)), "wb") as f:
-        pickle.dump(X.detach().cpu().numpy(), f)
-    with open(os.path.join(cache_dir, 'test_data', "user{}_y.pkl".format(cache_id)), "wb") as f:
-        pickle.dump(y.detach().cpu().numpy(), f)
-    return spec, test_dataset
-
-
-class Recorder:
-
-    def __init__(self, headers, formats):
-        assert len(headers) == len(formats)
-        self.data = defaultdict(list)
-        self.headers = headers
-        self.formats = formats
-
-    def record(self, name, *args):
-        self.data[name].append(args)
-
-    def summary(self):
-        table = []
-
-        for name, values in self.data.items():
-            value_mean = [np.mean(v) for v in zip(*values)]
-            value_std = [np.std(v) for v in zip(*values)]
-            table.append([name] + [f.format(m, s) for f, m, s in zip(self.formats, value_mean, value_std)])
-
-        return str(tabulate(table, headers=["Case"] + self.headers, tablefmt='orgtbl'))
-
-    def __getitem__(self, item):
-        return [[x[item] for x in v] for k, v in self.data.items()]
-
-    def save(self, path):
-        with open(path, "w") as f:
-            json.dump(self.data, f)
-
-    def load(self, path):
-        with open(path, "r") as f:
-            self.data = json.load(f)
-
-
-def plot_labeled_performance_curves(name, user_mat, pruning_mat, n_labeled_list, save_path=None):
-    plt.figure(figsize=(10, 6))
-    plt.xticks(range(len(n_labeled_list)), n_labeled_list)
-
-    mats = [user_mat, pruning_mat]
-
-    styles = [
-        {"color": "navy", "linestyle": "-", "marker": "o"},
-        {"color": "magenta", "linestyle": "-.", "marker": "d"},
-    ]
-
-    labels = [
-        "User Model",
-        "Multiple Learnware Reuse (EnsemblePrune)",
-    ]
-
-    for mat, style, label in zip(mats, styles, labels):
-        array_mat = 1 - np.asarray(mat) / 100
-        mean_curve, std_curve = np.mean(array_mat, axis=1), np.std(array_mat, axis=1)
-        plt.plot(mean_curve, **style, label=label)
-        plt.fill_between(
-            range(len(n_labeled_list)),
-            mean_curve - 0.5 * std_curve,
-            mean_curve + 0.5 * std_curve,
-            color=style["color"],
-            alpha=0.2,
-        )
-
-    plt.xlabel("Labeled Data Size")
-    plt.ylabel("1 - Accuracy")
-    plt.title(f"{name} Homo Limited Labeled Data")
-    plt.legend()
-    plt.tight_layout()
-    if save_path:
-        plt.savefig(
-            save_path, bbox_inches="tight", dpi=600
-        )
-    plt.show()

examples/dataset_image_workflow/main.py

@@ -1,161 +0,0 @@
-import os
-import random
-from datetime import datetime
-
-import fire
-import numpy as np
-import tqdm
-from numpy import mean
-import torch
-from torch.utils.data import DataLoader, TensorDataset
-
-import learnware
-from benchmarks.utils import *
-from benchmarks.dataset.data import faster_train, uploader_data
-from benchmarks.models.conv import ConvModel
-from learnware.client import LearnwareClient
-from learnware.market import instantiate_learnware_market, BaseUserInfo
-from learnware.reuse import JobSelectorReuser, AveragingReuser, EnsemblePruningReuser
-from learnware.utils import choose_device
-
-PROXY_IP = "172.27.138.61"
-os.environ["HTTP_PROXY"] = "http://" + PROXY_IP + ":7890"
-os.environ["HTTPS_PROXY"] = "http://" + PROXY_IP + ":7890"
-
-
-class ImageDatasetWorkflow:
-
-    def prepare(self, market_size=50, market_id=None, rebuild=False, faster=True):
-        """initialize learnware market"""
-        learnware.init()
-        assert not rebuild
-
-        np.random.seed(0)
-        random.seed(0)
-        market_id = "dataset_image_workflow" if market_id is None else market_id
-        orders = np.stack([np.random.permutation(10) for _ in range(market_size)])
-
-        print("Using market_id", market_id)
-        market = instantiate_learnware_market(name="easy", market_id=market_id, rebuild=rebuild)
-
-        device = choose_device(0)
-        if faster:
-            faster_train(device)
-        for i, order in enumerate(orders[len(market):]):
-            print("=" * 20 + "learnware {}".format(len(market)) + "=" * 20)
-            print("order:", order)
-            build_learnware("cifar10", market, order, device=device)
-
-        print("Total Item:", len(market))
-
-    def evaluate(self, user_size=100, market_id=None, faster=True):
-        learnware.init()
-
-        np.random.seed(1)
-        random.seed(1)
-        market_id = "dataset_image_workflow" if market_id is None else market_id
-        orders = np.stack([np.random.permutation(10) for _ in range(user_size)])
-
-        print("Using market_id", market_id)
-        market = instantiate_learnware_market(name="easy", market_id=market_id, rebuild=False)
-
-        # Create Folder to save data
-        train_data_cache_folder = os.path.abspath(os.path.join(__file__, '..', "cache", "train_data"))
-        test_data_cache_folder = os.path.abspath(os.path.join(__file__, '..', "cache", "test_data"))
-        os.makedirs(train_data_cache_folder, exist_ok=True)
-        os.makedirs(test_data_cache_folder, exist_ok=True)
-
-        device = choose_device(0)
-        if faster:
-            faster_train(device)
-        unlabeled = Recorder(["Accuracy", "Loss"], ["{:.3f}% ± {:.3f}%", "{:.3f} ± {:.3f}"])
-        labeled = Recorder(["Training", "Pruning"], ["{:.3f}% ± {:.3f}%", "{:.3f}% ± {:.3f}%"])
-        for i, order in enumerate(orders):
-            print("=" * 20 + "user {}".format(i) + "=" * 20)
-            print("order:", order)
-            user_spec, dataset = build_specification("cifar10", i, order)
-
-            user_info = BaseUserInfo(semantic_spec=LearnwareClient.create_semantic_specification(
-                self=None,
-                description="For Cifar Dataset Workflow",
-                data_type="Image",
-                task_type="Classification",
-                library_type="PyTorch",
-                scenarios=["Computer"],
-                output_description={"Dimension": 10, "Description": {str(i): "i" for i in range(10)}}),
-                stat_info={"RKMEImageSpecification": user_spec})
-
-            search_result = market.search_learnware(user_info)
-            single_result = search_result.get_single_results()
-            multiple_result = search_result.get_multiple_results()
-
-            loss_list, acc_list = [], []
-            for item in market.get_learnwares():
-                loss, acc = evaluate(item, dataset)
-                loss_list.append(loss)
-                acc_list.append(acc)
-            unlabeled.record("Best", max(acc_list), min(loss_list))
-            unlabeled.record("Average", mean(acc_list), mean(loss_list))
-
-            top_1_loss, top_1_acc = evaluate(single_result[0].learnware, dataset)
-            unlabeled.record("Top-1 Learnware", top_1_acc, top_1_loss)
-
-            reuse_ensemble = AveragingReuser(learnware_list=multiple_result[0].learnwares, mode="vote_by_prob")
-            ensemble_loss, ensemble_acc = evaluate(reuse_ensemble, dataset)
-            unlabeled.record("Voting Reuse", ensemble_acc, ensemble_loss)
-
-            reuse_job_selector = JobSelectorReuser(learnware_list=multiple_result[0].learnwares, use_herding=False)
-            job_loss, job_acc = evaluate(reuse_job_selector, dataset)
-            unlabeled.record("Job Selector", job_acc, job_loss)
-
-            train_set, _, _, _ = uploader_data(order=order)
-            X, y = next(iter(DataLoader(train_set, batch_size=len(train_set))))
-            with open(os.path.join(train_data_cache_folder, "user{}_X.pkl".format(i)), "wb") as f:
-                pickle.dump(X.detach().cpu().numpy(), f)
-            with open(os.path.join(train_data_cache_folder, "user{}_y.pkl".format(i)), "wb") as f:
-                pickle.dump(y.detach().cpu().numpy(), f)
-
-            for labeled_size in tqdm.tqdm([100, 200, 500, 1000, 2000, 4000, 6000, 8000, 10000]):
-                loader = DataLoader(train_set, batch_size=labeled_size, shuffle=True)
-                X, y = next(iter(loader))
-
-                sampled_dataset = TensorDataset(X, y)
-                mode_save_path = os.path.abspath(os.path.join(__file__, "..", "cache", "model.pth"))
-                model = ConvModel(channel=X.shape[1], im_size=(X.shape[2], X.shape[3]),
-                                  n_random_features=10).to(device)
-                train_model(model, sampled_dataset, sampled_dataset, mode_save_path,
-                            epochs=35, batch_size=128, device=device, verbose=False)
-                model.load_state_dict(torch.load(mode_save_path))
-                _, train_acc = evaluate(model, dataset, distribution=True)
-
-                ensemble_pruning = EnsemblePruningReuser(learnware_list=multiple_result[0].learnwares)
-                ensemble_pruning.fit(val_X=X, val_y=y)
-                _, pruning_acc = evaluate(ensemble_pruning, dataset, distribution=False)
-
-                labeled.record("{:d}".format(labeled_size), train_acc, pruning_acc)
-
-            print(unlabeled.summary())
-            print(labeled.summary())
-
-        # Save recorder
-        current_time = datetime.now()
-        formatted_time = current_time.strftime("%Y-%m-%d_%H-%M-%S")
-        log_dir = os.path.abspath(os.path.join(__file__, "..", "log", formatted_time))
-        os.makedirs(log_dir, exist_ok=True)
-        unlabeled.save(os.path.join(log_dir, "unlabeled.json"))
-        labeled.save(os.path.join(log_dir, "labeled.json"))
-
-    def plot(self, record_dir):
-        unlabeled = Recorder(["Accuracy", "Loss"], ["{:.3f}% ± {:.3f}%", "{:.3f} ± {:.3f}"])
-        labeled = Recorder(["Training", "Pruning"], ["{:.3f}% ± {:.3f}%", "{:.3f}% ± {:.3f}%"])
-
-        unlabeled.load(os.path.join(record_dir, "unlabeled.json"))
-        labeled.load(os.path.join(record_dir, "labeled.json"))
-
-        plot_labeled_performance_curves("Image", labeled[0], labeled[1],
-                                        [100, 200, 500, 1000, 2000, 4000, 6000, 8000, 10000],
-                                        save_path=os.path.abspath(os.path.join(__file__, "..", "labeled.png")))
-
-
-if __name__ == "__main__":
-    fire.Fire(ImageDatasetWorkflow)

examples/dataset_image_workflow/utils.py

@@ -13,7 +13,7 @@ def evaluate(model, evaluate_set: Dataset, device=None, distribution=True):
     if isinstance(model, nn.Module):
         model.eval()
         mapping = lambda m, x: m(x)
-    else:  # For predict interface
+    else:
         mapping = lambda m, x: m.predict(x)
 
     criterion = nn.CrossEntropyLoss(reduction="sum")
@@ -56,17 +56,11 @@ def train_model(
     device = choose_device(0) if device is None else device
 
     model.train()
-    # SGD optimizer with learning rate 1e-2
     optimizer = optim.SGD(model.parameters(), lr=1e-2, momentum=0.9)
-    # Scheduler
-    # scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=20)
-    # mean-squared error loss
     criterion = nn.CrossEntropyLoss()
-    # Prepare DataLoader
     dataloader = DataLoader(train_set, batch_size=batch_size, shuffle=True)
-    # valid loss
-    best_loss = 100000  # initially
-    # Optimizing...
+    best_loss = 100000
+
     for epoch in range(epochs):
         running_loss = []
         model.train()

examples/dataset_image_workflow/workflow.py

@@ -34,25 +34,25 @@ class ImageDatasetWorkflow:
         ]
         labels = ["User Model", "Multiple Learnware Reuse (EnsemblePrune)"]
 
-        user_mat, pruning_mat = all_user_curves_data
-        user_mat, pruning_mat = np.array(user_mat), np.array(pruning_mat)
-        for mat, style, label in zip([user_mat, pruning_mat], styles, labels):
-            mean_curve, std_curve = 1 - np.mean(mat, axis=0), np.std(mat, axis=0)
+        user_array, pruning_array = all_user_curves_data
+        for array, style, label in zip([user_array, pruning_array], styles, labels):
+            mean_curve = np.array([item[0] for item in array])
+            std_curve = np.array([item[1] for item in array])
             plt.plot(mean_curve, **style, label=label)
             plt.fill_between(
                 range(len(mean_curve)),
-                mean_curve - 0.5 * std_curve,
-                mean_curve + 0.5 * std_curve,
+                mean_curve - std_curve,
+                mean_curve + std_curve,
                 color=style["color"],
                 alpha=0.2,
             )
 
-        plt.xlabel("Labeled Data Size")
-        plt.ylabel("1 - Accuracy")
-        plt.title(f"Text Limited Labeled Data")
-        plt.legend()
+        plt.xlabel("Labeled Data Size", fontsize=14)
+        plt.ylabel("1 - Accuracy", fontsize=14)
+        plt.title(f"Image Limited Labeled Data", fontsize=16)
+        plt.legend(fontsize=14)
         plt.tight_layout()
-        plt.savefig(os.path.join(self.fig_path, "image_labeled_curves.png"), bbox_inches="tight", dpi=700)
+        plt.savefig(os.path.join(self.fig_path, "image_labeled_curves.svg"), bbox_inches="tight", dpi=700)
 
     def _prepare_market(self, rebuild=False):
         client = LearnwareClient()
@@ -81,8 +81,8 @@ class ImageDatasetWorkflow:
         np.random.seed(1)
         random.seed(1)
         self._prepare_market(rebuild)
-        self.n_labeled_list = [100, 200, 500, 1000, 2000, 4000, 6000, 8000, 10000]
-        self.repeated_list = [10, 10, 10, 3, 3, 3, 3, 3, 3]
+        self.n_labeled_list = [100, 200, 500, 1000, 2000, 4000]
+        self.repeated_list = [10, 10, 10, 3, 3, 3]
         device = choose_device(0)
 
         self.root_path = os.path.dirname(os.path.abspath(__file__))
@@ -232,15 +232,25 @@ class ImageDatasetWorkflow:
         )
 
         pruning_curves_data, user_model_curves_data = [], []
-        for i in range(self.image_benchmark.user_num):
-            with open(os.path.join(self.curve_path, f"curve{str(i)}.pkl"), "rb") as f:
+        total_user_model_score_mat = [np.zeros(self.repeated_list[i]) for i in range(len(self.n_labeled_list))]
+        total_pruning_score_mat = [np.zeros(self.repeated_list[i]) for i in range(len(self.n_labeled_list))]
+        for user_idx in range(self.image_benchmark.user_num):
+            with open(os.path.join(self.curve_path, f"curve{str(user_idx)}.pkl"), "rb") as f:
                 user_curves_data = pickle.load(f)
                 (single_score_mat, user_model_score_mat, pruning_score_mat) = user_curves_data
-            for i in range(len(single_score_mat)):
-                user_model_score_mat[i] = np.mean(user_model_score_mat[i])
-                pruning_score_mat[i] = np.mean(pruning_score_mat[i])
-            user_model_curves_data.append(user_model_score_mat)
-            pruning_curves_data.append(pruning_score_mat)
+
+                for i in range(len(self.n_labeled_list)):
+                    total_user_model_score_mat[i] += 1 - np.array(user_model_score_mat[i]) / 100
+                    total_pruning_score_mat[i] += 1 - np.array(pruning_score_mat[i]) / 100
+
+        for i in range(len(self.n_labeled_list)):
+            total_user_model_score_mat[i] /= self.image_benchmark.user_num
+            total_pruning_score_mat[i] /= self.image_benchmark.user_num
+            user_model_curves_data.append(
+                (np.mean(total_user_model_score_mat[i]), np.std(total_user_model_score_mat[i]))
+            )
+            pruning_curves_data.append((np.mean(total_pruning_score_mat[i]), np.std(total_pruning_score_mat[i])))
+
         self._plot_labeled_peformance_curves([user_model_curves_data, pruning_curves_data])