[ENH] Finish Unlabeled Benchmark for Image

examples/dataset_cifar_workflow/benchmarks/utils.py

@@ -116,7 +116,7 @@ def build_learnware(name: str, market: LearnwareMarket, order, model_name="conv"
     # build specification
     loader = DataLoader(spec_set, batch_size=3000, shuffle=True)
     sampled_X, _ = next(iter(loader))
-    spec = generate_rkme_image_spec(sampled_X, whitening=False)
+    spec = generate_rkme_image_spec(sampled_X, whitening=False, cross_platform=False)
 
     # add to market
     model_dir = os.path.abspath(os.path.join(__file__, "..", "models"))
@@ -174,7 +174,7 @@ def build_specification(name: str, cache_id, order, sampled_size=3000):
         test_dataset, spec_dataset, indices, _ = user_data(order=order)
         loader = DataLoader(spec_dataset, batch_size=sampled_size, shuffle=True)
         sampled_X, _ = next(iter(loader))
-        spec = generate_rkme_image_spec(sampled_X, whitening=False)
+        spec = generate_rkme_image_spec(sampled_X, whitening=False, cross_platform=False)
 
         spec.msg = indices.tolist()
         spec.save(cache_path)
@@ -199,7 +199,7 @@ class Recorder:
 
         return str(tabulate(table, headers=["Case", "Accuracy", "Loss"], tablefmt='orgtbl'))
 
-    def accumulated(self):
+    def summary(self):
         table = []
 
         for name, values in self.data.items():

examples/dataset_cifar_workflow/evaluate.bash

@@ -10,7 +10,7 @@ token="$(date +%s)"
 mkdir -p "./log"
 echo "The output is redirected to log/${token}.log with token ${token}"
 
-export CUDA_VISIBLE_DEVICES=0
+export CUDA_VISIBLE_DEVICES=5.
 # shellcheck disable=SC2086
-nohup python -u main.py evaluate --market_id="momo" > "./log/${token}.log" 2>&1 &
+nohup python -u main.py evaluate_unlabeled --market_id="momo" > "./log/${token}.log" 2>&1 &
 echo "With PID = $!"

examples/dataset_cifar_workflow/main.py

@@ -9,7 +9,7 @@ import learnware
 from benchmarks.utils import build_learnware, build_specification, evaluate, Recorder
 from learnware.client import LearnwareClient
 from learnware.market import instantiate_learnware_market, BaseUserInfo
-from learnware.reuse import AveragingReuser
+from learnware.reuse import JobSelectorReuser, AveragingReuser
 from learnware.specification import generate_rkme_image_spec
 
 PROXY_IP = "172.24.57.111"
@@ -36,7 +36,7 @@ class CifarDatasetWorkflow:
 
         print("Total Item:", len(market))
 
-    def evaluate(self, user_size=100, market_id=None):
+    def evaluate_unlabeled(self, user_size=100, market_id=None):
         learnware.init()
 
         market_id = "dataset_cifar_workflow" if market_id is None else market_id
@@ -73,41 +73,22 @@ class CifarDatasetWorkflow:
                 loss, acc = evaluate(item, dataset)
                 loss_list.append(loss)
                 acc_list.append(acc)
-
-            best_acc_record.append(max(acc_list))
-            best_loss_record.append(min(loss_list))
-            print("Best Accuracy: {:.3f}% ({:.3f}%), Best Loss: {:.3f} ({:.3f})".format(
-                max(acc_list), mean(best_acc_record), min(loss_list), mean(best_loss_record)))
             recorder.record("Best", accuracy=max(acc_list), loss=min(loss_list))
-
-            mean_acc_record.append(mean(acc_list))
-            mean_loss_record.append(mean(loss_list))
-            print("Avg Accuracy: {:.3f}% ({:.3f}%), Avg Loss: {:.3f} ({:.3f})".format(
-                mean(acc_list), mean(mean_acc_record), mean(loss_list), mean(mean_loss_record)))
             recorder.record("Average", accuracy=mean(acc_list), loss=mean(loss_list))
 
             top_1_loss, top_1_acc  = evaluate(single_result[0].learnware, dataset)
-            top_1_acc_record.append(top_1_acc)
-            top_1_loss_record.append(top_1_loss)
-            print(
-                "Top-1\tAccuracy: {:.3f}% ({:.3f}%), Loss: {:.3f}({:.3f})".format(
-                    top_1_acc, mean(top_1_acc_record), top_1_loss, mean(top_1_loss_record))
-            )
-            recorder.record("Top-1", accuracy=top_1_acc, loss=top_1_loss)
+            recorder.record("Top-1 Learnware", accuracy=top_1_acc, loss=top_1_loss)
 
             reuse_ensemble = AveragingReuser(learnware_list=multiple_result[0].learnwares, mode="vote_by_prob")
             # reuse_ensemble = AveragingReuser(learnware_list=[item.learnware for item in single_result[:3]], mode="vote_by_prob")
             ensemble_loss, ensemble_acc = evaluate(reuse_ensemble, dataset)
-            ensemble_acc_record.append(ensemble_acc)
-            ensemble_loss_record.append(ensemble_loss)
-            print(
-                "Averaging Reuse\tAccuracy: {:.3f}% ({:.3f}%), Loss: {:.3f} ({:.3f})".format(
-                    ensemble_acc, mean(ensemble_acc_record), ensemble_loss, mean(ensemble_loss_record))
-            )
             recorder.record("Voting Reuse", accuracy=ensemble_acc, loss=ensemble_loss)
 
-            print(recorder.latest())
-            print(recorder.accumulated())
+            reuse_job_selector = JobSelectorReuser(learnware_list=multiple_result[0].learnwares, use_herding=False)
+            job_loss, job_acc = evaluate(reuse_job_selector, dataset)
+            recorder.record("Job Selector", accuracy=job_acc, loss=job_loss)
+
+            print(recorder.summary())
 
 
 if __name__ == "__main__":

learnware/specification/regular/image/rkme.py

@@ -155,7 +155,13 @@ class RKMEImageSpecification(RegularStatSpecification):
 
         self._random_generator = RandomGenerator(0)
         # crucial
-        setup_seed(0)
+        torch.manual_seed(0)
+        torch.cuda.manual_seed_all(0)
+        torch.backends.cudnn.deterministic = True
+        if ("cross_platform" not in kwargs or kwargs["cross_platform"])
+            torch.cuda.set_rng_state(
+                new_state=torch.cuda.get_rng_state(self._device.index),
+                device="cpu")
 
         random_models = list(self._generate_models(n_models=self.n_models, channel=X.shape[1]))
         self.z = torch.zeros(Z_shape).to(self._device).float()
@@ -181,6 +187,12 @@ class RKMEImageSpecification(RegularStatSpecification):
             self._update_z(x_features, optimizer, random_models=random_models)
             self._update_beta(x_features, nonnegative_beta, random_models=random_models)
 
+        # Recovering Random Number Generation Settings
+        if ("cross-platform" not in kwargs or kwargs["cross-platform"])
+            torch.cuda.set_rng_state(
+                new_state=torch.cuda.get_rng_state(self._device.index),
+                device="cuda")
+
     @torch.no_grad()
     def _update_beta(self, x_features: Any, nonnegative_beta: bool = True, random_models=None):
         Z = self.z

setup.py

@@ -37,6 +37,7 @@ REQUIRED = [
     "pandas>=0.25.1",
     "scipy>=1.0.0",
     "tqdm>=4.65.0",
+    "tabulate",
     "scikit-learn>=0.22",
     "joblib>=1.2.0",
     "pyyaml>=6.0",