From f07d92da718dcd31d87a683a1efe46c7f1c5c282 Mon Sep 17 00:00:00 2001 From: shihy Date: Mon, 4 Dec 2023 23:58:53 +0800 Subject: [PATCH] [ENH] Finish Unlabeled Benchmark for Image --- .../benchmarks/utils.py | 6 ++-- examples/dataset_cifar_workflow/evaluate.bash | 4 +-- examples/dataset_cifar_workflow/main.py | 35 +++++-------------- learnware/specification/regular/image/rkme.py | 14 +++++++- setup.py | 1 + 5 files changed, 27 insertions(+), 33 deletions(-) diff --git a/examples/dataset_cifar_workflow/benchmarks/utils.py b/examples/dataset_cifar_workflow/benchmarks/utils.py index b1f42fd..66a8f0c 100644 --- a/examples/dataset_cifar_workflow/benchmarks/utils.py +++ b/examples/dataset_cifar_workflow/benchmarks/utils.py @@ -116,7 +116,7 @@ def build_learnware(name: str, market: LearnwareMarket, order, model_name="conv" # build specification loader = DataLoader(spec_set, batch_size=3000, shuffle=True) sampled_X, _ = next(iter(loader)) - spec = generate_rkme_image_spec(sampled_X, whitening=False) + spec = generate_rkme_image_spec(sampled_X, whitening=False, cross_platform=False) # add to market model_dir = os.path.abspath(os.path.join(__file__, "..", "models")) @@ -174,7 +174,7 @@ def build_specification(name: str, cache_id, order, sampled_size=3000): test_dataset, spec_dataset, indices, _ = user_data(order=order) loader = DataLoader(spec_dataset, batch_size=sampled_size, shuffle=True) sampled_X, _ = next(iter(loader)) - spec = generate_rkme_image_spec(sampled_X, whitening=False) + spec = generate_rkme_image_spec(sampled_X, whitening=False, cross_platform=False) spec.msg = indices.tolist() spec.save(cache_path) @@ -199,7 +199,7 @@ class Recorder: return str(tabulate(table, headers=["Case", "Accuracy", "Loss"], tablefmt='orgtbl')) - def accumulated(self): + def summary(self): table = [] for name, values in self.data.items(): diff --git a/examples/dataset_cifar_workflow/evaluate.bash b/examples/dataset_cifar_workflow/evaluate.bash index 99cbc5a..bf70a96 100644 --- a/examples/dataset_cifar_workflow/evaluate.bash +++ b/examples/dataset_cifar_workflow/evaluate.bash @@ -10,7 +10,7 @@ token="$(date +%s)" mkdir -p "./log" echo "The output is redirected to log/${token}.log with token ${token}" -export CUDA_VISIBLE_DEVICES=0 +export CUDA_VISIBLE_DEVICES=5. # shellcheck disable=SC2086 -nohup python -u main.py evaluate --market_id="momo" > "./log/${token}.log" 2>&1 & +nohup python -u main.py evaluate_unlabeled --market_id="momo" > "./log/${token}.log" 2>&1 & echo "With PID = $!" \ No newline at end of file diff --git a/examples/dataset_cifar_workflow/main.py b/examples/dataset_cifar_workflow/main.py index 3842de2..6fcbc0f 100644 --- a/examples/dataset_cifar_workflow/main.py +++ b/examples/dataset_cifar_workflow/main.py @@ -9,7 +9,7 @@ import learnware from benchmarks.utils import build_learnware, build_specification, evaluate, Recorder from learnware.client import LearnwareClient from learnware.market import instantiate_learnware_market, BaseUserInfo -from learnware.reuse import AveragingReuser +from learnware.reuse import JobSelectorReuser, AveragingReuser from learnware.specification import generate_rkme_image_spec PROXY_IP = "172.24.57.111" @@ -36,7 +36,7 @@ class CifarDatasetWorkflow: print("Total Item:", len(market)) - def evaluate(self, user_size=100, market_id=None): + def evaluate_unlabeled(self, user_size=100, market_id=None): learnware.init() market_id = "dataset_cifar_workflow" if market_id is None else market_id @@ -73,41 +73,22 @@ class CifarDatasetWorkflow: loss, acc = evaluate(item, dataset) loss_list.append(loss) acc_list.append(acc) - - best_acc_record.append(max(acc_list)) - best_loss_record.append(min(loss_list)) - print("Best Accuracy: {:.3f}% ({:.3f}%), Best Loss: {:.3f} ({:.3f})".format( - max(acc_list), mean(best_acc_record), min(loss_list), mean(best_loss_record))) recorder.record("Best", accuracy=max(acc_list), loss=min(loss_list)) - - mean_acc_record.append(mean(acc_list)) - mean_loss_record.append(mean(loss_list)) - print("Avg Accuracy: {:.3f}% ({:.3f}%), Avg Loss: {:.3f} ({:.3f})".format( - mean(acc_list), mean(mean_acc_record), mean(loss_list), mean(mean_loss_record))) recorder.record("Average", accuracy=mean(acc_list), loss=mean(loss_list)) top_1_loss, top_1_acc = evaluate(single_result[0].learnware, dataset) - top_1_acc_record.append(top_1_acc) - top_1_loss_record.append(top_1_loss) - print( - "Top-1\tAccuracy: {:.3f}% ({:.3f}%), Loss: {:.3f}({:.3f})".format( - top_1_acc, mean(top_1_acc_record), top_1_loss, mean(top_1_loss_record)) - ) - recorder.record("Top-1", accuracy=top_1_acc, loss=top_1_loss) + recorder.record("Top-1 Learnware", accuracy=top_1_acc, loss=top_1_loss) reuse_ensemble = AveragingReuser(learnware_list=multiple_result[0].learnwares, mode="vote_by_prob") # reuse_ensemble = AveragingReuser(learnware_list=[item.learnware for item in single_result[:3]], mode="vote_by_prob") ensemble_loss, ensemble_acc = evaluate(reuse_ensemble, dataset) - ensemble_acc_record.append(ensemble_acc) - ensemble_loss_record.append(ensemble_loss) - print( - "Averaging Reuse\tAccuracy: {:.3f}% ({:.3f}%), Loss: {:.3f} ({:.3f})".format( - ensemble_acc, mean(ensemble_acc_record), ensemble_loss, mean(ensemble_loss_record)) - ) recorder.record("Voting Reuse", accuracy=ensemble_acc, loss=ensemble_loss) - print(recorder.latest()) - print(recorder.accumulated()) + reuse_job_selector = JobSelectorReuser(learnware_list=multiple_result[0].learnwares, use_herding=False) + job_loss, job_acc = evaluate(reuse_job_selector, dataset) + recorder.record("Job Selector", accuracy=job_acc, loss=job_loss) + + print(recorder.summary()) if __name__ == "__main__": diff --git a/learnware/specification/regular/image/rkme.py b/learnware/specification/regular/image/rkme.py index 757aaf7..a10948d 100644 --- a/learnware/specification/regular/image/rkme.py +++ b/learnware/specification/regular/image/rkme.py @@ -155,7 +155,13 @@ class RKMEImageSpecification(RegularStatSpecification): self._random_generator = RandomGenerator(0) # crucial - setup_seed(0) + torch.manual_seed(0) + torch.cuda.manual_seed_all(0) + torch.backends.cudnn.deterministic = True + if ("cross_platform" not in kwargs or kwargs["cross_platform"]) + torch.cuda.set_rng_state( + new_state=torch.cuda.get_rng_state(self._device.index), + device="cpu") random_models = list(self._generate_models(n_models=self.n_models, channel=X.shape[1])) self.z = torch.zeros(Z_shape).to(self._device).float() @@ -181,6 +187,12 @@ class RKMEImageSpecification(RegularStatSpecification): self._update_z(x_features, optimizer, random_models=random_models) self._update_beta(x_features, nonnegative_beta, random_models=random_models) + # Recovering Random Number Generation Settings + if ("cross-platform" not in kwargs or kwargs["cross-platform"]) + torch.cuda.set_rng_state( + new_state=torch.cuda.get_rng_state(self._device.index), + device="cuda") + @torch.no_grad() def _update_beta(self, x_features: Any, nonnegative_beta: bool = True, random_models=None): Z = self.z diff --git a/setup.py b/setup.py index 0c636b8..2ade5e3 100644 --- a/setup.py +++ b/setup.py @@ -37,6 +37,7 @@ REQUIRED = [ "pandas>=0.25.1", "scipy>=1.0.0", "tqdm>=4.65.0", + "tabulate", "scikit-learn>=0.22", "joblib>=1.2.0", "pyyaml>=6.0",