[MNT] add skip_test flag in examples

examples/dataset_image_workflow/workflow.py

@@ -77,10 +77,9 @@ class ImageDatasetWorkflow:
 
         logger.info("Total Item: %d" % (len(self.image_market)))
 
-    def image_example(self, rebuild=False):
+    def image_example(self, rebuild=False, skip_test=True):
         np.random.seed(1)
         random.seed(1)
-        self._prepare_market(rebuild)
         self.n_labeled_list = [100, 200, 500, 1000, 2000, 4000]
         self.repeated_list = [10, 10, 10, 3, 3, 3]
         device = choose_device(0)
@@ -99,142 +98,145 @@ class ImageDatasetWorkflow:
         improve_list = []
         job_selector_score_list = []
         ensemble_score_list = []
-        all_learnwares = self.image_market.get_learnwares()
 
-        for i in range(self.image_benchmark.user_num):
-            test_x, test_y = self.image_benchmark.get_test_data(user_ids=i)
-            train_x, train_y = self.image_benchmark.get_train_data(user_ids=i)
+        if not skip_test:
+            self._prepare_market(rebuild)
+            all_learnwares = self.image_market.get_learnwares()
 
-            test_x = torch.from_numpy(test_x)
-            test_y = torch.from_numpy(test_y)
-            test_dataset = TensorDataset(test_x, test_y)
+            for i in range(image_benchmark_config.user_num):
+                test_x, test_y = self.image_benchmark.get_test_data(user_ids=i)
+                train_x, train_y = self.image_benchmark.get_train_data(user_ids=i)
 
-            user_stat_spec = generate_stat_spec(type="image", X=test_x, whitening=False)
-            user_info = BaseUserInfo(semantic_spec=self.user_semantic, stat_info={user_stat_spec.type: user_stat_spec})
-            logger.info("Searching Market for user: %d" % (i))
+                test_x = torch.from_numpy(test_x)
+                test_y = torch.from_numpy(test_y)
+                test_dataset = TensorDataset(test_x, test_y)
 
-            search_result = self.image_market.search_learnware(user_info)
-            single_result = search_result.get_single_results()
-            multiple_result = search_result.get_multiple_results()
+                user_stat_spec = generate_stat_spec(type="image", X=test_x, whitening=False)
+                user_info = BaseUserInfo(semantic_spec=self.user_semantic, stat_info={user_stat_spec.type: user_stat_spec})
+                logger.info("Searching Market for user: %d" % (i))
 
-            print(f"search result of user{i}:")
-            print(
-                f"single model num: {len(single_result)}, max_score: {single_result[0].score}, min_score: {single_result[-1].score}"
-            )
+                search_result = self.image_market.search_learnware(user_info)
+                single_result = search_result.get_single_results()
+                multiple_result = search_result.get_multiple_results()
 
-            acc_list = []
-            for idx in range(len(all_learnwares)):
-                learnware = all_learnwares[idx]
-                loss, acc = evaluate(learnware, test_dataset)
-                acc_list.append(acc)
-
-            learnware = single_result[0].learnware
-            best_loss, best_acc = evaluate(learnware, test_dataset)
-            best_list.append(np.max(acc_list))
-            select_list.append(best_acc)
-            avg_list.append(np.mean(acc_list))
-            improve_list.append((best_acc - np.mean(acc_list)) / np.mean(acc_list))
-            print(f"market mean accuracy: {np.mean(acc_list)}, market best accuracy: {np.max(acc_list)}")
-            print(
-                f"Top1-score: {single_result[0].score}, learnware_id: {single_result[0].learnware.id}, acc: {best_acc}"
-            )
+                print(f"search result of user{i}:")
+                print(
+                    f"single model num: {len(single_result)}, max_score: {single_result[0].score}, min_score: {single_result[-1].score}"
+                )
 
-            if len(multiple_result) > 0:
-                mixture_id = " ".join([learnware.id for learnware in multiple_result[0].learnwares])
-                print(f"mixture_score: {multiple_result[0].score}, mixture_learnware: {mixture_id}")
-                mixture_learnware_list = multiple_result[0].learnwares
-            else:
-                mixture_learnware_list = [single_result[0].learnware]
-
-            # test reuse (job selector)
-            reuse_job_selector = JobSelectorReuser(learnware_list=mixture_learnware_list, use_herding=False)
-            job_loss, job_acc = evaluate(reuse_job_selector, test_dataset)
-            job_selector_score_list.append(job_acc)
-            print(f"mixture reuse accuracy (job selector): {job_acc}")
-
-            # test reuse (ensemble)
-            reuse_ensemble = AveragingReuser(learnware_list=mixture_learnware_list, mode="vote_by_prob")
-            ensemble_loss, ensemble_acc = evaluate(reuse_ensemble, test_dataset)
-            ensemble_score_list.append(ensemble_acc)
-            print(f"mixture reuse accuracy (ensemble): {ensemble_acc}\n")
-
-            user_model_score_mat = []
-            pruning_score_mat = []
-            single_score_mat = []
-
-            for n_label, repeated in zip(self.n_labeled_list, self.repeated_list):
-                user_model_score_list, reuse_pruning_score_list = [], []
-                if n_label > len(train_x):
-                    n_label = len(train_x)
-                for _ in range(repeated):
-                    x_train, y_train = zip(*random.sample(list(zip(train_x, train_y)), k=n_label))
-                    x_train = np.array(list(x_train))
-                    y_train = np.array(list(y_train))
-
-                    x_train = torch.from_numpy(x_train)
-                    y_train = torch.from_numpy(y_train)
-                    sampled_dataset = TensorDataset(x_train, y_train)
-
-                    mode_save_path = os.path.abspath(os.path.join(self.model_path, "model.pth"))
-                    model = ConvModel(
-                        channel=x_train.shape[1], im_size=(x_train.shape[2], x_train.shape[3]), n_random_features=10
-                    ).to(device)
-                    train_model(
-                        model,
-                        sampled_dataset,
-                        sampled_dataset,
-                        mode_save_path,
-                        epochs=35,
-                        batch_size=128,
-                        device=device,
-                        verbose=False,
-                    )
-                    model.load_state_dict(torch.load(mode_save_path))
-                    _, user_model_acc = evaluate(model, test_dataset, distribution=True)
-                    user_model_score_list.append(user_model_acc)
-
-                    reuse_pruning = EnsemblePruningReuser(learnware_list=mixture_learnware_list, mode="classification")
-                    reuse_pruning.fit(x_train, y_train)
-                    _, pruning_acc = evaluate(reuse_pruning, test_dataset, distribution=False)
-                    reuse_pruning_score_list.append(pruning_acc)
-
-                single_score_mat.append([best_acc] * repeated)
-                user_model_score_mat.append(user_model_score_list)
-                pruning_score_mat.append(reuse_pruning_score_list)
+                acc_list = []
+                for idx in range(len(all_learnwares)):
+                    learnware = all_learnwares[idx]
+                    loss, acc = evaluate(learnware, test_dataset)
+                    acc_list.append(acc)
+
+                learnware = single_result[0].learnware
+                best_loss, best_acc = evaluate(learnware, test_dataset)
+                best_list.append(np.max(acc_list))
+                select_list.append(best_acc)
+                avg_list.append(np.mean(acc_list))
+                improve_list.append((best_acc - np.mean(acc_list)) / np.mean(acc_list))
+                print(f"market mean accuracy: {np.mean(acc_list)}, market best accuracy: {np.max(acc_list)}")
                 print(
-                    f"user_label_num: {n_label}, user_acc: {np.mean(user_model_score_mat[-1])}, pruning_acc: {np.mean(pruning_score_mat[-1])}"
+                    f"Top1-score: {single_result[0].score}, learnware_id: {single_result[0].learnware.id}, acc: {best_acc}"
                 )
 
-            logger.info(f"Saving Curves for User_{i}")
-            user_curves_data = (single_score_mat, user_model_score_mat, pruning_score_mat)
-            with open(os.path.join(self.curve_path, f"curve{str(i)}.pkl"), "wb") as f:
-                pickle.dump(user_curves_data, f)
-
-        logger.info(
-            "Accuracy of selected learnware: %.3f +/- %.3f, Average performance: %.3f +/- %.3f, Best performance: %.3f +/- %.3f"
-            % (
-                np.mean(select_list),
-                np.std(select_list),
-                np.mean(avg_list),
-                np.std(avg_list),
-                np.mean(best_list),
-                np.std(best_list),
+                if len(multiple_result) > 0:
+                    mixture_id = " ".join([learnware.id for learnware in multiple_result[0].learnwares])
+                    print(f"mixture_score: {multiple_result[0].score}, mixture_learnware: {mixture_id}")
+                    mixture_learnware_list = multiple_result[0].learnwares
+                else:
+                    mixture_learnware_list = [single_result[0].learnware]
+
+                # test reuse (job selector)
+                reuse_job_selector = JobSelectorReuser(learnware_list=mixture_learnware_list, use_herding=False)
+                job_loss, job_acc = evaluate(reuse_job_selector, test_dataset)
+                job_selector_score_list.append(job_acc)
+                print(f"mixture reuse accuracy (job selector): {job_acc}")
+
+                # test reuse (ensemble)
+                reuse_ensemble = AveragingReuser(learnware_list=mixture_learnware_list, mode="vote_by_prob")
+                ensemble_loss, ensemble_acc = evaluate(reuse_ensemble, test_dataset)
+                ensemble_score_list.append(ensemble_acc)
+                print(f"mixture reuse accuracy (ensemble): {ensemble_acc}\n")
+
+                user_model_score_mat = []
+                pruning_score_mat = []
+                single_score_mat = []
+
+                for n_label, repeated in zip(self.n_labeled_list, self.repeated_list):
+                    user_model_score_list, reuse_pruning_score_list = [], []
+                    if n_label > len(train_x):
+                        n_label = len(train_x)
+                    for _ in range(repeated):
+                        x_train, y_train = zip(*random.sample(list(zip(train_x, train_y)), k=n_label))
+                        x_train = np.array(list(x_train))
+                        y_train = np.array(list(y_train))
+
+                        x_train = torch.from_numpy(x_train)
+                        y_train = torch.from_numpy(y_train)
+                        sampled_dataset = TensorDataset(x_train, y_train)
+
+                        mode_save_path = os.path.abspath(os.path.join(self.model_path, "model.pth"))
+                        model = ConvModel(
+                            channel=x_train.shape[1], im_size=(x_train.shape[2], x_train.shape[3]), n_random_features=10
+                        ).to(device)
+                        train_model(
+                            model,
+                            sampled_dataset,
+                            sampled_dataset,
+                            mode_save_path,
+                            epochs=35,
+                            batch_size=128,
+                            device=device,
+                            verbose=False,
+                        )
+                        model.load_state_dict(torch.load(mode_save_path))
+                        _, user_model_acc = evaluate(model, test_dataset, distribution=True)
+                        user_model_score_list.append(user_model_acc)
+
+                        reuse_pruning = EnsemblePruningReuser(learnware_list=mixture_learnware_list, mode="classification")
+                        reuse_pruning.fit(x_train, y_train)
+                        _, pruning_acc = evaluate(reuse_pruning, test_dataset, distribution=False)
+                        reuse_pruning_score_list.append(pruning_acc)
+
+                    single_score_mat.append([best_acc] * repeated)
+                    user_model_score_mat.append(user_model_score_list)
+                    pruning_score_mat.append(reuse_pruning_score_list)
+                    print(
+                        f"user_label_num: {n_label}, user_acc: {np.mean(user_model_score_mat[-1])}, pruning_acc: {np.mean(pruning_score_mat[-1])}"
+                    )
+
+                logger.info(f"Saving Curves for User_{i}")
+                user_curves_data = (single_score_mat, user_model_score_mat, pruning_score_mat)
+                with open(os.path.join(self.curve_path, f"curve{str(i)}.pkl"), "wb") as f:
+                    pickle.dump(user_curves_data, f)
+
+            logger.info(
+                "Accuracy of selected learnware: %.3f +/- %.3f, Average performance: %.3f +/- %.3f, Best performance: %.3f +/- %.3f"
+                % (
+                    np.mean(select_list),
+                    np.std(select_list),
+                    np.mean(avg_list),
+                    np.std(avg_list),
+                    np.mean(best_list),
+                    np.std(best_list),
+                )
+            )
+            logger.info("Average performance improvement: %.3f" % (np.mean(improve_list)))
+            logger.info(
+                "Average Job Selector Reuse Performance: %.3f +/- %.3f"
+                % (np.mean(job_selector_score_list), np.std(job_selector_score_list))
+            )
+            logger.info(
+                "Averaging Ensemble Reuse Performance: %.3f +/- %.3f"
+                % (np.mean(ensemble_score_list), np.std(ensemble_score_list))
             )
-        )
-        logger.info("Average performance improvement: %.3f" % (np.mean(improve_list)))
-        logger.info(
-            "Average Job Selector Reuse Performance: %.3f +/- %.3f"
-            % (np.mean(job_selector_score_list), np.std(job_selector_score_list))
-        )
-        logger.info(
-            "Averaging Ensemble Reuse Performance: %.3f +/- %.3f"
-            % (np.mean(ensemble_score_list), np.std(ensemble_score_list))
-        )
 
         pruning_curves_data, user_model_curves_data = [], []
         total_user_model_score_mat = [np.zeros(self.repeated_list[i]) for i in range(len(self.n_labeled_list))]
         total_pruning_score_mat = [np.zeros(self.repeated_list[i]) for i in range(len(self.n_labeled_list))]
-        for user_idx in range(self.image_benchmark.user_num):
+        for user_idx in range(image_benchmark_config.user_num):
             with open(os.path.join(self.curve_path, f"curve{str(user_idx)}.pkl"), "rb") as f:
                 user_curves_data = pickle.load(f)
                 (single_score_mat, user_model_score_mat, pruning_score_mat) = user_curves_data
@@ -244,8 +246,8 @@ class ImageDatasetWorkflow:
                     total_pruning_score_mat[i] += 1 - np.array(pruning_score_mat[i]) / 100
 
         for i in range(len(self.n_labeled_list)):
-            total_user_model_score_mat[i] /= self.image_benchmark.user_num
-            total_pruning_score_mat[i] /= self.image_benchmark.user_num
+            total_user_model_score_mat[i] /= image_benchmark_config.user_num
+            total_pruning_score_mat[i] /= image_benchmark_config.user_num
             user_model_curves_data.append(
                 (np.mean(total_user_model_score_mat[i]), np.std(total_user_model_score_mat[i]))
             )

examples/dataset_text_workflow/workflow.py

@@ -103,7 +103,7 @@ class TextDatasetWorkflow:
         ensemble_score_list = []
         all_learnwares = self.text_market.get_learnwares()
 
-        for i in range(self.text_benchmark.user_num):
+        for i in range(text_benchmark_config.user_num):
             user_data, user_label = self.text_benchmark.get_test_data(user_ids=i)
 
             user_stat_spec = RKMETextSpecification()
@@ -183,19 +183,19 @@ class TextDatasetWorkflow:
             % (np.mean(ensemble_score_list), np.std(ensemble_score_list))
         )
 
-    def labeled_text_example(self, rebuild=False, train_flag=True):
+    def labeled_text_example(self, rebuild=False, skip_test=False):
         self.n_labeled_list = [100, 200, 500, 1000, 2000, 4000]
         self.repeated_list = [10, 10, 10, 3, 3, 3]
         self.root_path = os.path.dirname(os.path.abspath(__file__))
         self.fig_path = os.path.join(self.root_path, "figs")
         self.curve_path = os.path.join(self.root_path, "curves")
-        self._prepare_market(rebuild)
 
-        if train_flag:
+        if not skip_test:
+            self._prepare_market(rebuild)
             os.makedirs(self.fig_path, exist_ok=True)
             os.makedirs(self.curve_path, exist_ok=True)
 
-            for i in range(self.text_benchmark.user_num):
+            for i in range(text_benchmark_config.user_num):
                 user_model_score_mat = []
                 pruning_score_mat = []
                 single_score_mat = []
@@ -268,7 +268,7 @@ class TextDatasetWorkflow:
         pruning_curves_data, user_model_curves_data = [], []
         total_user_model_score_mat = [np.zeros(self.repeated_list[i]) for i in range(len(self.n_labeled_list))]
         total_pruning_score_mat = [np.zeros(self.repeated_list[i]) for i in range(len(self.n_labeled_list))]
-        for user_idx in range(self.text_benchmark.user_num):
+        for user_idx in range(text_benchmark_config.user_num):
             with open(os.path.join(self.curve_path, f"curve{str(user_idx)}.pkl"), "rb") as f:
                 user_curves_data = pickle.load(f)
                 (single_score_mat, user_model_score_mat, pruning_score_mat) = user_curves_data
@@ -278,8 +278,8 @@ class TextDatasetWorkflow:
                     total_pruning_score_mat[i] += 1 - np.array(pruning_score_mat[i])
 
         for i in range(len(self.n_labeled_list)):
-            total_user_model_score_mat[i] /= self.text_benchmark.user_num
-            total_pruning_score_mat[i] /= self.text_benchmark.user_num
+            total_user_model_score_mat[i] /= text_benchmark_config.user_num
+            total_pruning_score_mat[i] /= text_benchmark_config.user_num
             user_model_curves_data.append(
                 (np.mean(total_user_model_score_mat[i]), np.std(total_user_model_score_mat[i]))
             )