[MNT] optimize the way of generating datasets for uploaders and users

examples/dataset_text_workflow2/get_data.py

@@ -11,8 +11,6 @@ def get_data(data_root):
     X_train = np.array(dataset_train["data"])
     y_train = pd.Categorical.from_codes(dataset_train["target"], categories=target_names)
 
-    # y_train = [target_names[label] for label in dataset_train["target"]]
-
     X_test, y_test = fetch_20newsgroups(data_home=data_root, subset='test', return_X_y=True)
     X_test = np.array(X_test)
     y_test = pd.Categorical.from_codes(y_test, categories=target_names)

examples/dataset_text_workflow2/main.py

@@ -26,9 +26,12 @@ processed_data_root = "./data/processed_data"
 tmp_dir = "./data/tmp"
 learnware_pool_dir = "./data/learnware_pool"
 dataset = "20newsgroups"
-n_uploaders = 10 # max num = 10
-n_users = 5 # max num = 10
+
+n_uploaders = 50 # max = 10 * n_samples
+n_samples = 5
+n_users = 10  # max = 10
 n_classes = 20
+
 n_labeled_list = [100, 200, 500, 1000, 2000, 4000, 6000, 8000, 10000]
 repeated_list = [10, 10, 10, 3, 3, 3, 3, 3, 3]
 
@@ -37,10 +40,13 @@ data_save_root = os.path.join(processed_data_root, dataset)
 user_save_root = os.path.join(data_save_root, "user")
 uploader_save_root = os.path.join(data_save_root, "uploader")
 model_save_root = os.path.join(data_save_root, "uploader_model")
+user_train_save_root = os.path.join(data_save_root, "user_train")
+
 os.makedirs(data_root, exist_ok=True)
 os.makedirs(user_save_root, exist_ok=True)
 os.makedirs(uploader_save_root, exist_ok=True)
 os.makedirs(model_save_root, exist_ok=True)
+os.makedirs(user_train_save_root, exist_ok=True)
 
 output_description = {
     "Dimension": 20,
@@ -82,9 +88,12 @@ class TextDatasetWorkflow:
     def _prepare_data(self):
         X_train, y_train, X_test, y_test = get_data(data_root)
 
-        generate_uploader(X_train, y_train, n_uploaders=n_uploaders, data_save_root=uploader_save_root)
+        generate_uploader(X_train, y_train, n_uploaders=n_uploaders, n_samples=n_samples,
+                          data_save_root=uploader_save_root)
         generate_user(X_test, y_test, n_users=n_users, data_save_root=user_save_root)
 
+        generate_user(X_train, y_train, n_users=n_users, data_save_root=user_train_save_root)
+
     def _prepare_model(self):
         dataloader = TextDataLoader(data_save_root, train=True)
         for i in range(n_uploaders):
@@ -247,7 +256,7 @@ class TextDatasetWorkflow:
             reuse_predict = reuse_baseline.predict(user_data=user_data)
             reuse_score = eval_prediction(reuse_predict, user_label)
             job_selector_score_list.append(reuse_score)
-            print(f"mixture reuse loss(job selector): {reuse_score}")
+            print(f"mixture reuse accuracy (job selector): {reuse_score}")
 
             # test reuse (ensemble)
             # be careful with the ensemble mode
@@ -301,8 +310,8 @@ class TextDatasetWorkflow:
                     test_y = pickle.load(f)
                     test_y = np.array(test_y)
 
-                train_data_path = os.path.join(uploader_save_root, "uploader_%d_X.pkl" % (i))
-                train_label_path = os.path.join(uploader_save_root, "uploader_%d_y.pkl" % (i))
+                train_data_path = os.path.join(user_train_save_root, "user_%d_X.pkl" % (i))
+                train_label_path = os.path.join(user_train_save_root, "user_%d_y.pkl" % (i))
                 with open(train_data_path, "rb") as f:
                     train_x = pickle.load(f)
                 with open(train_label_path, "rb") as f:

examples/dataset_text_workflow2/utils.py

@@ -43,55 +43,42 @@ class TextDataLoader:
         return X, y
 
 
-def generate_uploader(data_x, data_y, n_uploaders=50, data_save_root=None):
+def generate_uploader(data_x, data_y, n_uploaders=50, n_samples=5, data_save_root=None):
     if data_save_root is None:
         return
     os.makedirs(data_save_root, exist_ok=True)
-    n = len(data_x)
 
-    for i, labels in enumerate(super_classes_select3[:n_uploaders]):
+    for i, labels in enumerate(super_classes_select3[:n_uploaders // n_samples]):
         indices = [idx for idx, label in enumerate(data_y) if label.split('.')[0] in labels]
-        selected_X = data_x[indices]
-        selected_y = data_y[indices].codes
 
-        X_save_dir = os.path.join(data_save_root, "uploader_%d_X.pkl" % (i))
-        y_save_dir = os.path.join(data_save_root, "uploader_%d_y.pkl" % (i))
+        for j in range(n_samples):
+            # sample 50% data to selected_X and selected_y
+            selected_indices = random.sample(indices, len(indices) // 2)
+            selected_X = data_x[selected_indices]
+            selected_y = data_y[selected_indices].codes
 
-        with open(X_save_dir, "wb") as f:
-            pickle.dump(selected_X, f)
-        with open(y_save_dir, "wb") as f:
-            pickle.dump(selected_y, f)
-        print("Saving to %s" % (X_save_dir))
+            X_save_dir = os.path.join(data_save_root, "uploader_%d_X.pkl" % (i * n_samples + j))
+            y_save_dir = os.path.join(data_save_root, "uploader_%d_y.pkl" % (i * n_samples + j))
 
-# 随机选取
-# def generate_user(data_x, data_y, n_users=50, data_save_root=None):
-#     if data_save_root is None:
-#         return
-#     os.makedirs(data_save_root, exist_ok=True)
-#     n = len(data_x)
-#     for i in range(n_users):
-#         selected_X = data_x[i * (n // n_users): (i + 1) * (n // n_users)]
-#         selected_y = data_y[i * (n // n_users): (i + 1) * (n // n_users)].codes
-#         X_save_dir = os.path.join(data_save_root, "user_%d_X.pkl" % (i))
-#         y_save_dir = os.path.join(data_save_root, "user_%d_y.pkl" % (i))
-#         with open(X_save_dir, "wb") as f:
-#             pickle.dump(selected_X, f)
-#         with open(y_save_dir, "wb") as f:
-#             pickle.dump(selected_y, f)
-#         print("Saving to %s" % (X_save_dir))
+            with open(X_save_dir, "wb") as f:
+                pickle.dump(selected_X, f)
+            with open(y_save_dir, "wb") as f:
+                pickle.dump(selected_y, f)
+            print("Saving to %s" % (X_save_dir))
 
 def generate_user(data_x, data_y, n_users=50, data_save_root=None):
     if data_save_root is None:
         return
     os.makedirs(data_save_root, exist_ok=True)
-    n = len(data_x)
-    for i, labels in enumerate(super_classes_select3[:n_users]):
+
+    for i, labels in enumerate(super_classes_select2[:n_users]):
         indices = [idx for idx, label in enumerate(data_y) if label.split('.')[0] in labels]
         selected_X = data_x[indices]
         selected_y = data_y[indices].codes
 
         X_save_dir = os.path.join(data_save_root, "user_%d_X.pkl" % (i))
         y_save_dir = os.path.join(data_save_root, "user_%d_y.pkl" % (i))
+
         with open(X_save_dir, "wb") as f:
             pickle.dump(selected_X, f)
         with open(y_save_dir, "wb") as f: