diff --git a/examples/dataset_text_workflow/config.py b/examples/dataset_text_workflow/config.py new file mode 100644 index 0000000..d44f629 --- /dev/null +++ b/examples/dataset_text_workflow/config.py @@ -0,0 +1,62 @@ +from learnware.tests.benchmarks import BenchmarkConfig + + +text_benchmark_config = BenchmarkConfig( + name="20-Newsgroups", + user_num=10, + learnware_ids=[ + "00002193", + "00002192", + "00002191", + "00002190", + "00002189", + "00002188", + "00002187", + "00002186", + "00002185", + "00002184", + "00002183", + "00002182", + "00002181", + "00002180", + "00002179", + "00002178", + "00002177", + "00002176", + "00002175", + "00002174", + "00002173", + "00002172", + "00002171", + "00002170", + "00002169", + "00002168", + "00002167", + "00002166", + "00002165", + "00002164", + "00002163", + "00002162", + "00002161", + "00002160", + "00002159", + "00002158", + "00002157", + "00002156", + "00002155", + "00002154", + "00002153", + "00002152", + "00002151", + "00002150", + "00002149", + "00002148", + "00002147", + "00002146", + "00002145", + "00002144", + ], + test_data_path="20-Newsgroups/test_data.zip", + train_data_path="20-Newsgroups/train_data.zip", + extra_info_path="20-Newsgroup/extra_info.zip", +) diff --git a/examples/dataset_text_workflow/example_files/example_init.py b/examples/dataset_text_workflow/example_files/example_init.py deleted file mode 100644 index 1772a19..0000000 --- a/examples/dataset_text_workflow/example_files/example_init.py +++ /dev/null @@ -1,29 +0,0 @@ -import os -import pickle - -import numpy as np - -from learnware.model import BaseModel - - -class Model(BaseModel): - def __init__(self): - super(Model, self).__init__(input_shape=(1,), output_shape=(1,)) - dir_path = os.path.dirname(os.path.abspath(__file__)) - - modelv_path = os.path.join(dir_path, "modelv.pth") - with open(modelv_path, "rb") as f: - self.modelv = pickle.load(f) - - modell_path = os.path.join(dir_path, "modell.pth") - with open(modell_path, "rb") as f: - self.modell = pickle.load(f) - - def fit(self, X: np.ndarray, y: np.ndarray): - pass - - def predict(self, X: np.ndarray) -> np.ndarray: - return self.modell.predict(self.modelv.transform(X)) - - def finetune(self, X: np.ndarray, y: np.ndarray): - pass diff --git a/examples/dataset_text_workflow/example_files/example_yaml.yaml b/examples/dataset_text_workflow/example_files/example_yaml.yaml deleted file mode 100644 index d29f7dd..0000000 --- a/examples/dataset_text_workflow/example_files/example_yaml.yaml +++ /dev/null @@ -1,8 +0,0 @@ -model: - class_name: Model - kwargs: { } -stat_specifications: - - module_path: learnware.specification - class_name: RKMETextSpecification - file_name: rkme.json - kwargs: { } \ No newline at end of file diff --git a/examples/dataset_text_workflow/example_files/requirements.txt b/examples/dataset_text_workflow/example_files/requirements.txt deleted file mode 100644 index 1a4b344..0000000 --- a/examples/dataset_text_workflow/example_files/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -numpy -pickle -scikit-learn \ No newline at end of file diff --git a/examples/dataset_text_workflow/get_data.py b/examples/dataset_text_workflow/get_data.py deleted file mode 100644 index cee4162..0000000 --- a/examples/dataset_text_workflow/get_data.py +++ /dev/null @@ -1,18 +0,0 @@ -import os -import json -import numpy as np -from sklearn.datasets import fetch_20newsgroups -import pandas as pd - -def get_data(data_root): - dataset_train = fetch_20newsgroups(data_home=data_root, subset='train') - target_names = dataset_train["target_names"] - - X_train = np.array(dataset_train["data"]) - y_train = pd.Categorical.from_codes(dataset_train["target"], categories=target_names) - - X_test, y_test = fetch_20newsgroups(data_home=data_root, subset='test', return_X_y=True) - X_test = np.array(X_test) - y_test = pd.Categorical.from_codes(y_test, categories=target_names) - - return X_train, y_train, X_test, y_test \ No newline at end of file diff --git a/examples/dataset_text_workflow/main.py b/examples/dataset_text_workflow/main.py index 5b4dc82..57b5081 100644 --- a/examples/dataset_text_workflow/main.py +++ b/examples/dataset_text_workflow/main.py @@ -1,202 +1,76 @@ import os import fire -import pickle import time -import zipfile -from shutil import copyfile, rmtree import random - +import pickle +import tempfile import numpy as np +import matplotlib.pyplot as plt +from sklearn.naive_bayes import MultinomialNB +from sklearn.feature_extraction.text import TfidfVectorizer -import learnware.specification as specification -from get_data import get_data +from learnware.client import LearnwareClient from learnware.logger import get_module_logger +from learnware.specification import RKMETextSpecification +from learnware.tests.benchmarks import LearnwareBenchmark from learnware.market import instantiate_learnware_market, BaseUserInfo -from learnware.reuse import JobSelectorReuser, AveragingReuser, EnsemblePruningReuser, FeatureAugmentReuser -from utils import generate_uploader, generate_user, TextDataLoader, train, eval_prediction -from learnware.client import LearnwareClient, SemanticSpecificationKey -import matplotlib.pyplot as plt -from learnware.specification import generate_semantic_spec - -# Login to Beiming system -client = LearnwareClient() +from learnware.reuse import JobSelectorReuser, AveragingReuser, EnsemblePruningReuser +from config import text_benchmark_config logger = get_module_logger("text_workflow", level="INFO") -origin_data_root = "./data/origin_data" -processed_data_root = "./data/processed_data" -tmp_dir = "./data/tmp" -learnware_pool_dir = "./data/learnware_pool" -dataset = "20newsgroups" - -n_uploaders = 50 # max = 10 * n_samples -n_samples = 5 -n_users = 10 # max = 10 -n_classes = 20 - -n_labeled_list = [100, 200, 500, 1000, 2000, 4000] -repeated_list = [10, 10, 10, 3, 3, 3] - -data_root = os.path.join(origin_data_root, dataset) -data_save_root = os.path.join(processed_data_root, dataset) -user_save_root = os.path.join(data_save_root, "user") -uploader_save_root = os.path.join(data_save_root, "uploader") -model_save_root = os.path.join(data_save_root, "uploader_model") -user_train_save_root = os.path.join(data_save_root, "user_train") - -os.makedirs(data_root, exist_ok=True) -os.makedirs(user_save_root, exist_ok=True) -os.makedirs(uploader_save_root, exist_ok=True) -os.makedirs(model_save_root, exist_ok=True) -os.makedirs(user_train_save_root, exist_ok=True) - -output_description = { - "Dimension": 20, - "Description": {"0": "0", "1": "1", "2": "2", "3": "3", "4": "4", "5": "5", "6": "6", - "7": "7", "8": "8", "9": "9", "10": "10", "11": "11", "12": "12", "13": "13", - "14": "14", "15": "15", "16": "16", "17": "17", "18": "18", "19": "19"} -} - - -semantic_spec = generate_semantic_spec( - name="learnware_example", - description="Just a example for text learnware", - data_type="Text", - task_type="Classification", - library_type="Scikit-learn", - scenarios=["Education"], - license="MIT", - input_description=None, - output_description=output_description, -) - -user_semantic = generate_semantic_spec( - # name="learnware_example", - description="Just a example for text learnware", - data_type="Text", - task_type="Classification", - library_type="Scikit-learn", - scenarios=["Education"], - license="MIT", - input_description=None, - output_description=output_description, -) -class TextDatasetWorkflow: - def _init_text_dataset(self): - self._prepare_data() - self._prepare_model() - - def _prepare_data(self): - X_train, y_train, X_test, y_test = get_data(data_root) - - generate_uploader(X_train, y_train, n_uploaders=n_uploaders, n_samples=n_samples, - data_save_root=uploader_save_root) - generate_user(X_test, y_test, n_users=n_users, data_save_root=user_save_root) - - generate_user(X_train, y_train, n_users=n_users, data_save_root=user_train_save_root) - - def _prepare_model(self): - dataloader = TextDataLoader(data_save_root, train=True) - for i in range(n_uploaders): - logger.info("Train on uploader: %d" % (i)) - X, y = dataloader.get_idx_data(i) - vectorizer, clf = train(X, y, out_classes=n_classes) - - modelv_save_path = os.path.join(model_save_root, "uploader_v_%d.pth" % (i)) - modell_save_path = os.path.join(model_save_root, "uploader_l_%d.pth" % (i)) - - with open(modelv_save_path, "wb") as f: - pickle.dump(vectorizer, f) - - with open(modell_save_path, "wb") as f: - pickle.dump(clf, f) - - logger.info("Model saved to '%s' and '%s'" % (modelv_save_path, modell_save_path)) - - def _prepare_learnware( - self, data_path, modelv_path, modell_path, init_file_path, yaml_path, env_file_path, save_root, zip_name - ): - os.makedirs(save_root, exist_ok=True) - tmp_spec_path = os.path.join(save_root, "rkme.json") - - tmp_modelv_path = os.path.join(save_root, "modelv.pth") - tmp_modell_path = os.path.join(save_root, "modell.pth") - - tmp_yaml_path = os.path.join(save_root, "learnware.yaml") - tmp_init_path = os.path.join(save_root, "__init__.py") - tmp_env_path = os.path.join(save_root, "requirements.txt") - - with open(data_path, "rb") as f: - X = pickle.load(f) - - st = time.time() - - user_spec = specification.RKMETextSpecification() - - user_spec.generate_stat_spec_from_data(X=X) - ed = time.time() - logger.info("Stat spec generated in %.3f s" % (ed - st)) - user_spec.save(tmp_spec_path) - - copyfile(modelv_path, tmp_modelv_path) - copyfile(modell_path, tmp_modell_path) - - copyfile(yaml_path, tmp_yaml_path) - copyfile(init_file_path, tmp_init_path) - copyfile(env_file_path, tmp_env_path) - zip_file_name = os.path.join(learnware_pool_dir, "%s.zip" % (zip_name)) - with zipfile.ZipFile(zip_file_name, "w", compression=zipfile.ZIP_DEFLATED) as zip_obj: - zip_obj.write(tmp_spec_path, "rkme.json") - - zip_obj.write(tmp_modelv_path, "modelv.pth") - zip_obj.write(tmp_modell_path, "modell.pth") - - zip_obj.write(tmp_yaml_path, "learnware.yaml") - zip_obj.write(tmp_init_path, "__init__.py") - zip_obj.write(tmp_env_path, "requirements.txt") - rmtree(save_root) - logger.info("New Learnware Saved to %s" % (zip_file_name)) - return zip_file_name - - def prepare_market(self, regenerate_flag=False): - if regenerate_flag: - self._init_text_dataset() - text_market = instantiate_learnware_market(market_id=dataset, rebuild=True) - try: - rmtree(learnware_pool_dir) - except: - pass - os.makedirs(learnware_pool_dir, exist_ok=True) - for i in range(n_uploaders): - data_path = os.path.join(uploader_save_root, "uploader_%d_X.pkl" % (i)) - - modelv_path = os.path.join(model_save_root, "uploader_v_%d.pth" % (i)) - modell_path = os.path.join(model_save_root, "uploader_l_%d.pth" % (i)) - - init_file_path = "./example_files/example_init.py" - yaml_file_path = "./example_files/example_yaml.yaml" - env_file_path = "./example_files/requirements.txt" - new_learnware_path = self._prepare_learnware( - data_path, - modelv_path, - modell_path, - init_file_path, - yaml_file_path, - env_file_path, - tmp_dir, - "%s_%d" % (dataset, i), - ) - semantic_spec["Name"]["Values"] = "learnware_%d" % (i) - semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (i) - text_market.add_learnware(new_learnware_path, semantic_spec) +def train(X, y): + # Train Uploaders' models + vectorizer = TfidfVectorizer(stop_words="english") + X_tfidf = vectorizer.fit_transform(X) + + clf = MultinomialNB(alpha=0.1) + clf.fit(X_tfidf, y) + + return vectorizer, clf + + +def eval_prediction(pred_y, target_y): + if not isinstance(pred_y, np.ndarray): + pred_y = pred_y.detach().cpu().numpy() + if len(pred_y.shape) == 1: + predicted = np.array(pred_y) + else: + predicted = np.argmax(pred_y, 1) + annos = np.array(target_y) + + total = predicted.shape[0] + correct = (predicted == annos).sum().item() - logger.info("Total Item: %d" % (len(text_market))) + return correct / total - def test_unlabeled(self, regenerate_flag=False): - self.prepare_market(regenerate_flag) - text_market = instantiate_learnware_market(market_id=dataset) - print("Total Item: %d" % len(text_market)) + +class TextDatasetWorkflow: + def prepare_market(self, rebuild=False): + client = LearnwareClient() + self.text_benchmark = LearnwareBenchmark().get_benchmark(text_benchmark_config) + self.text_market = instantiate_learnware_market(market_id=self.text_benchmark.name, rebuild=rebuild) + self.user_semantic = client.get_semantic_specification(self.text_benchmark.learnware_ids[0]) + + if len(self.text_market) == 0 or rebuild == True: + for learnware_id in self.text_benchmark.learnware_ids: + with tempfile.TemporaryDirectory(prefix="text_benchmark_") as tempdir: + zip_path = os.path.join(tempdir, f"{learnware_id}.zip") + for i in range(20): + try: + semantic_spec = client.get_semantic_specification(learnware_id) + client.download_learnware(learnware_id, zip_path) + break + except: + time.sleep(1) + continue + self.text_market.add_learnware(zip_path, semantic_spec) + + logger.info("Total Item: %d" % (len(self.text_market))) + + def test_unlabeled(self, rebuild=False): + self.prepare_market(rebuild) select_list = [] avg_list = [] @@ -204,21 +78,19 @@ class TextDatasetWorkflow: improve_list = [] job_selector_score_list = [] ensemble_score_list = [] - all_learnwares = text_market.get_learnwares() - for i in range(n_users): - user_data_path = os.path.join(user_save_root, "user_%d_X.pkl" % (i)) - user_label_path = os.path.join(user_save_root, "user_%d_y.pkl" % (i)) - with open(user_data_path, "rb") as f: - user_data = pickle.load(f) - with open(user_label_path, "rb") as f: - user_label = pickle.load(f) - - user_stat_spec = specification.RKMETextSpecification() + all_learnwares = self.text_market.get_learnwares() + + for i in range(self.text_benchmark.user_num): + user_data, user_label = self.text_benchmark.get_test_data(user_ids=i) + + user_stat_spec = RKMETextSpecification() user_stat_spec.generate_stat_spec_from_data(X=user_data) - user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={"RKMETextSpecification": user_stat_spec}) + user_info = BaseUserInfo( + semantic_spec=self.user_semantic, stat_info={"RKMETextSpecification": user_stat_spec} + ) logger.info("Searching Market for user: %d" % (i)) - search_result = text_market.search_learnware(user_info) + search_result = self.text_market.search_learnware(user_info) single_result = search_result.get_single_results() multiple_result = search_result.get_multiple_results() @@ -267,7 +139,6 @@ class TextDatasetWorkflow: ensemble_score = eval_prediction(ensemble_predict_y, user_label) ensemble_score_list.append(ensemble_score) print(f"mixture reuse accuracy (ensemble): {ensemble_score}") - print("\n") logger.info( @@ -291,43 +162,36 @@ class TextDatasetWorkflow: % (np.mean(ensemble_score_list), np.std(ensemble_score_list)) ) - def test_labeled(self, regenerate_flag=False, train_flag=True): - if train_flag: - self.prepare_market(regenerate_flag) - text_market = instantiate_learnware_market(market_id=dataset) - print("Total Item: %d" % len(text_market)) + def test_labeled(self, rebuild=False, train_flag=True): + self.n_labeled_list = [100, 200, 500, 1000, 2000, 4000] + self.repeated_list = [10, 10, 10, 3, 3, 3] + self.root_path = os.path.dirname(os.path.abspath(__file__)) + self.fig_path = os.path.join(self.root_path, "figs") + self.curve_path = os.path.join(self.root_path, "curves") - os.makedirs("./figs", exist_ok=True) - os.makedirs("./curves", exist_ok=True) + if train_flag: + self.prepare_market(rebuild) + os.makedirs(self.fig_path, exist_ok=True) + os.makedirs(self.curve_path, exist_ok=True) - for i in range(n_users): + for i in range(self.text_benchmark.user_num): user_model_score_mat = [] pruning_score_mat = [] single_score_mat = [] - user_data_path = os.path.join(user_save_root, "user_%d_X.pkl" % (i)) - user_label_path = os.path.join(user_save_root, "user_%d_y.pkl" % (i)) - with open(user_data_path, "rb") as f: - test_x = pickle.load(f) - with open(user_label_path, "rb") as f: - test_y = pickle.load(f) - test_y = np.array(test_y) - - train_data_path = os.path.join(user_train_save_root, "user_%d_X.pkl" % (i)) - train_label_path = os.path.join(user_train_save_root, "user_%d_y.pkl" % (i)) - with open(train_data_path, "rb") as f: - train_x = pickle.load(f) - with open(train_label_path, "rb") as f: - train_y = pickle.load(f) - train_y = np.array(train_y) - - user_stat_spec = specification.RKMETextSpecification() + test_x, test_y = self.text_benchmark.get_test_data(user_ids=i) + test_y = np.array(test_y) + + train_x, train_y = self.text_benchmark.get_train_data(user_ids=i) + train_y = np.array(train_y) + + user_stat_spec = RKMETextSpecification() user_stat_spec.generate_stat_spec_from_data(X=test_x) user_info = BaseUserInfo( - semantic_spec=user_semantic, stat_info={"RKMETextSpecification": user_stat_spec} + semantic_spec=self.user_semantic, stat_info={"RKMETextSpecification": user_stat_spec} ) logger.info(f"Searching Market for user_{i}") - search_result = text_market.search_learnware(user_info) + search_result = self.text_market.search_learnware(user_info) single_result = search_result.get_single_results() multiple_result = search_result.get_multiple_results() @@ -347,7 +211,8 @@ class TextDatasetWorkflow: else: mixture_learnware_list = [single_result[0].learnware] print(len(train_x)) - for n_label, repeated in zip(n_labeled_list, repeated_list): + + for n_label, repeated in zip(self.n_labeled_list, self.repeated_list): user_model_score_list, reuse_pruning_score_list = [], [] if n_label > len(train_x): n_label = len(train_x) @@ -357,7 +222,7 @@ class TextDatasetWorkflow: x_train = list(x_train) y_train = np.array(list(y_train)) - modelv, modell = train(x_train, y_train, out_classes=n_classes) + modelv, modell = train(x_train, y_train) user_model_predict_y = modell.predict(modelv.transform(test_x)) user_model_score = eval_prediction(user_model_predict_y, test_y) user_model_score_list.append(user_model_score) @@ -377,12 +242,12 @@ class TextDatasetWorkflow: logger.info(f"Saving Curves for User_{i}") user_curves_data = (single_score_mat, user_model_score_mat, pruning_score_mat) - # np.save("./curves/curve" + str(i), user_curves_data) - with open("./curves/curve" + str(i) + ".pkl", "wb") as f: + with open(os.path.join(self.curve_path, f"curve{str(i)}.pkl"), "wb") as f: pickle.dump(user_curves_data, f) + pruning_curves_data, user_model_curves_data = [], [] - for i in range(n_users): - with open("./curves/curve" + str(i) + ".pkl", "rb") as f: + for i in range(self.text_benchmark.user_num): + with open(os.path.join(self.curve_path, f"curve{str(i)}.pkl"), "rb") as f: user_curves_data = pickle.load(f) (single_score_mat, user_model_score_mat, pruning_score_mat) = user_curves_data for i in range(len(single_score_mat)): @@ -398,7 +263,7 @@ class TextDatasetWorkflow: def _plot_labeled_peformance_curves(self, all_user_curves_data): plt.figure(figsize=(10, 6)) - plt.xticks(range(len(n_labeled_list)), n_labeled_list) + plt.xticks(range(len(self.n_labeled_list)), self.n_labeled_list) styles = [ # {"color": "orange", "linestyle": "--", "marker": "s"}, @@ -427,7 +292,7 @@ class TextDatasetWorkflow: plt.title(f"Text Limited Labeled Data") plt.legend() plt.tight_layout() - plt.savefig(os.path.join("figs", f"text_labeled_curves.png"), bbox_inches="tight", dpi=700) + plt.savefig(os.path.join(self.fig_path, "text_labeled_curves.png"), bbox_inches="tight", dpi=700) if __name__ == "__main__": diff --git a/examples/dataset_text_workflow/utils.py b/examples/dataset_text_workflow/utils.py deleted file mode 100644 index c38d246..0000000 --- a/examples/dataset_text_workflow/utils.py +++ /dev/null @@ -1,112 +0,0 @@ -import os -import pickle -import random -from itertools import combinations - -import numpy as np -import pandas as pd -from lightgbm import LGBMClassifier, Booster -from sklearn.feature_extraction.text import TfidfVectorizer -from sklearn.model_selection import train_test_split, StratifiedShuffleSplit -from sklearn.naive_bayes import MultinomialNB -from sklearn.metrics import accuracy_score, f1_score - -super_classes = ["comp", "rec", "sci", "talk", "misc"] -super_classes_select2 = list(combinations(super_classes, 2)) -super_classes_select3 = list(combinations(super_classes, 3)) - - -class TextDataLoader: - def __init__(self, data_root, train: bool = True): - self.data_root = data_root - self.train = train - - def get_idx_data(self, idx=0): - if self.train: - X_path = os.path.join(self.data_root, "uploader", "uploader_%d_X.pkl" % (idx)) - y_path = os.path.join(self.data_root, "uploader", "uploader_%d_y.pkl" % (idx)) - if not (os.path.exists(X_path) and os.path.exists(y_path)): - raise Exception("Index Error") - with open(X_path, "rb") as f: - X = pickle.load(f) - with open(y_path, "rb") as f: - y = pickle.load(f) - else: - X_path = os.path.join(self.data_root, "user", "user_%d_X.pkl" % (idx)) - y_path = os.path.join(self.data_root, "user", "user_%d_y.pkl" % (idx)) - if not (os.path.exists(X_path) and os.path.exists(y_path)): - raise Exception("Index Error") - with open(X_path, "rb") as f: - X = pickle.load(f) - with open(y_path, "rb") as f: - y = pickle.load(f) - return X, y - - -def generate_uploader(data_x, data_y, n_uploaders=50, n_samples=5, data_save_root=None): - if data_save_root is None: - return - os.makedirs(data_save_root, exist_ok=True) - - for i, labels in enumerate(super_classes_select3[:n_uploaders // n_samples]): - indices = [idx for idx, label in enumerate(data_y) if label.split('.')[0] in labels] - - for j in range(n_samples): - # sample 50% data to selected_X and selected_y - selected_indices = random.sample(indices, len(indices) // 2) - selected_X = data_x[selected_indices] - selected_y = data_y[selected_indices].codes - - X_save_dir = os.path.join(data_save_root, "uploader_%d_X.pkl" % (i * n_samples + j)) - y_save_dir = os.path.join(data_save_root, "uploader_%d_y.pkl" % (i * n_samples + j)) - - with open(X_save_dir, "wb") as f: - pickle.dump(selected_X, f) - with open(y_save_dir, "wb") as f: - pickle.dump(selected_y, f) - print("Saving to %s" % (X_save_dir)) - -def generate_user(data_x, data_y, n_users=50, data_save_root=None): - if data_save_root is None: - return - os.makedirs(data_save_root, exist_ok=True) - - for i, labels in enumerate(super_classes_select2[:n_users]): - indices = [idx for idx, label in enumerate(data_y) if label.split('.')[0] in labels] - selected_X = data_x[indices] - selected_y = data_y[indices].codes - - X_save_dir = os.path.join(data_save_root, "user_%d_X.pkl" % (i)) - y_save_dir = os.path.join(data_save_root, "user_%d_y.pkl" % (i)) - - with open(X_save_dir, "wb") as f: - pickle.dump(selected_X, f) - with open(y_save_dir, "wb") as f: - pickle.dump(selected_y, f) - print("Saving to %s" % (X_save_dir)) - - -# Train Uploaders' models -def train(X, y, out_classes): - vectorizer = TfidfVectorizer(stop_words="english") - X_tfidf = vectorizer.fit_transform(X) - - clf = MultinomialNB(alpha=0.1) - clf.fit(X_tfidf, y) - - return vectorizer, clf - - -def eval_prediction(pred_y, target_y): - if not isinstance(pred_y, np.ndarray): - pred_y = pred_y.detach().cpu().numpy() - if len(pred_y.shape) == 1: - predicted = np.array(pred_y) - else: - predicted = np.argmax(pred_y, 1) - annos = np.array(target_y) - - total = predicted.shape[0] - correct = (predicted == annos).sum().item() - - return correct / total