From c31f5f4dc87724784c9c7000fa32dd9fb7481cb3 Mon Sep 17 00:00:00 2001 From: bxdd Date: Tue, 7 Nov 2023 14:25:51 +0800 Subject: [PATCH 1/6] [FIX] fix cannot import generate_rkme bugs --- docs/workflow/submit.rst | 4 ++-- examples/dataset_m5_workflow/main.py | 7 +++---- examples/dataset_pfs_workflow/main.py | 8 +++----- examples/dataset_text_workflow/main.py | 13 ++++--------- examples/workflow_by_code/main.py | 7 +++---- learnware/reuse/job_selector.py | 2 +- tests/test_market/test_easy.py | 6 +++--- tests/test_specification/test_rkme.py | 5 ++--- tests/test_workflow/test_workflow.py | 8 ++++---- 9 files changed, 25 insertions(+), 35 deletions(-) diff --git a/docs/workflow/submit.rst b/docs/workflow/submit.rst index fe097c3..2d82936 100644 --- a/docs/workflow/submit.rst +++ b/docs/workflow/submit.rst @@ -80,10 +80,10 @@ the following code snippet offers guidance on how to construct and store the RKM .. code-block:: python - import learnware.specification as specification + from learnware.specification import generate_rkme_spec # generate rkme specification for digits dataset - spec = specification.utils.generate_rkme_spec(X=data_X) + spec = generate_rkme_spec(X=data_X) spec.save("stat.json") Significantly, the RKME generation process is entirely conducted on your local machine, without any involvement of cloud services, diff --git a/examples/dataset_m5_workflow/main.py b/examples/dataset_m5_workflow/main.py index 009b557..bfdbe71 100644 --- a/examples/dataset_m5_workflow/main.py +++ b/examples/dataset_m5_workflow/main.py @@ -9,9 +9,8 @@ from shutil import copyfile, rmtree import learnware from learnware.market import EasyMarket, BaseUserInfo from learnware.market import database_ops -from learnware.learnware import Learnware from learnware.reuse import JobSelectorReuser, AveragingReuser -import learnware.specification as specification +from learnware.specification import generate_rkme_spec from m5 import DataLoader from learnware.logger import get_module_logger @@ -88,7 +87,7 @@ class M5DatasetWorkflow: for idx in tqdm(idx_list): train_x, train_y, test_x, test_y = m5.get_idx_data(idx) st = time.time() - spec = specification.utils.generate_rkme_spec(X=train_x, gamma=0.1, cuda_idx=0) + spec = generate_rkme_spec(X=train_x, gamma=0.1, cuda_idx=0) ed = time.time() logger.info("Stat spec generated in %.3f s" % (ed - st)) @@ -140,7 +139,7 @@ class M5DatasetWorkflow: for idx in idx_list: train_x, train_y, test_x, test_y = m5.get_idx_data(idx) - user_spec = specification.utils.generate_rkme_spec(X=test_x, gamma=0.1, cuda_idx=0) + user_spec = generate_rkme_spec(X=test_x, gamma=0.1, cuda_idx=0) user_spec_path = f"./user_spec/user_{idx}.json" user_spec.save(user_spec_path) diff --git a/examples/dataset_pfs_workflow/main.py b/examples/dataset_pfs_workflow/main.py index b5cbdd8..abe80cd 100644 --- a/examples/dataset_pfs_workflow/main.py +++ b/examples/dataset_pfs_workflow/main.py @@ -8,10 +8,8 @@ from shutil import copyfile, rmtree import learnware from learnware.market import EasyMarket, BaseUserInfo -from learnware.market import database_ops -from learnware.learnware import Learnware from learnware.reuse import JobSelectorReuser, AveragingReuser -import learnware.specification as specification +from learnware.specification import generate_rkme_spec from pfs import Dataloader from learnware.logger import get_module_logger @@ -86,7 +84,7 @@ class PFSDatasetWorkflow: for idx in tqdm(idx_list): train_x, train_y, test_x, test_y = pfs.get_idx_data(idx) st = time.time() - spec = specification.utils.generate_rkme_spec(X=train_x, gamma=0.1, cuda_idx=0) + spec = generate_rkme_spec(X=train_x, gamma=0.1, cuda_idx=0) ed = time.time() logger.info("Stat spec generated in %.3f s" % (ed - st)) @@ -138,7 +136,7 @@ class PFSDatasetWorkflow: for idx in idx_list: train_x, train_y, test_x, test_y = pfs.get_idx_data(idx) - user_spec = specification.utils.generate_rkme_spec(X=test_x, gamma=0.1, cuda_idx=0) + user_spec = generate_rkme_spec(X=test_x, gamma=0.1, cuda_idx=0) user_spec_path = f"./user_spec/user_{idx}.json" user_spec.save(user_spec_path) diff --git a/examples/dataset_text_workflow/main.py b/examples/dataset_text_workflow/main.py index e7e1c38..406aad9 100644 --- a/examples/dataset_text_workflow/main.py +++ b/examples/dataset_text_workflow/main.py @@ -10,9 +10,7 @@ import time import pickle from learnware.market import instantiate_learnware_market, BaseUserInfo -from learnware.market import database_ops -from learnware.learnware import Learnware -import learnware.specification as specification +from learnware.specification import RKMETextSpecification from learnware.logger import get_module_logger from shutil import copyfile, rmtree @@ -99,8 +97,7 @@ def prepare_learnware(data_path, model_path, init_file_path, yaml_path, save_roo semantic_spec = semantic_specs[0] st = time.time() - # user_spec = specification.utils.generate_rkme_spec(X=X, gamma=0.1, cuda_idx=0) - user_spec = specification.RKMETextSpecification() + user_spec = RKMETextSpecification() user_spec.generate_stat_spec_from_data(X=X) ed = time.time() logger.info("Stat spec generated in %.3f s" % (ed - st)) @@ -163,10 +160,8 @@ def test_search(gamma=0.1, load_market=True): user_data = pickle.load(f) with open(user_label_path, "rb") as f: user_label = pickle.load(f) - # user_data = np.load(user_data_path) - # user_label = np.load(user_label_path) - # user_stat_spec = specification.utils.generate_rkme_spec(X=user_data, gamma=gamma, cuda_idx=0) - user_stat_spec = specification.RKMETextSpecification() + + user_stat_spec = RKMETextSpecification() user_stat_spec.generate_stat_spec_from_data(X=user_data) user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={"RKMETextSpecification": user_stat_spec}) logger.info("Searching Market for user: %d" % (i)) diff --git a/examples/workflow_by_code/main.py b/examples/workflow_by_code/main.py index 2f62db0..afedc41 100644 --- a/examples/workflow_by_code/main.py +++ b/examples/workflow_by_code/main.py @@ -12,8 +12,7 @@ from shutil import copyfile, rmtree import learnware from learnware.market import EasyMarket, BaseUserInfo from learnware.reuse import JobSelectorReuser, AveragingReuser -import learnware.specification as specification -from learnware.utils import get_module_by_module_path +from learnware.specification import generate_rkme_spec curr_root = os.path.dirname(os.path.abspath(__file__)) @@ -54,7 +53,7 @@ class LearnwareMarketWorkflow: joblib.dump(clf, os.path.join(dir_path, "svm.pkl")) - spec = specification.utils.generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0) + spec = generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0) spec.save(os.path.join(dir_path, "svm.json")) init_file = os.path.join(dir_path, "__init__.py") @@ -174,7 +173,7 @@ class LearnwareMarketWorkflow: X, y = load_digits(return_X_y=True) _, data_X, _, data_y = train_test_split(X, y, test_size=0.3, shuffle=True) - stat_spec = specification.utils.generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0) + stat_spec = generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0) user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={"RKMETableSpecification": stat_spec}) _, _, _, mixture_learnware_list = easy_market.search_learnware(user_info) diff --git a/learnware/reuse/job_selector.py b/learnware/reuse/job_selector.py index a131fd5..5e3a71f 100644 --- a/learnware/reuse/job_selector.py +++ b/learnware/reuse/job_selector.py @@ -11,7 +11,7 @@ from .base import BaseReuser from ..market.utils import parse_specification_type from ..learnware import Learnware from ..specification import RKMETableSpecification, RKMETextSpecification -from ..specification.utils import generate_rkme_spec +from ..specification import generate_rkme_spec from ..logger import get_module_logger logger = get_module_logger("job_selector_reuse") diff --git a/tests/test_market/test_easy.py b/tests/test_market/test_easy.py index bb03839..0ca779f 100644 --- a/tests/test_market/test_easy.py +++ b/tests/test_market/test_easy.py @@ -12,7 +12,7 @@ from shutil import copyfile, rmtree import learnware from learnware.market import instantiate_learnware_market, BaseUserInfo -import learnware.specification as specification +from learnware.specification import RKMETableSpecification, generate_rkme_spec curr_root = os.path.dirname(os.path.abspath(__file__)) @@ -62,7 +62,7 @@ class TestMarket(unittest.TestCase): joblib.dump(clf, os.path.join(dir_path, "svm.pkl")) - spec = specification.utils.generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0) + spec = generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0) spec.save(os.path.join(dir_path, "svm.json")) init_file = os.path.join(dir_path, "__init__.py") @@ -170,7 +170,7 @@ class TestMarket(unittest.TestCase): with zipfile.ZipFile(zip_path, "r") as zip_obj: zip_obj.extractall(path=unzip_dir) - user_spec = specification.rkme.RKMETableSpecification() + user_spec = RKMETableSpecification() user_spec.load(os.path.join(unzip_dir, "svm.json")) user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={"RKMETableSpecification": user_spec}) ( diff --git a/tests/test_specification/test_rkme.py b/tests/test_specification/test_rkme.py index 143bf22..ba280b2 100644 --- a/tests/test_specification/test_rkme.py +++ b/tests/test_specification/test_rkme.py @@ -7,9 +7,8 @@ import unittest import tempfile import numpy as np -import learnware.specification as specification from learnware.specification import RKMETableSpecification, RKMEImageSpecification, RKMETextSpecification -from learnware.specification import generate_rkme_image_spec, generate_rkme_spec +from learnware.specification import generate_rkme_image_spec, generate_rkme_spec, generate_rkme_text_spec class TestRKME(unittest.TestCase): @@ -71,7 +70,7 @@ class TestRKME(unittest.TestCase): return text_list def _test_text_rkme(X): - rkme = specification.utils.generate_rkme_text_spec(X) + rkme = generate_rkme_text_spec(X) with tempfile.TemporaryDirectory(prefix="learnware_") as tempdir: rkme_path = os.path.join(tempdir, "rkme.json") diff --git a/tests/test_workflow/test_workflow.py b/tests/test_workflow/test_workflow.py index fea00d9..3b3579c 100644 --- a/tests/test_workflow/test_workflow.py +++ b/tests/test_workflow/test_workflow.py @@ -13,7 +13,7 @@ from shutil import copyfile, rmtree import learnware from learnware.market import EasyMarket, BaseUserInfo from learnware.reuse import JobSelectorReuser, AveragingReuser, EnsemblePruningReuser -import learnware.specification as specification +from learnware.specification import generate_rkme_spec, RKMETableSpecification curr_root = os.path.dirname(os.path.abspath(__file__)) @@ -57,7 +57,7 @@ class TestAllWorkflow(unittest.TestCase): joblib.dump(clf, os.path.join(dir_path, "svm.pkl")) - spec = specification.utils.generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0) + spec = generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0) spec.save(os.path.join(dir_path, "svm.json")) init_file = os.path.join(dir_path, "__init__.py") @@ -159,7 +159,7 @@ class TestAllWorkflow(unittest.TestCase): with zipfile.ZipFile(zip_path, "r") as zip_obj: zip_obj.extractall(path=unzip_dir) - user_spec = specification.RKMETableSpecification() + user_spec = RKMETableSpecification() user_spec.load(os.path.join(unzip_dir, "svm.json")) user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={"RKMETableSpecification": user_spec}) ( @@ -185,7 +185,7 @@ class TestAllWorkflow(unittest.TestCase): X, y = load_digits(return_X_y=True) train_X, data_X, train_y, data_y = train_test_split(X, y, test_size=0.3, shuffle=True) - stat_spec = specification.utils.generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0) + stat_spec = generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0) user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={"RKMETableSpecification": stat_spec}) _, _, _, mixture_learnware_list = easy_market.search_learnware(user_info) From 4d24827533fa0e05a15756658f6cdc342bd68c13 Mon Sep 17 00:00:00 2001 From: bxdd Date: Tue, 7 Nov 2023 14:58:20 +0800 Subject: [PATCH 2/6] [MNT] add traceback --- learnware/market/base.py | 2 + learnware/market/classes.py | 2 + tests/test_market/test_easy.py | 64 +++++++++++++++++++++------- tests/test_workflow/test_workflow.py | 8 +--- 4 files changed, 54 insertions(+), 22 deletions(-) diff --git a/learnware/market/base.py b/learnware/market/base.py index 9894c65..1921323 100644 --- a/learnware/market/base.py +++ b/learnware/market/base.py @@ -1,5 +1,6 @@ from __future__ import annotations +import traceback import zipfile import tempfile from typing import Tuple, Any, List, Union @@ -90,6 +91,7 @@ class LearnwareMarket: return BaseChecker.INVALID_LEARNWARE return final_status except Exception as err: + traceback.print_exc( ) logger.warning(f"Check learnware failed! Due to {err}.") return BaseChecker.INVALID_LEARNWARE diff --git a/learnware/market/classes.py b/learnware/market/classes.py index deee3d8..1808b74 100644 --- a/learnware/market/classes.py +++ b/learnware/market/classes.py @@ -1,3 +1,4 @@ +import traceback from .base import BaseChecker from ..learnware import Learnware from ..client.container import LearnwaresContainer @@ -17,6 +18,7 @@ class CondaChecker(BaseChecker): learnwares = env_container.get_learnwares_with_container() check_status = self.inner_checker(learnwares[0]) except Exception as e: + traceback.print_exc( ) logger.warning(f"Conda Checker failed due to installed learnware failed and {e}") return BaseChecker.INVALID_LEARNWARE return check_status diff --git a/tests/test_market/test_easy.py b/tests/test_market/test_easy.py index 0ca779f..5d4fb02 100644 --- a/tests/test_market/test_easy.py +++ b/tests/test_market/test_easy.py @@ -13,11 +13,12 @@ from shutil import copyfile, rmtree import learnware from learnware.market import instantiate_learnware_market, BaseUserInfo from learnware.specification import RKMETableSpecification, generate_rkme_spec +from learnware.reuse import JobSelectorReuser, AveragingReuser, EnsemblePruningReuser curr_root = os.path.dirname(os.path.abspath(__file__)) user_semantic = { - "Data": {"Values": ["Image"], "Type": "Class"}, + "Data": {"Values": ["Table"], "Type": "Class"}, "Task": { "Values": ["Classification"], "Type": "Class", @@ -26,6 +27,9 @@ user_semantic = { "Scenario": {"Values": ["Education"], "Type": "Tag"}, "Description": {"Values": "", "Type": "String"}, "Name": {"Values": "", "Type": "String"}, + "Input": { + + }, "Output": { "Dimension": 10, "Description": { @@ -103,11 +107,11 @@ class TestMarket(unittest.TestCase): semantic_spec = copy.deepcopy(user_semantic) semantic_spec["Name"]["Values"] = "learnware_%d" % (idx) semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx) + semantic_spec["Output"] = {"Dimension": 1, "Description": {"0": "The label of the hand-written digit."}} easy_market.add_learnware(zip_path, semantic_spec) print("Total Item:", len(easy_market)) assert len(easy_market) == self.learnware_num, f"The number of learnwares must be {self.learnware_num}!" - curr_inds = easy_market.get_learnware_ids() print("Available ids After Uploading Learnwares:", curr_inds) assert len(curr_inds) == self.learnware_num, f"The number of learnwares must be {self.learnware_num}!" @@ -128,31 +132,29 @@ class TestMarket(unittest.TestCase): easy_market = self.test_upload_delete_learnware(learnware_num, delete=False) print("Total Item:", len(easy_market)) assert len(easy_market) == self.learnware_num, f"The number of learnwares must be {self.learnware_num}!" + test_folder = os.path.join(curr_root, "test_semantics") + + # unzip -o -q zip_path -d unzip_dir + if os.path.exists(test_folder): + rmtree(test_folder) + os.makedirs(test_folder, exist_ok=True) + + with zipfile.ZipFile(self.zip_path_list[0], "r") as zip_obj: + zip_obj.extractall(path=test_folder) semantic_spec = copy.deepcopy(user_semantic) semantic_spec["Name"]["Values"] = f"learnware_{learnware_num - 1}" + semantic_spec["Description"]["Values"] = f"test_learnware_number_{learnware_num - 1}" user_info = BaseUserInfo(semantic_spec=semantic_spec) _, single_learnware_list, _, _ = easy_market.search_learnware(user_info) print("User info:", user_info.get_semantic_spec()) print(f"Search result:") - assert len(single_learnware_list) == 1, f"Exact semantic search failed!" for learnware in single_learnware_list: - semantic_spec1 = learnware.get_specification().get_semantic_spec() - print("Choose learnware:", learnware.id, semantic_spec1) - assert semantic_spec1["Name"]["Values"] == semantic_spec["Name"]["Values"], f"Exact semantic search failed!" + print("Choose learnware:", learnware.id, learnware.get_specification().get_semantic_spec()) - semantic_spec["Name"]["Values"] = "laernwaer" - user_info = BaseUserInfo(semantic_spec=semantic_spec) - _, single_learnware_list, _, _ = easy_market.search_learnware(user_info) - - print("User info:", user_info.get_semantic_spec()) - print(f"Search result:") - assert len(single_learnware_list) == self.learnware_num, f"Fuzzy semantic search failed!" - for learnware in single_learnware_list: - semantic_spec1 = learnware.get_specification().get_semantic_spec() - print("Choose learnware:", learnware.id, semantic_spec1) + rmtree(test_folder) # rm -r test_folder def test_stat_search(self, learnware_num=5): easy_market = self.test_upload_delete_learnware(learnware_num, delete=False) @@ -190,6 +192,35 @@ class TestMarket(unittest.TestCase): rmtree(test_folder) # rm -r test_folder + def test_learnware_reuse(self, learnware_num=5): + easy_market = self.test_upload_delete_learnware(learnware_num, delete=False) + print("Total Item:", len(easy_market)) + + X, y = load_digits(return_X_y=True) + train_X, data_X, train_y, data_y = train_test_split(X, y, test_size=0.3, shuffle=True) + + stat_spec = generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0) + user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={"RKMETableSpecification": stat_spec}) + + _, _, _, mixture_learnware_list = easy_market.search_learnware(user_info) + + # Based on user information, the learnware market returns a list of learnwares (learnware_list) + # Use jobselector reuser to reuse the searched learnwares to make prediction + reuse_job_selector = JobSelectorReuser(learnware_list=mixture_learnware_list) + job_selector_predict_y = reuse_job_selector.predict(user_data=data_X) + + # Use averaging ensemble reuser to reuse the searched learnwares to make prediction + reuse_ensemble = AveragingReuser(learnware_list=mixture_learnware_list, mode="vote_by_prob") + ensemble_predict_y = reuse_ensemble.predict(user_data=data_X) + + # Use ensemble pruning reuser to reuse the searched learnwares to make prediction + reuse_ensemble = EnsemblePruningReuser(learnware_list=mixture_learnware_list, mode="classification") + reuse_ensemble.fit(train_X[-200:], train_y[-200:]) + ensemble_pruning_predict_y = reuse_ensemble.predict(user_data=data_X) + + print("Job Selector Acc:", np.sum(np.argmax(job_selector_predict_y, axis=1) == data_y) / len(data_y)) + print("Averaging Reuser Acc:", np.sum(np.argmax(ensemble_predict_y, axis=1) == data_y) / len(data_y)) + print("Ensemble Pruning Reuser Acc:", np.sum(ensemble_pruning_predict_y == data_y) / len(data_y)) def suite(): _suite = unittest.TestSuite() @@ -197,6 +228,7 @@ def suite(): _suite.addTest(TestMarket("test_upload_delete_learnware")) _suite.addTest(TestMarket("test_search_semantics")) _suite.addTest(TestMarket("test_stat_search")) + _suite.addTest(TestMarket("test_learnware_reuse")) return _suite diff --git a/tests/test_workflow/test_workflow.py b/tests/test_workflow/test_workflow.py index 3b3579c..aaeab40 100644 --- a/tests/test_workflow/test_workflow.py +++ b/tests/test_workflow/test_workflow.py @@ -13,12 +13,12 @@ from shutil import copyfile, rmtree import learnware from learnware.market import EasyMarket, BaseUserInfo from learnware.reuse import JobSelectorReuser, AveragingReuser, EnsemblePruningReuser -from learnware.specification import generate_rkme_spec, RKMETableSpecification +from learnware.specification import RKMETableSpecification, generate_rkme_spec curr_root = os.path.dirname(os.path.abspath(__file__)) user_semantic = { - "Data": {"Values": ["Table"], "Type": "Class"}, + "Data": {"Values": ["Image"], "Type": "Class"}, "Task": { "Values": ["Classification"], "Type": "Class", @@ -96,10 +96,6 @@ class TestAllWorkflow(unittest.TestCase): semantic_spec = copy.deepcopy(user_semantic) semantic_spec["Name"]["Values"] = "learnware_%d" % (idx) semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx) - semantic_spec["Input"] = {"Dimension": 64} - semantic_spec["Input"].update( - {f"{i}": f"The value in the digit image with row is {i // 8} and col is {i % 8}." for i in range(64)} - ) semantic_spec["Output"] = {"Dimension": 1, "Description": {"0": "The label of the hand-written digit."}} easy_market.add_learnware(zip_path, semantic_spec) From 4e66754de79e0f573bc992f369ed026700663f79 Mon Sep 17 00:00:00 2001 From: bxdd Date: Tue, 7 Nov 2023 15:16:55 +0800 Subject: [PATCH 3/6] [FIX] fix bug s in test_workflow --- learnware/market/easy2/organizer.py | 2 +- tests/test_market/test_easy.py | 14 +++----------- tests/test_workflow/test_workflow.py | 2 +- 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/learnware/market/easy2/organizer.py b/learnware/market/easy2/organizer.py index 6c0327d..f6122ca 100644 --- a/learnware/market/easy2/organizer.py +++ b/learnware/market/easy2/organizer.py @@ -337,7 +337,7 @@ class EasyOrganizer(BaseOrganizer): Learnware ids """ if check_status is None: - filtered_ids = self.use_flags.keys() + filtered_ids = list(self.use_flags.keys()) elif check_status in [BaseChecker.NONUSABLE_LEARNWARE, BaseChecker.USABLE_LEARWARE]: filtered_ids = [key for key, value in self.use_flags.items() if value == check_status] else: diff --git a/tests/test_market/test_easy.py b/tests/test_market/test_easy.py index 5d4fb02..973864d 100644 --- a/tests/test_market/test_easy.py +++ b/tests/test_market/test_easy.py @@ -27,15 +27,6 @@ user_semantic = { "Scenario": {"Values": ["Education"], "Type": "Tag"}, "Description": {"Values": "", "Type": "String"}, "Name": {"Values": "", "Type": "String"}, - "Input": { - - }, - "Output": { - "Dimension": 10, - "Description": { - "0": "the probability of the label is zero", - }, - }, } @@ -47,7 +38,7 @@ class TestMarket(unittest.TestCase): def _init_learnware_market(self): """initialize learnware market""" - easy_market = instantiate_learnware_market(market_id="sklearn_digits", name="easy", rebuild=True) + easy_market = instantiate_learnware_market(market_id="sklearn_digits_easy", name="easy", rebuild=True) return easy_market def test_prepare_learnware_randomly(self, learnware_num=5): @@ -107,7 +98,8 @@ class TestMarket(unittest.TestCase): semantic_spec = copy.deepcopy(user_semantic) semantic_spec["Name"]["Values"] = "learnware_%d" % (idx) semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx) - semantic_spec["Output"] = {"Dimension": 1, "Description": {"0": "The label of the hand-written digit."}} + semantic_spec["Input"] = {"Dimension": 64, "Description": {f"{i}": f"The value in the grid {i // 8}{i % 8} of the image of hand-written digit." for i in range(64)}} + semantic_spec["Output"] = {"Dimension": 10, "Description": {f"{i}": "The probability for each digit for 0 to 9." for i in range(10)}} easy_market.add_learnware(zip_path, semantic_spec) print("Total Item:", len(easy_market)) diff --git a/tests/test_workflow/test_workflow.py b/tests/test_workflow/test_workflow.py index aaeab40..d2b9c38 100644 --- a/tests/test_workflow/test_workflow.py +++ b/tests/test_workflow/test_workflow.py @@ -96,7 +96,7 @@ class TestAllWorkflow(unittest.TestCase): semantic_spec = copy.deepcopy(user_semantic) semantic_spec["Name"]["Values"] = "learnware_%d" % (idx) semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx) - semantic_spec["Output"] = {"Dimension": 1, "Description": {"0": "The label of the hand-written digit."}} + semantic_spec["Output"] = {"Dimension": 10, "Description": {f"{i}": "The probability for each digit for 0 to 9." for i in range(10)}} easy_market.add_learnware(zip_path, semantic_spec) print("Total Item:", len(easy_market)) From abe8489e0124f25789a3bac8638b7fa3bbf664d9 Mon Sep 17 00:00:00 2001 From: bxdd Date: Tue, 7 Nov 2023 15:19:07 +0800 Subject: [PATCH 4/6] [MNT] black foramt --- .../dataset_pfs_workflow/pfs/pfs_cross_transfer.py | 4 +--- learnware/market/base.py | 2 +- learnware/market/classes.py | 2 +- tests/test_market/test_easy.py | 14 ++++++++++++-- tests/test_workflow/test_workflow.py | 5 ++++- 5 files changed, 19 insertions(+), 8 deletions(-) diff --git a/examples/dataset_pfs_workflow/pfs/pfs_cross_transfer.py b/examples/dataset_pfs_workflow/pfs/pfs_cross_transfer.py index 5f69127..93a3fa3 100644 --- a/examples/dataset_pfs_workflow/pfs/pfs_cross_transfer.py +++ b/examples/dataset_pfs_workflow/pfs/pfs_cross_transfer.py @@ -85,9 +85,7 @@ def get_split_errs(algo): split = train_xs.shape[0] - proportion_list[tmp] model.fit( - train_xs[ - split:, - ], + train_xs[split:,], train_ys[split:], eval_set=[(val_xs, val_ys)], early_stopping_rounds=50, diff --git a/learnware/market/base.py b/learnware/market/base.py index 1921323..2349614 100644 --- a/learnware/market/base.py +++ b/learnware/market/base.py @@ -91,7 +91,7 @@ class LearnwareMarket: return BaseChecker.INVALID_LEARNWARE return final_status except Exception as err: - traceback.print_exc( ) + traceback.print_exc() logger.warning(f"Check learnware failed! Due to {err}.") return BaseChecker.INVALID_LEARNWARE diff --git a/learnware/market/classes.py b/learnware/market/classes.py index 1808b74..9c99555 100644 --- a/learnware/market/classes.py +++ b/learnware/market/classes.py @@ -18,7 +18,7 @@ class CondaChecker(BaseChecker): learnwares = env_container.get_learnwares_with_container() check_status = self.inner_checker(learnwares[0]) except Exception as e: - traceback.print_exc( ) + traceback.print_exc() logger.warning(f"Conda Checker failed due to installed learnware failed and {e}") return BaseChecker.INVALID_LEARNWARE return check_status diff --git a/tests/test_market/test_easy.py b/tests/test_market/test_easy.py index 973864d..492c8a9 100644 --- a/tests/test_market/test_easy.py +++ b/tests/test_market/test_easy.py @@ -98,8 +98,17 @@ class TestMarket(unittest.TestCase): semantic_spec = copy.deepcopy(user_semantic) semantic_spec["Name"]["Values"] = "learnware_%d" % (idx) semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx) - semantic_spec["Input"] = {"Dimension": 64, "Description": {f"{i}": f"The value in the grid {i // 8}{i % 8} of the image of hand-written digit." for i in range(64)}} - semantic_spec["Output"] = {"Dimension": 10, "Description": {f"{i}": "The probability for each digit for 0 to 9." for i in range(10)}} + semantic_spec["Input"] = { + "Dimension": 64, + "Description": { + f"{i}": f"The value in the grid {i // 8}{i % 8} of the image of hand-written digit." + for i in range(64) + }, + } + semantic_spec["Output"] = { + "Dimension": 10, + "Description": {f"{i}": "The probability for each digit for 0 to 9." for i in range(10)}, + } easy_market.add_learnware(zip_path, semantic_spec) print("Total Item:", len(easy_market)) @@ -214,6 +223,7 @@ class TestMarket(unittest.TestCase): print("Averaging Reuser Acc:", np.sum(np.argmax(ensemble_predict_y, axis=1) == data_y) / len(data_y)) print("Ensemble Pruning Reuser Acc:", np.sum(ensemble_pruning_predict_y == data_y) / len(data_y)) + def suite(): _suite = unittest.TestSuite() _suite.addTest(TestMarket("test_prepare_learnware_randomly")) diff --git a/tests/test_workflow/test_workflow.py b/tests/test_workflow/test_workflow.py index d2b9c38..e9439a0 100644 --- a/tests/test_workflow/test_workflow.py +++ b/tests/test_workflow/test_workflow.py @@ -96,7 +96,10 @@ class TestAllWorkflow(unittest.TestCase): semantic_spec = copy.deepcopy(user_semantic) semantic_spec["Name"]["Values"] = "learnware_%d" % (idx) semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx) - semantic_spec["Output"] = {"Dimension": 10, "Description": {f"{i}": "The probability for each digit for 0 to 9." for i in range(10)}} + semantic_spec["Output"] = { + "Dimension": 10, + "Description": {f"{i}": "The probability for each digit for 0 to 9." for i in range(10)}, + } easy_market.add_learnware(zip_path, semantic_spec) print("Total Item:", len(easy_market)) From 6573f01e6272c273aeb0aef927cfad77af75ab7a Mon Sep 17 00:00:00 2001 From: bxdd Date: Tue, 7 Nov 2023 15:28:38 +0800 Subject: [PATCH 5/6] [MNT] modify generate_stat_spec interface --- learnware/specification/utils.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/learnware/specification/utils.py b/learnware/specification/utils.py index 09d66c1..276fbb7 100644 --- a/learnware/specification/utils.py +++ b/learnware/specification/utils.py @@ -221,7 +221,7 @@ def generate_rkme_text_spec( return rkme_text_spec -def generate_stat_spec(X: np.ndarray) -> BaseStatSpecification: +def generate_stat_spec(type="table", *args, **kwargs) -> BaseStatSpecification: """ Interface for users to generate statistical specification. Return a StatSpecification object, use .save() method to save as npy file. @@ -237,4 +237,12 @@ def generate_stat_spec(X: np.ndarray) -> BaseStatSpecification: StatSpecification A StatSpecification object """ + if type == "table": + return generate_rkme_spec(*args, **kwargs) + elif type == "text": + return generate_rkme_text_spec(*args, **kwargs) + elif type == "image": + return generate_rkme_image_spec(*args, **kwargs) + else: + raise TypeError(f"type {type} is not supported!") return None From 53e9dc1bf4f587b1d498124d5580517b2537ecc1 Mon Sep 17 00:00:00 2001 From: bxdd Date: Tue, 7 Nov 2023 15:33:05 +0800 Subject: [PATCH 6/6] [MNT] refactor specifiation module --- learnware/specification/__init__.py | 2 +- learnware/specification/module.py | 223 ++++++++++++++++++++++++++++ learnware/specification/utils.py | 221 +-------------------------- 3 files changed, 225 insertions(+), 221 deletions(-) create mode 100644 learnware/specification/module.py diff --git a/learnware/specification/__init__.py b/learnware/specification/__init__.py index b27ef5b..4ecedc3 100644 --- a/learnware/specification/__init__.py +++ b/learnware/specification/__init__.py @@ -1,4 +1,4 @@ -from .utils import generate_stat_spec, generate_rkme_spec, generate_rkme_image_spec +from .module import generate_stat_spec, generate_rkme_spec, generate_rkme_image_spec, generate_rkme_text_spec from .base import Specification, BaseStatSpecification from .regular import ( RegularStatsSpecification, diff --git a/learnware/specification/module.py b/learnware/specification/module.py new file mode 100644 index 0000000..7ae6ded --- /dev/null +++ b/learnware/specification/module.py @@ -0,0 +1,223 @@ +import torch +import numpy as np +import pandas as pd +from typing import Union, List + +from .utils import convert_to_numpy +from .base import BaseStatSpecification +from .regular import RKMETableSpecification, RKMEImageSpecification, RKMETextSpecification +from ..config import C + + +def generate_rkme_spec( + X: Union[np.ndarray, pd.DataFrame, torch.Tensor], + gamma: float = 0.1, + reduced_set_size: int = 100, + step_size: float = 0.1, + steps: int = 3, + nonnegative_beta: bool = True, + reduce: bool = True, + cuda_idx: int = None, +) -> RKMETableSpecification: + """ + Interface for users to generate Reduced Kernel Mean Embedding (RKME) specification. + Return a RKMETableSpecification object, use .save() method to save as json file. + + Parameters + ---------- + X : np.ndarray, pd.DataFrame, or torch.Tensor + Raw data in np.ndarray, pd.DataFrame, or torch.Tensor format. + The shape of X: + First dimension represents the number of samples (data points). + The remaining dimensions represent the dimensions (features) of each sample. + For example, if X has shape (100, 3), it means there are 100 samples, and each sample has 3 features. + gamma : float + Bandwidth in gaussian kernel, by default 0.1. + reduced_set_size : int + Size of the construced reduced set. + step_size : float + Step size for gradient descent in the iterative optimization. + steps : int + Total rounds in the iterative optimization. + nonnegative_beta : bool, optional + True if weights for the reduced set are intended to be kept non-negative, by default False. + reduce : bool, optional + Whether shrink original data to a smaller set, by default True + cuda_idx : int + A flag indicating whether use CUDA during RKME computation. -1 indicates CUDA not used. + None indicates that CUDA is automatically selected. + + Returns + ------- + RKMETableSpecification + A RKMETableSpecification object + """ + # Convert data type + X = convert_to_numpy(X) + X = np.ascontiguousarray(X).astype(np.float32) + + # Check reduced_set_size + max_reduced_set_size = C.max_reduced_set_size + if reduced_set_size * X[0].size > max_reduced_set_size: + reduced_set_size = max(20, max_reduced_set_size // X[0].size) + + # Check cuda_idx + if not torch.cuda.is_available() or cuda_idx == -1: + cuda_idx = -1 + else: + num_cuda_devices = torch.cuda.device_count() + if cuda_idx is None or not (cuda_idx >= 0 and cuda_idx < num_cuda_devices): + cuda_idx = 0 + + # Generate rkme spec + rkme_spec = RKMETableSpecification(gamma=gamma, cuda_idx=cuda_idx) + rkme_spec.generate_stat_spec_from_data(X, reduced_set_size, step_size, steps, nonnegative_beta, reduce) + return rkme_spec + + +def generate_rkme_image_spec( + X: Union[np.ndarray, torch.Tensor], + reduced_set_size: int = 50, + step_size: float = 0.01, + steps: int = 100, + resize: bool = True, + nonnegative_beta: bool = True, + reduce: bool = True, + verbose: bool = True, + cuda_idx: int = None, +) -> RKMEImageSpecification: + """ + Interface for users to generate Reduced Kernel Mean Embedding (RKME) specification for Image. + Return a RKMEImageSpecification object, use .save() method to save as json file. + + Parameters + ---------- + X : np.ndarray, or torch.Tensor + Raw data in np.ndarray, or torch.Tensor format. + The shape of X: [N, C, H, W] + N: Number of images. + C: Number of channels. + H: Height of images. + W: Width of images.s + For example, if X has shape (100, 3, 32, 32), it means there are 100 samples, and each sample is a 3-channel (RGB) image of size 32x32. + reduced_set_size : int + Size of the construced reduced set. + step_size : float + Step size for gradient descent in the iterative optimization. + steps : int + Total rounds in the iterative optimization. + resize : bool + Whether to scale the image to the requested size, by default True. + nonnegative_beta : bool, optional + True if weights for the reduced set are intended to be kept non-negative, by default False. + reduce : bool, optional + Whether shrink original data to a smaller set, by default True + cuda_idx : int + A flag indicating whether use CUDA during RKME computation. -1 indicates CUDA not used. + None indicates that CUDA is automatically selected. + verbose : bool, optional + Whether to print training progress, by default True + + Returns + ------- + RKMEImageSpecification + A RKMEImageSpecification object + """ + + # Check cuda_idx + if not torch.cuda.is_available() or cuda_idx == -1: + cuda_idx = -1 + else: + num_cuda_devices = torch.cuda.device_count() + if cuda_idx is None or not (0 <= cuda_idx < num_cuda_devices): + cuda_idx = 0 + + # Generate rkme spec + rkme_image_spec = RKMEImageSpecification(cuda_idx=cuda_idx) + rkme_image_spec.generate_stat_spec_from_data( + X, reduced_set_size, step_size, steps, resize, nonnegative_beta, reduce, verbose + ) + return rkme_image_spec + + +def generate_rkme_text_spec( + X: List[str], + gamma: float = 0.1, + reduced_set_size: int = 100, + step_size: float = 0.1, + steps: int = 3, + nonnegative_beta: bool = True, + reduce: bool = True, + cuda_idx: int = None, +) -> RKMETextSpecification: + """ + Interface for users to generate Reduced Kernel Mean Embedding (RKME) specification for Text. + Return a RKMETextSpecification object, use .save() method to save as json file. + + Parameters + ---------- + X : List[str] + Raw data of text. + gamma : float + Bandwidth in gaussian kernel, by default 0.1. + reduced_set_size : int + Size of the construced reduced set. + step_size : float + Step size for gradient descent in the iterative optimization. + steps : int + Total rounds in the iterative optimization. + nonnegative_beta : bool, optional + True if weights for the reduced set are intended to be kept non-negative, by default False. + reduce : bool, optional + Whether shrink original data to a smaller set, by default True + cuda_idx : int + A flag indicating whether use CUDA during RKME computation. -1 indicates CUDA not used. + None indicates that CUDA is automatically selected. + + Returns + ------- + RKMETextSpecification + A RKMETextSpecification object + """ + # Check input type + if not isinstance(X, list) or not all(isinstance(item, str) for item in X): + raise TypeError("Input data must be a list of strings.") + + # Check cuda_idx + if not torch.cuda.is_available() or cuda_idx == -1: + cuda_idx = -1 + else: + num_cuda_devices = torch.cuda.device_count() + if cuda_idx is None or not (cuda_idx >= 0 and cuda_idx < num_cuda_devices): + cuda_idx = 0 + + # Generate rkme text spec + rkme_text_spec = RKMETextSpecification(gamma=gamma, cuda_idx=cuda_idx) + rkme_text_spec.generate_stat_spec_from_data(X, reduced_set_size, step_size, steps, nonnegative_beta, reduce) + return rkme_text_spec + + +def generate_stat_spec(type="table", *args, **kwargs) -> BaseStatSpecification: + """ + Interface for users to generate statistical specification. + Return a StatSpecification object, use .save() method to save as npy file. + + Parameters + ---------- + X : np.ndarray + Raw data in np.ndarray format. + Size of array: (n*d) + + Returns + ------- + StatSpecification + A StatSpecification object + """ + if type == "table": + return generate_rkme_spec(*args, **kwargs) + elif type == "text": + return generate_rkme_text_spec(*args, **kwargs) + elif type == "image": + return generate_rkme_image_spec(*args, **kwargs) + else: + raise TypeError(f"type {type} is not supported!") diff --git a/learnware/specification/utils.py b/learnware/specification/utils.py index 276fbb7..fdf9fc0 100644 --- a/learnware/specification/utils.py +++ b/learnware/specification/utils.py @@ -1,11 +1,7 @@ import torch import numpy as np import pandas as pd -from typing import Union, List - -from .base import BaseStatSpecification -from .regular import RKMETableSpecification, RKMEImageSpecification, RKMETextSpecification -from ..config import C +from typing import Union def convert_to_numpy(data: Union[np.ndarray, pd.DataFrame, torch.Tensor]): @@ -31,218 +27,3 @@ def convert_to_numpy(data: Union[np.ndarray, pd.DataFrame, torch.Tensor]): raise TypeError( "Unsupported data format. Please provide a NumPy array, a Pandas DataFrame, or a PyTorch Tensor." ) - - -def generate_rkme_spec( - X: Union[np.ndarray, pd.DataFrame, torch.Tensor], - gamma: float = 0.1, - reduced_set_size: int = 100, - step_size: float = 0.1, - steps: int = 3, - nonnegative_beta: bool = True, - reduce: bool = True, - cuda_idx: int = None, -) -> RKMETableSpecification: - """ - Interface for users to generate Reduced Kernel Mean Embedding (RKME) specification. - Return a RKMETableSpecification object, use .save() method to save as json file. - - Parameters - ---------- - X : np.ndarray, pd.DataFrame, or torch.Tensor - Raw data in np.ndarray, pd.DataFrame, or torch.Tensor format. - The shape of X: - First dimension represents the number of samples (data points). - The remaining dimensions represent the dimensions (features) of each sample. - For example, if X has shape (100, 3), it means there are 100 samples, and each sample has 3 features. - gamma : float - Bandwidth in gaussian kernel, by default 0.1. - reduced_set_size : int - Size of the construced reduced set. - step_size : float - Step size for gradient descent in the iterative optimization. - steps : int - Total rounds in the iterative optimization. - nonnegative_beta : bool, optional - True if weights for the reduced set are intended to be kept non-negative, by default False. - reduce : bool, optional - Whether shrink original data to a smaller set, by default True - cuda_idx : int - A flag indicating whether use CUDA during RKME computation. -1 indicates CUDA not used. - None indicates that CUDA is automatically selected. - - Returns - ------- - RKMETableSpecification - A RKMETableSpecification object - """ - # Convert data type - X = convert_to_numpy(X) - X = np.ascontiguousarray(X).astype(np.float32) - - # Check reduced_set_size - max_reduced_set_size = C.max_reduced_set_size - if reduced_set_size * X[0].size > max_reduced_set_size: - reduced_set_size = max(20, max_reduced_set_size // X[0].size) - - # Check cuda_idx - if not torch.cuda.is_available() or cuda_idx == -1: - cuda_idx = -1 - else: - num_cuda_devices = torch.cuda.device_count() - if cuda_idx is None or not (cuda_idx >= 0 and cuda_idx < num_cuda_devices): - cuda_idx = 0 - - # Generate rkme spec - rkme_spec = RKMETableSpecification(gamma=gamma, cuda_idx=cuda_idx) - rkme_spec.generate_stat_spec_from_data(X, reduced_set_size, step_size, steps, nonnegative_beta, reduce) - return rkme_spec - - -def generate_rkme_image_spec( - X: Union[np.ndarray, torch.Tensor], - reduced_set_size: int = 50, - step_size: float = 0.01, - steps: int = 100, - resize: bool = True, - nonnegative_beta: bool = True, - reduce: bool = True, - verbose: bool = True, - cuda_idx: int = None, -) -> RKMEImageSpecification: - """ - Interface for users to generate Reduced Kernel Mean Embedding (RKME) specification for Image. - Return a RKMEImageSpecification object, use .save() method to save as json file. - - Parameters - ---------- - X : np.ndarray, or torch.Tensor - Raw data in np.ndarray, or torch.Tensor format. - The shape of X: [N, C, H, W] - N: Number of images. - C: Number of channels. - H: Height of images. - W: Width of images.s - For example, if X has shape (100, 3, 32, 32), it means there are 100 samples, and each sample is a 3-channel (RGB) image of size 32x32. - reduced_set_size : int - Size of the construced reduced set. - step_size : float - Step size for gradient descent in the iterative optimization. - steps : int - Total rounds in the iterative optimization. - resize : bool - Whether to scale the image to the requested size, by default True. - nonnegative_beta : bool, optional - True if weights for the reduced set are intended to be kept non-negative, by default False. - reduce : bool, optional - Whether shrink original data to a smaller set, by default True - cuda_idx : int - A flag indicating whether use CUDA during RKME computation. -1 indicates CUDA not used. - None indicates that CUDA is automatically selected. - verbose : bool, optional - Whether to print training progress, by default True - - Returns - ------- - RKMEImageSpecification - A RKMEImageSpecification object - """ - - # Check cuda_idx - if not torch.cuda.is_available() or cuda_idx == -1: - cuda_idx = -1 - else: - num_cuda_devices = torch.cuda.device_count() - if cuda_idx is None or not (0 <= cuda_idx < num_cuda_devices): - cuda_idx = 0 - - # Generate rkme spec - rkme_image_spec = RKMEImageSpecification(cuda_idx=cuda_idx) - rkme_image_spec.generate_stat_spec_from_data( - X, reduced_set_size, step_size, steps, resize, nonnegative_beta, reduce, verbose - ) - return rkme_image_spec - - -def generate_rkme_text_spec( - X: List[str], - gamma: float = 0.1, - reduced_set_size: int = 100, - step_size: float = 0.1, - steps: int = 3, - nonnegative_beta: bool = True, - reduce: bool = True, - cuda_idx: int = None, -) -> RKMETextSpecification: - """ - Interface for users to generate Reduced Kernel Mean Embedding (RKME) specification for Text. - Return a RKMETextSpecification object, use .save() method to save as json file. - - Parameters - ---------- - X : List[str] - Raw data of text. - gamma : float - Bandwidth in gaussian kernel, by default 0.1. - reduced_set_size : int - Size of the construced reduced set. - step_size : float - Step size for gradient descent in the iterative optimization. - steps : int - Total rounds in the iterative optimization. - nonnegative_beta : bool, optional - True if weights for the reduced set are intended to be kept non-negative, by default False. - reduce : bool, optional - Whether shrink original data to a smaller set, by default True - cuda_idx : int - A flag indicating whether use CUDA during RKME computation. -1 indicates CUDA not used. - None indicates that CUDA is automatically selected. - - Returns - ------- - RKMETextSpecification - A RKMETextSpecification object - """ - # Check input type - if not isinstance(X, list) or not all(isinstance(item, str) for item in X): - raise TypeError("Input data must be a list of strings.") - - # Check cuda_idx - if not torch.cuda.is_available() or cuda_idx == -1: - cuda_idx = -1 - else: - num_cuda_devices = torch.cuda.device_count() - if cuda_idx is None or not (cuda_idx >= 0 and cuda_idx < num_cuda_devices): - cuda_idx = 0 - - # Generate rkme text spec - rkme_text_spec = RKMETextSpecification(gamma=gamma, cuda_idx=cuda_idx) - rkme_text_spec.generate_stat_spec_from_data(X, reduced_set_size, step_size, steps, nonnegative_beta, reduce) - return rkme_text_spec - - -def generate_stat_spec(type="table", *args, **kwargs) -> BaseStatSpecification: - """ - Interface for users to generate statistical specification. - Return a StatSpecification object, use .save() method to save as npy file. - - Parameters - ---------- - X : np.ndarray - Raw data in np.ndarray format. - Size of array: (n*d) - - Returns - ------- - StatSpecification - A StatSpecification object - """ - if type == "table": - return generate_rkme_spec(*args, **kwargs) - elif type == "text": - return generate_rkme_text_spec(*args, **kwargs) - elif type == "image": - return generate_rkme_image_spec(*args, **kwargs) - else: - raise TypeError(f"type {type} is not supported!") - return None