diff --git a/docs/workflow/submit.rst b/docs/workflow/submit.rst index fe097c3..2d82936 100644 --- a/docs/workflow/submit.rst +++ b/docs/workflow/submit.rst @@ -80,10 +80,10 @@ the following code snippet offers guidance on how to construct and store the RKM .. code-block:: python - import learnware.specification as specification + from learnware.specification import generate_rkme_spec # generate rkme specification for digits dataset - spec = specification.utils.generate_rkme_spec(X=data_X) + spec = generate_rkme_spec(X=data_X) spec.save("stat.json") Significantly, the RKME generation process is entirely conducted on your local machine, without any involvement of cloud services, diff --git a/examples/dataset_m5_workflow/main.py b/examples/dataset_m5_workflow/main.py index 009b557..bfdbe71 100644 --- a/examples/dataset_m5_workflow/main.py +++ b/examples/dataset_m5_workflow/main.py @@ -9,9 +9,8 @@ from shutil import copyfile, rmtree import learnware from learnware.market import EasyMarket, BaseUserInfo from learnware.market import database_ops -from learnware.learnware import Learnware from learnware.reuse import JobSelectorReuser, AveragingReuser -import learnware.specification as specification +from learnware.specification import generate_rkme_spec from m5 import DataLoader from learnware.logger import get_module_logger @@ -88,7 +87,7 @@ class M5DatasetWorkflow: for idx in tqdm(idx_list): train_x, train_y, test_x, test_y = m5.get_idx_data(idx) st = time.time() - spec = specification.utils.generate_rkme_spec(X=train_x, gamma=0.1, cuda_idx=0) + spec = generate_rkme_spec(X=train_x, gamma=0.1, cuda_idx=0) ed = time.time() logger.info("Stat spec generated in %.3f s" % (ed - st)) @@ -140,7 +139,7 @@ class M5DatasetWorkflow: for idx in idx_list: train_x, train_y, test_x, test_y = m5.get_idx_data(idx) - user_spec = specification.utils.generate_rkme_spec(X=test_x, gamma=0.1, cuda_idx=0) + user_spec = generate_rkme_spec(X=test_x, gamma=0.1, cuda_idx=0) user_spec_path = f"./user_spec/user_{idx}.json" user_spec.save(user_spec_path) diff --git a/examples/dataset_pfs_workflow/main.py b/examples/dataset_pfs_workflow/main.py index b5cbdd8..abe80cd 100644 --- a/examples/dataset_pfs_workflow/main.py +++ b/examples/dataset_pfs_workflow/main.py @@ -8,10 +8,8 @@ from shutil import copyfile, rmtree import learnware from learnware.market import EasyMarket, BaseUserInfo -from learnware.market import database_ops -from learnware.learnware import Learnware from learnware.reuse import JobSelectorReuser, AveragingReuser -import learnware.specification as specification +from learnware.specification import generate_rkme_spec from pfs import Dataloader from learnware.logger import get_module_logger @@ -86,7 +84,7 @@ class PFSDatasetWorkflow: for idx in tqdm(idx_list): train_x, train_y, test_x, test_y = pfs.get_idx_data(idx) st = time.time() - spec = specification.utils.generate_rkme_spec(X=train_x, gamma=0.1, cuda_idx=0) + spec = generate_rkme_spec(X=train_x, gamma=0.1, cuda_idx=0) ed = time.time() logger.info("Stat spec generated in %.3f s" % (ed - st)) @@ -138,7 +136,7 @@ class PFSDatasetWorkflow: for idx in idx_list: train_x, train_y, test_x, test_y = pfs.get_idx_data(idx) - user_spec = specification.utils.generate_rkme_spec(X=test_x, gamma=0.1, cuda_idx=0) + user_spec = generate_rkme_spec(X=test_x, gamma=0.1, cuda_idx=0) user_spec_path = f"./user_spec/user_{idx}.json" user_spec.save(user_spec_path) diff --git a/examples/dataset_text_workflow/main.py b/examples/dataset_text_workflow/main.py index e7e1c38..406aad9 100644 --- a/examples/dataset_text_workflow/main.py +++ b/examples/dataset_text_workflow/main.py @@ -10,9 +10,7 @@ import time import pickle from learnware.market import instantiate_learnware_market, BaseUserInfo -from learnware.market import database_ops -from learnware.learnware import Learnware -import learnware.specification as specification +from learnware.specification import RKMETextSpecification from learnware.logger import get_module_logger from shutil import copyfile, rmtree @@ -99,8 +97,7 @@ def prepare_learnware(data_path, model_path, init_file_path, yaml_path, save_roo semantic_spec = semantic_specs[0] st = time.time() - # user_spec = specification.utils.generate_rkme_spec(X=X, gamma=0.1, cuda_idx=0) - user_spec = specification.RKMETextSpecification() + user_spec = RKMETextSpecification() user_spec.generate_stat_spec_from_data(X=X) ed = time.time() logger.info("Stat spec generated in %.3f s" % (ed - st)) @@ -163,10 +160,8 @@ def test_search(gamma=0.1, load_market=True): user_data = pickle.load(f) with open(user_label_path, "rb") as f: user_label = pickle.load(f) - # user_data = np.load(user_data_path) - # user_label = np.load(user_label_path) - # user_stat_spec = specification.utils.generate_rkme_spec(X=user_data, gamma=gamma, cuda_idx=0) - user_stat_spec = specification.RKMETextSpecification() + + user_stat_spec = RKMETextSpecification() user_stat_spec.generate_stat_spec_from_data(X=user_data) user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={"RKMETextSpecification": user_stat_spec}) logger.info("Searching Market for user: %d" % (i)) diff --git a/examples/workflow_by_code/main.py b/examples/workflow_by_code/main.py index 2f62db0..39b1b6c 100644 --- a/examples/workflow_by_code/main.py +++ b/examples/workflow_by_code/main.py @@ -12,8 +12,7 @@ from shutil import copyfile, rmtree import learnware from learnware.market import EasyMarket, BaseUserInfo from learnware.reuse import JobSelectorReuser, AveragingReuser -import learnware.specification as specification -from learnware.utils import get_module_by_module_path +from learnware.specification import generate_rkme_spec, RKMETableSpecification curr_root = os.path.dirname(os.path.abspath(__file__)) @@ -54,7 +53,7 @@ class LearnwareMarketWorkflow: joblib.dump(clf, os.path.join(dir_path, "svm.pkl")) - spec = specification.utils.generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0) + spec = generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0) spec.save(os.path.join(dir_path, "svm.json")) init_file = os.path.join(dir_path, "__init__.py") @@ -148,7 +147,7 @@ class LearnwareMarketWorkflow: with zipfile.ZipFile(zip_path, "r") as zip_obj: zip_obj.extractall(path=unzip_dir) - user_spec = specification.RKMETableSpecification() + user_spec = RKMETableSpecification() user_spec.load(os.path.join(unzip_dir, "svm.json")) user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={"RKMETableSpecification": user_spec}) ( @@ -174,7 +173,7 @@ class LearnwareMarketWorkflow: X, y = load_digits(return_X_y=True) _, data_X, _, data_y = train_test_split(X, y, test_size=0.3, shuffle=True) - stat_spec = specification.utils.generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0) + stat_spec = generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0) user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={"RKMETableSpecification": stat_spec}) _, _, _, mixture_learnware_list = easy_market.search_learnware(user_info) diff --git a/learnware/client/container.py b/learnware/client/container.py index 6d67e0e..e7ae9ca 100644 --- a/learnware/client/container.py +++ b/learnware/client/container.py @@ -500,6 +500,7 @@ class LearnwaresContainer: learnwares: Union[List[Learnware], Learnware], cleanup=True, mode="conda", + ignore_error=True, ): """The initializaiton method for base reuser @@ -519,6 +520,7 @@ class LearnwaresContainer: assert self.mode in {"conda", "docker"}, f"mode must be in ['conda', 'docker'], should not be {self.mode}" self.learnware_list = learnwares self.cleanup = cleanup + self.ignore_error = ignore_error def __enter__(self): if self.mode == "conda": @@ -548,7 +550,7 @@ class LearnwaresContainer: model_list = [_learnware.get_model() for _learnware in self.learnware_containers] with ThreadPoolExecutor(max_workers=max(os.cpu_count() // 2, 1)) as executor: - results = executor.map(self._initialize_model_container, model_list) + results = executor.map(self._initialize_model_container, model_list, [self.ignore_error] * len(model_list)) self.results = list(results) if sum(self.results) < len(self.learnware_list): @@ -556,11 +558,6 @@ class LearnwaresContainer: f"{len(self.learnware_list) - sum(results)} of {len(self.learnware_list)} learnwares init failed! This learnware will be ignored" ) - # if not self.cleanup and self.mode == "docker": - # _model_docker_container = self.learnware_containers[0].get_model() - # _model_docker_container.cleanup_flag = True - # atexit.register(_model_docker_container.remove_env) - return self def __exit__(self, exc_type, exc_val, exc_tb): @@ -572,7 +569,7 @@ class LearnwaresContainer: model_list = [_learnware.get_model() for _learnware in self.learnware_containers] with ThreadPoolExecutor(max_workers=max(os.cpu_count() // 2, 1)) as executor: - executor.map(self._destroy_model_container, model_list) + executor.map(self._destroy_model_container, model_list, [self.ignore_error] * len(model_list)) self.learnware_containers = None self.results = None @@ -581,20 +578,24 @@ class LearnwaresContainer: ModelDockerContainer._destroy_docker_container(self._docker_container) @staticmethod - def _initialize_model_container(model: ModelCondaContainer): + def _initialize_model_container(model: ModelCondaContainer, ignore_error=True): try: model.init_and_setup_env() except Exception as err: - logger.error(f"build env {model.conda_env} failed due to {err}") + if not ignore_error: + raise err + logger.warning(f"build env {model.conda_env} failed due to {err}") return False return True @staticmethod - def _destroy_model_container(model: ModelCondaContainer): + def _destroy_model_container(model: ModelCondaContainer, ignore_error=True): try: model.remove_env() except Exception as err: - logger.error(f"remove env {model.conda_env} failed due to {err}") + if not ignore_error: + raise err + logger.warning(f"remove env {model.conda_env} failed due to {err}") return False return True diff --git a/learnware/client/learnware_client.py b/learnware/client/learnware_client.py index df9121d..7bc23bd 100644 --- a/learnware/client/learnware_client.py +++ b/learnware/client/learnware_client.py @@ -312,7 +312,7 @@ class LearnwareClient: tempdir = self.tempdir_list[-1].name zip_path = os.path.join(tempdir, f"{str(uuid.uuid4())}.zip") self.download_learnware(_learnware_id, zip_path) - return zip_path, _get_learnware_by_path(zip_path, tempdir=tempdir) + return _get_learnware_by_path(zip_path, tempdir=tempdir) def _get_learnware_by_path(_learnware_zippath, tempdir=None): if tempdir is None: @@ -342,16 +342,13 @@ class LearnwareClient: return learnware.get_learnware_from_dirpath(learnware_id, semantic_specification, tempdir) learnware_list = [] - zip_paths = [] if learnware_path is not None: - if isinstance(learnware_path, str): - zip_paths = [learnware_path] - elif isinstance(learnware_path, list): - zip_paths = learnware_path + zip_paths = [learnware_path] if isinstance(learnware_path, str) else learnware_path for zip_path in zip_paths: learnware_obj = _get_learnware_by_path(zip_path) learnware_list.append(learnware_obj) + elif learnware_id is not None: if isinstance(learnware_id, str): id_list = [learnware_id] @@ -359,8 +356,7 @@ class LearnwareClient: id_list = learnware_id for idx in id_list: - zip_path, learnware_obj = _get_learnware_by_id(idx) - zip_paths.append(zip_path) + learnware_obj = _get_learnware_by_id(idx) learnware_list.append(learnware_obj) if runnable_option is not None: diff --git a/learnware/client/package_utils.py b/learnware/client/package_utils.py index 3c85767..0f26955 100644 --- a/learnware/client/package_utils.py +++ b/learnware/client/package_utils.py @@ -4,6 +4,7 @@ import yaml import tempfile import subprocess from typing import List, Tuple +from . import utils from ..logger import get_module_logger @@ -15,7 +16,7 @@ def try_to_run(args, timeout=5, retry=5): sucess = False for i in range(retry): try: - subprocess.check_call(args=args, timeout=timeout, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + utils.system_execute(args=args, timeout=timeout, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) sucess = True break except subprocess.TimeoutExpired as e: @@ -120,7 +121,7 @@ def filter_nonexist_conda_packages(packages: list) -> Tuple[List[str], List[str] if not any(package.startswith("python=") for package in exist_packages): exist_packages = ["python=3.8"] + exist_packages - + return exist_packages, nonexist_packages else: return packages, [] diff --git a/learnware/client/utils.py b/learnware/client/utils.py index bb7b16f..09e5142 100644 --- a/learnware/client/utils.py +++ b/learnware/client/utils.py @@ -9,17 +9,29 @@ from .package_utils import filter_nonexist_conda_packages_file, filter_nonexist_ logger = get_module_logger(module_name="client_utils") -def system_execute(args, timeout=None): - com_process = subprocess.run(args, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, timeout=timeout) +def system_execute(args, timeout=None, env=None, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE): + if env is None: + env = os.environ.copy() + pass + + if isinstance(args, str): + pass + else: + args = " ".join(args) + pass + + com_process = subprocess.run(args, stdout=stdout, stderr=stderr, timeout=timeout, env=env, shell=True) + try: com_process.check_returncode() except subprocess.CalledProcessError as err: - logger.error(f"System Execute Error: {com_process.stderr.decode()}") + logger.warning(f"System Execute Error: {com_process.stderr.decode()}") raise err def remove_enviroment(conda_env): system_execute(args=["conda", "env", "remove", "-n", f"{conda_env}"]) + logger.info(f"The learnware conda env [{conda_env}] is removed.") def install_environment(learnware_dirpath, conda_env): diff --git a/learnware/learnware/__init__.py b/learnware/learnware/__init__.py index dcd3dc9..738a55d 100644 --- a/learnware/learnware/__init__.py +++ b/learnware/learnware/__init__.py @@ -12,7 +12,7 @@ from ..config import C logger = get_module_logger("learnware.learnware") -def get_learnware_from_dirpath(id: str, semantic_spec: dict, learnware_dirpath) -> Learnware: +def get_learnware_from_dirpath(id: str, semantic_spec: dict, learnware_dirpath, ignore_error=True) -> Learnware: """Get the learnware object from dirpath, and provide the manage interface tor Learnware class Parameters @@ -67,6 +67,8 @@ def get_learnware_from_dirpath(id: str, semantic_spec: dict, learnware_dirpath) learnware_spec.update_semantic_spec(copy.deepcopy(semantic_spec)) except Exception as e: + if not ignore_error: + raise e logger.warning(f"Load Learnware {id} failed! Due to {repr(e)}") return None diff --git a/learnware/market/base.py b/learnware/market/base.py index 8a2f445..2349614 100644 --- a/learnware/market/base.py +++ b/learnware/market/base.py @@ -1,5 +1,6 @@ from __future__ import annotations +import traceback import zipfile import tempfile from typing import Tuple, Any, List, Union @@ -81,8 +82,6 @@ class LearnwareMarket: pending_learnware = get_learnware_from_dirpath( id="pending", semantic_spec=semantic_spec, learnware_dirpath=tempdir ) - checker_names = list(self.learnware_checker.keys()) if checker_names is None else checker_names - for name in checker_names: checker = self.learnware_checker[name] check_status = checker(pending_learnware) @@ -92,6 +91,7 @@ class LearnwareMarket: return BaseChecker.INVALID_LEARNWARE return final_status except Exception as err: + traceback.print_exc() logger.warning(f"Check learnware failed! Due to {err}.") return BaseChecker.INVALID_LEARNWARE @@ -115,6 +115,7 @@ class LearnwareMarket: - str indicating model_id - int indicating the final learnware check_status """ + checker_names = list(self.learnware_checker.keys()) if checker_names is None else checker_names check_status = self.check_learnware(zip_path, semantic_spec, checker_names) return self.learnware_organizer.add_learnware( zip_path=zip_path, semantic_spec=semantic_spec, check_status=check_status, **kwargs @@ -172,12 +173,13 @@ class LearnwareMarket: int The final learnware check_status. """ - zip_path = self.get_learnware_path_by_ids(id) if zip_path is None else zip_path + zip_path = self.get_learnware_zip_path_by_ids(id) if zip_path is None else zip_path semantic_spec = ( self.get_learnware_by_ids(id).get_specification().get_semantic_spec() if semantic_spec is None else semantic_spec ) + checker_names = list(self.learnware_checker.keys()) if checker_names is None else checker_names update_status = self.check_learnware(zip_path, semantic_spec, checker_names) check_status = ( update_status if check_status is None or update_status == BaseChecker.INVALID_LEARNWARE else check_status @@ -223,8 +225,11 @@ class LearnwareMarket: """ return self.learnware_organizer.get_learnwares(top, check_status, **kwargs) - def get_learnware_path_by_ids(self, ids: Union[str, List[str]], **kwargs) -> Union[Learnware, List[Learnware]]: - return self.learnware_organizer.get_learnware_path_by_ids(ids, **kwargs) + def get_learnware_zip_path_by_ids(self, ids: Union[str, List[str]], **kwargs) -> Union[Learnware, List[Learnware]]: + return self.learnware_organizer.get_learnware_zip_path_by_ids(ids, **kwargs) + + def get_learnware_dir_path_by_ids(self, ids: Union[str, List[str]], **kwargs) -> Union[Learnware, List[Learnware]]: + return self.learnware_organizer.get_learnware_dir_path_by_ids(ids, **kwargs) def get_learnware_by_ids(self, id: Union[str, List[str]], **kwargs) -> Union[Learnware, List[Learnware]]: return self.learnware_organizer.get_learnware_by_ids(id, **kwargs) @@ -331,7 +336,7 @@ class BaseOrganizer: """ raise NotImplementedError("get_learnware_by_ids is not implemented in BaseOrganizer") - def get_learnware_path_by_ids(self, ids: Union[str, List[str]]) -> Union[Learnware, List[Learnware]]: + def get_learnware_zip_path_by_ids(self, ids: Union[str, List[str]]) -> Union[Learnware, List[Learnware]]: """Get Zipped Learnware file by id Parameters @@ -347,7 +352,25 @@ class BaseOrganizer: Return the path for target learnware or list of path. None for Learnware NOT Found. """ - raise NotImplementedError("get_learnware_path_by_ids is not implemented in BaseOrganizer") + raise NotImplementedError("get_learnware_zip_path_by_ids is not implemented in BaseOrganizer") + + def get_learnware_dir_path_by_ids(self, ids: Union[str, List[str]]) -> Union[Learnware, List[Learnware]]: + """Get Learnware dir path by id + + Parameters + ---------- + ids : Union[str, List[str]] + Give a id or a list of ids + str: id of targer learware + List[str]: A list of ids of target learnwares + + Returns + ------- + Union[Learnware, List[Learnware]] + Return the dir path for target learnware or list of path. + None for Learnware NOT Found. + """ + raise NotImplementedError("get_learnware_dir_path_by_ids is not implemented in BaseOrganizer") def get_learnware_ids(self, top: int = None, check_status: int = None) -> List[str]: """get the list of learnware ids diff --git a/learnware/market/classes.py b/learnware/market/classes.py index 8fa9ab7..9c99555 100644 --- a/learnware/market/classes.py +++ b/learnware/market/classes.py @@ -1,3 +1,4 @@ +import traceback from .base import BaseChecker from ..learnware import Learnware from ..client.container import LearnwaresContainer @@ -12,11 +13,12 @@ class CondaChecker(BaseChecker): super(CondaChecker, self).__init__(**kwargs) def __call__(self, learnware: Learnware) -> int: - with LearnwaresContainer(learnware) as env_container: - if not all(env_container.get_learnware_flags()): - logger.warning(f"Conda Checker failed due to installed learnware failed") - return BaseChecker.INVALID_LEARNWARE - learnwares = env_container.get_learnwares_with_container() - check_status = self.inner_checker(learnwares[0]) - + try: + with LearnwaresContainer(learnware, ignore_error=False) as env_container: + learnwares = env_container.get_learnwares_with_container() + check_status = self.inner_checker(learnwares[0]) + except Exception as e: + traceback.print_exc() + logger.warning(f"Conda Checker failed due to installed learnware failed and {e}") + return BaseChecker.INVALID_LEARNWARE return check_status diff --git a/learnware/market/easy.py b/learnware/market/easy.py index 098887b..b57e1c0 100644 --- a/learnware/market/easy.py +++ b/learnware/market/easy.py @@ -926,7 +926,7 @@ class EasyMarket(LearnwareMarket): logger.warning("Learnware ID '%s' NOT Found!" % (ids)) return None - def get_learnware_path_by_ids(self, ids: Union[str, List[str]]) -> Union[Learnware, List[Learnware]]: + def get_learnware_zip_path_by_ids(self, ids: Union[str, List[str]]) -> Union[Learnware, List[Learnware]]: """Get Zipped Learnware file by id Parameters diff --git a/learnware/market/easy2/checker.py b/learnware/market/easy2/checker.py index f9ae5df..589f3e2 100644 --- a/learnware/market/easy2/checker.py +++ b/learnware/market/easy2/checker.py @@ -95,14 +95,13 @@ class EasyStatChecker(BaseChecker): # Check input shape input_shape = learnware_model.input_shape - ## WHY: why write this? if semantic_spec["Data"]["Values"][0] == "Table" and input_shape != ( int(semantic_spec["Input"]["Dimension"]), ): logger.warning("input shapes of model and semantic specifications are different") return self.INVALID_LEARNWARE - spec_type = parse_specification_type(learnware.get_specification()) + spec_type = parse_specification_type(learnware.get_specification().stat_spec) if spec_type is None: logger.warning(f"No valid specification is found in stat spec {spec_type}") return self.INVALID_LEARNWARE @@ -119,13 +118,12 @@ class EasyStatChecker(BaseChecker): inputs = np.random.randint(0, 255, size=(10, *input_shape)) else: raise ValueError(f"not supported spec type for spec_type = {spec_type}") - outputs = learnware.predict(inputs) - # Check output - if outputs.ndim == 1: - outputs = outputs.reshape(-1, 1) - if outputs.shape[1:] != learnware_model.output_shape: - logger.warning(f"The learnware [{learnware.id}] output dimention mismatch!") + # Check output + try: + outputs = learnware.predict(inputs) + except Exception: + logger.warning(f"learnware {learnware} prediction method is not valid!") return self.INVALID_LEARNWARE if semantic_spec["Task"]["Values"][0] in ("Classification", "Regression", "Feature Extraction"): @@ -136,11 +134,15 @@ class EasyStatChecker(BaseChecker): logger.warning(f"The learnware [{learnware.id}] output must be np.ndarray or torch.Tensor!") return self.INVALID_LEARNWARE + if outputs.ndim == 1: + outputs = outputs.reshape(-1, 1) # Check output shape - if outputs[0].shape != learnware_model.output_shape or learnware_model.output_shape != int( - semantic_spec["Output"]["Dimension"] + if outputs[0].shape != learnware_model.output_shape or learnware_model.output_shape != ( + int(semantic_spec["Output"]["Dimension"]), ): - logger.warning(f"The learnware [{learnware.id}] output dimention mismatch!") + logger.warning( + f"The learnware [{learnware.id}] output dimension mismatch!, where pred_shape={outputs[0].shape}, model_shape={learnware_model.output_shape}, semantic_shape={(int(semantic_spec['Output']['Dimension']), )}" + ) return self.INVALID_LEARNWARE except Exception as e: diff --git a/learnware/market/easy2/database_ops.py b/learnware/market/easy2/database_ops.py index 25f02e9..087e6fd 100644 --- a/learnware/market/easy2/database_ops.py +++ b/learnware/market/easy2/database_ops.py @@ -166,10 +166,9 @@ class DatabaseOperations(object): # assert new_learnware is not None zip_list[id] = zip_path folder_list[id] = folder_path - use_flags[id] = use_flag + use_flags[id] = int(use_flag) max_count = max(max_count, int(id)) pass - return learnware_list, zip_list, folder_list, use_flags, max_count + 1 pass diff --git a/learnware/market/easy2/organizer.py b/learnware/market/easy2/organizer.py index cef55fc..f6122ca 100644 --- a/learnware/market/easy2/organizer.py +++ b/learnware/market/easy2/organizer.py @@ -256,7 +256,7 @@ class EasyOrganizer(BaseOrganizer): logger.warning("Learnware ID '%s' NOT Found!" % (ids)) return None - def get_learnware_path_by_ids(self, ids: Union[str, List[str]]) -> Union[Learnware, List[Learnware]]: + def get_learnware_zip_path_by_ids(self, ids: Union[str, List[str]]) -> Union[Learnware, List[Learnware]]: """Get Zipped Learnware file by id Parameters @@ -288,6 +288,38 @@ class EasyOrganizer(BaseOrganizer): logger.warning("Learnware ID '%s' NOT Found!" % (ids)) return None + def get_learnware_dir_path_by_ids(self, ids: Union[str, List[str]]) -> Union[Learnware, List[Learnware]]: + """Get Learnware dir path by id + + Parameters + ---------- + ids : Union[str, List[str]] + Give a id or a list of ids + str: id of targer learware + List[str]: A list of ids of target learnwares + + Returns + ------- + Union[Learnware, List[Learnware]] + Return the dir path for target learnware or list of path. + None for Learnware NOT Found. + """ + if isinstance(ids, list): + ret = [] + for id in ids: + if id in self.learnware_folder_list: + ret.append(self.learnware_folder_list[id]) + else: + logger.warning("Learnware ID '%s' NOT Found!" % (id)) + ret.append(None) + return ret + else: + try: + return self.learnware_folder_list[ids] + except: + logger.warning("Learnware ID '%s' NOT Found!" % (ids)) + return None + def get_learnware_ids(self, top: int = None, check_status: int = None) -> List[str]: """Get learnware ids @@ -305,11 +337,14 @@ class EasyOrganizer(BaseOrganizer): Learnware ids """ if check_status is None: - filtered_ids = self.use_flags.keys() - elif check_status is True: - filtered_ids = [key for key, value in self.use_flags.items() if value == BaseChecker.USABLE_LEARWARE] - elif check_status is False: - filtered_ids = [key for key, value in self.use_flags.items() if value == BaseChecker.NONUSABLE_LEARNWARE] + filtered_ids = list(self.use_flags.keys()) + elif check_status in [BaseChecker.NONUSABLE_LEARNWARE, BaseChecker.USABLE_LEARWARE]: + filtered_ids = [key for key, value in self.use_flags.items() if value == check_status] + else: + logger.warning( + f"check_status must be in [{BaseChecker.NONUSABLE_LEARNWARE}, {BaseChecker.USABLE_LEARWARE}]!" + ) + return None if top is None: return filtered_ids diff --git a/learnware/market/easy2/searcher.py b/learnware/market/easy2/searcher.py index 8feb5d9..f86c06c 100644 --- a/learnware/market/easy2/searcher.py +++ b/learnware/market/easy2/searcher.py @@ -565,7 +565,7 @@ class EasyStatSearcher(BaseSearcher): max_search_num: int = 5, search_method: str = "greedy", ) -> Tuple[List[float], List[Learnware], float, List[Learnware]]: - self.stat_spec_type = parse_specification_type(stat_spec=user_info.stat_info) + self.stat_spec_type = parse_specification_type(stat_specs=user_info.stat_info) if self.stat_spec_type is None: raise KeyError("No supported stat specification is given in the user info") @@ -646,7 +646,7 @@ class EasySearcher(BaseSearcher): if len(learnware_list) == 0: return [], [], 0.0, [] - if parse_specification_type(stat_spec=user_info.stat_info) is not None: + if parse_specification_type(stat_specs=user_info.stat_info) is not None: return self.stat_searcher(learnware_list, user_info, max_search_num, search_method) else: return None, learnware_list, 0.0, None diff --git a/learnware/market/utils.py b/learnware/market/utils.py index c0cc319..76d41b9 100644 --- a/learnware/market/utils.py +++ b/learnware/market/utils.py @@ -2,9 +2,8 @@ from ..specification import Specification def parse_specification_type( - stat_spec: Specification, spec_list=["RKMETableSpecification", "RKMETextSpecification", "RKMEImageSpecification"] + stat_specs: dict, spec_list=["RKMETableSpecification", "RKMETextSpecification", "RKMEImageSpecification"] ): - stat_specs = stat_spec.stat_spec for spec in spec_list: if spec in stat_specs: return spec diff --git a/learnware/reuse/job_selector.py b/learnware/reuse/job_selector.py index 7503b4a..5e3a71f 100644 --- a/learnware/reuse/job_selector.py +++ b/learnware/reuse/job_selector.py @@ -11,7 +11,7 @@ from .base import BaseReuser from ..market.utils import parse_specification_type from ..learnware import Learnware from ..specification import RKMETableSpecification, RKMETextSpecification -from ..specification.utils import generate_rkme_spec +from ..specification import generate_rkme_spec from ..logger import get_module_logger logger = get_module_logger("job_selector_reuse") @@ -49,7 +49,7 @@ class JobSelectorReuser(BaseReuser): """ raw_user_data = user_data if isinstance(user_data[0], str): - stat_spec_type = parse_specification_type(self.learnware_list[0].get_specification()) + stat_spec_type = parse_specification_type(self.learnware_list[0].get_specification().stat_spec) assert ( stat_spec_type == "RKMETextSpecification" ), "stat_spec_type must be 'RKMETextSpecification' when user data is the List of string." @@ -97,7 +97,7 @@ class JobSelectorReuser(BaseReuser): user_data_num = len(user_data) return np.array([0] * user_data_num) else: - stat_spec_type = parse_specification_type(self.learnware_list[0].get_specification()) + stat_spec_type = parse_specification_type(self.learnware_list[0].get_specification().stat_spec) learnware_rkme_spec_list = [ learnware.specification.get_stat_spec_by_name(stat_spec_type) for learnware in self.learnware_list ] diff --git a/learnware/specification/__init__.py b/learnware/specification/__init__.py index c999210..90c8758 100644 --- a/learnware/specification/__init__.py +++ b/learnware/specification/__init__.py @@ -1,4 +1,4 @@ -from .utils import generate_stat_spec, generate_rkme_spec, generate_rkme_image_spec +from .module import generate_stat_spec, generate_rkme_spec, generate_rkme_image_spec, generate_rkme_text_spec from .base import Specification, BaseStatSpecification from .regular import ( RegularStatsSpecification, diff --git a/learnware/specification/module.py b/learnware/specification/module.py new file mode 100644 index 0000000..7ae6ded --- /dev/null +++ b/learnware/specification/module.py @@ -0,0 +1,223 @@ +import torch +import numpy as np +import pandas as pd +from typing import Union, List + +from .utils import convert_to_numpy +from .base import BaseStatSpecification +from .regular import RKMETableSpecification, RKMEImageSpecification, RKMETextSpecification +from ..config import C + + +def generate_rkme_spec( + X: Union[np.ndarray, pd.DataFrame, torch.Tensor], + gamma: float = 0.1, + reduced_set_size: int = 100, + step_size: float = 0.1, + steps: int = 3, + nonnegative_beta: bool = True, + reduce: bool = True, + cuda_idx: int = None, +) -> RKMETableSpecification: + """ + Interface for users to generate Reduced Kernel Mean Embedding (RKME) specification. + Return a RKMETableSpecification object, use .save() method to save as json file. + + Parameters + ---------- + X : np.ndarray, pd.DataFrame, or torch.Tensor + Raw data in np.ndarray, pd.DataFrame, or torch.Tensor format. + The shape of X: + First dimension represents the number of samples (data points). + The remaining dimensions represent the dimensions (features) of each sample. + For example, if X has shape (100, 3), it means there are 100 samples, and each sample has 3 features. + gamma : float + Bandwidth in gaussian kernel, by default 0.1. + reduced_set_size : int + Size of the construced reduced set. + step_size : float + Step size for gradient descent in the iterative optimization. + steps : int + Total rounds in the iterative optimization. + nonnegative_beta : bool, optional + True if weights for the reduced set are intended to be kept non-negative, by default False. + reduce : bool, optional + Whether shrink original data to a smaller set, by default True + cuda_idx : int + A flag indicating whether use CUDA during RKME computation. -1 indicates CUDA not used. + None indicates that CUDA is automatically selected. + + Returns + ------- + RKMETableSpecification + A RKMETableSpecification object + """ + # Convert data type + X = convert_to_numpy(X) + X = np.ascontiguousarray(X).astype(np.float32) + + # Check reduced_set_size + max_reduced_set_size = C.max_reduced_set_size + if reduced_set_size * X[0].size > max_reduced_set_size: + reduced_set_size = max(20, max_reduced_set_size // X[0].size) + + # Check cuda_idx + if not torch.cuda.is_available() or cuda_idx == -1: + cuda_idx = -1 + else: + num_cuda_devices = torch.cuda.device_count() + if cuda_idx is None or not (cuda_idx >= 0 and cuda_idx < num_cuda_devices): + cuda_idx = 0 + + # Generate rkme spec + rkme_spec = RKMETableSpecification(gamma=gamma, cuda_idx=cuda_idx) + rkme_spec.generate_stat_spec_from_data(X, reduced_set_size, step_size, steps, nonnegative_beta, reduce) + return rkme_spec + + +def generate_rkme_image_spec( + X: Union[np.ndarray, torch.Tensor], + reduced_set_size: int = 50, + step_size: float = 0.01, + steps: int = 100, + resize: bool = True, + nonnegative_beta: bool = True, + reduce: bool = True, + verbose: bool = True, + cuda_idx: int = None, +) -> RKMEImageSpecification: + """ + Interface for users to generate Reduced Kernel Mean Embedding (RKME) specification for Image. + Return a RKMEImageSpecification object, use .save() method to save as json file. + + Parameters + ---------- + X : np.ndarray, or torch.Tensor + Raw data in np.ndarray, or torch.Tensor format. + The shape of X: [N, C, H, W] + N: Number of images. + C: Number of channels. + H: Height of images. + W: Width of images.s + For example, if X has shape (100, 3, 32, 32), it means there are 100 samples, and each sample is a 3-channel (RGB) image of size 32x32. + reduced_set_size : int + Size of the construced reduced set. + step_size : float + Step size for gradient descent in the iterative optimization. + steps : int + Total rounds in the iterative optimization. + resize : bool + Whether to scale the image to the requested size, by default True. + nonnegative_beta : bool, optional + True if weights for the reduced set are intended to be kept non-negative, by default False. + reduce : bool, optional + Whether shrink original data to a smaller set, by default True + cuda_idx : int + A flag indicating whether use CUDA during RKME computation. -1 indicates CUDA not used. + None indicates that CUDA is automatically selected. + verbose : bool, optional + Whether to print training progress, by default True + + Returns + ------- + RKMEImageSpecification + A RKMEImageSpecification object + """ + + # Check cuda_idx + if not torch.cuda.is_available() or cuda_idx == -1: + cuda_idx = -1 + else: + num_cuda_devices = torch.cuda.device_count() + if cuda_idx is None or not (0 <= cuda_idx < num_cuda_devices): + cuda_idx = 0 + + # Generate rkme spec + rkme_image_spec = RKMEImageSpecification(cuda_idx=cuda_idx) + rkme_image_spec.generate_stat_spec_from_data( + X, reduced_set_size, step_size, steps, resize, nonnegative_beta, reduce, verbose + ) + return rkme_image_spec + + +def generate_rkme_text_spec( + X: List[str], + gamma: float = 0.1, + reduced_set_size: int = 100, + step_size: float = 0.1, + steps: int = 3, + nonnegative_beta: bool = True, + reduce: bool = True, + cuda_idx: int = None, +) -> RKMETextSpecification: + """ + Interface for users to generate Reduced Kernel Mean Embedding (RKME) specification for Text. + Return a RKMETextSpecification object, use .save() method to save as json file. + + Parameters + ---------- + X : List[str] + Raw data of text. + gamma : float + Bandwidth in gaussian kernel, by default 0.1. + reduced_set_size : int + Size of the construced reduced set. + step_size : float + Step size for gradient descent in the iterative optimization. + steps : int + Total rounds in the iterative optimization. + nonnegative_beta : bool, optional + True if weights for the reduced set are intended to be kept non-negative, by default False. + reduce : bool, optional + Whether shrink original data to a smaller set, by default True + cuda_idx : int + A flag indicating whether use CUDA during RKME computation. -1 indicates CUDA not used. + None indicates that CUDA is automatically selected. + + Returns + ------- + RKMETextSpecification + A RKMETextSpecification object + """ + # Check input type + if not isinstance(X, list) or not all(isinstance(item, str) for item in X): + raise TypeError("Input data must be a list of strings.") + + # Check cuda_idx + if not torch.cuda.is_available() or cuda_idx == -1: + cuda_idx = -1 + else: + num_cuda_devices = torch.cuda.device_count() + if cuda_idx is None or not (cuda_idx >= 0 and cuda_idx < num_cuda_devices): + cuda_idx = 0 + + # Generate rkme text spec + rkme_text_spec = RKMETextSpecification(gamma=gamma, cuda_idx=cuda_idx) + rkme_text_spec.generate_stat_spec_from_data(X, reduced_set_size, step_size, steps, nonnegative_beta, reduce) + return rkme_text_spec + + +def generate_stat_spec(type="table", *args, **kwargs) -> BaseStatSpecification: + """ + Interface for users to generate statistical specification. + Return a StatSpecification object, use .save() method to save as npy file. + + Parameters + ---------- + X : np.ndarray + Raw data in np.ndarray format. + Size of array: (n*d) + + Returns + ------- + StatSpecification + A StatSpecification object + """ + if type == "table": + return generate_rkme_spec(*args, **kwargs) + elif type == "text": + return generate_rkme_text_spec(*args, **kwargs) + elif type == "image": + return generate_rkme_image_spec(*args, **kwargs) + else: + raise TypeError(f"type {type} is not supported!") diff --git a/learnware/specification/utils.py b/learnware/specification/utils.py index 09d66c1..fdf9fc0 100644 --- a/learnware/specification/utils.py +++ b/learnware/specification/utils.py @@ -1,11 +1,7 @@ import torch import numpy as np import pandas as pd -from typing import Union, List - -from .base import BaseStatSpecification -from .regular import RKMETableSpecification, RKMEImageSpecification, RKMETextSpecification -from ..config import C +from typing import Union def convert_to_numpy(data: Union[np.ndarray, pd.DataFrame, torch.Tensor]): @@ -31,210 +27,3 @@ def convert_to_numpy(data: Union[np.ndarray, pd.DataFrame, torch.Tensor]): raise TypeError( "Unsupported data format. Please provide a NumPy array, a Pandas DataFrame, or a PyTorch Tensor." ) - - -def generate_rkme_spec( - X: Union[np.ndarray, pd.DataFrame, torch.Tensor], - gamma: float = 0.1, - reduced_set_size: int = 100, - step_size: float = 0.1, - steps: int = 3, - nonnegative_beta: bool = True, - reduce: bool = True, - cuda_idx: int = None, -) -> RKMETableSpecification: - """ - Interface for users to generate Reduced Kernel Mean Embedding (RKME) specification. - Return a RKMETableSpecification object, use .save() method to save as json file. - - Parameters - ---------- - X : np.ndarray, pd.DataFrame, or torch.Tensor - Raw data in np.ndarray, pd.DataFrame, or torch.Tensor format. - The shape of X: - First dimension represents the number of samples (data points). - The remaining dimensions represent the dimensions (features) of each sample. - For example, if X has shape (100, 3), it means there are 100 samples, and each sample has 3 features. - gamma : float - Bandwidth in gaussian kernel, by default 0.1. - reduced_set_size : int - Size of the construced reduced set. - step_size : float - Step size for gradient descent in the iterative optimization. - steps : int - Total rounds in the iterative optimization. - nonnegative_beta : bool, optional - True if weights for the reduced set are intended to be kept non-negative, by default False. - reduce : bool, optional - Whether shrink original data to a smaller set, by default True - cuda_idx : int - A flag indicating whether use CUDA during RKME computation. -1 indicates CUDA not used. - None indicates that CUDA is automatically selected. - - Returns - ------- - RKMETableSpecification - A RKMETableSpecification object - """ - # Convert data type - X = convert_to_numpy(X) - X = np.ascontiguousarray(X).astype(np.float32) - - # Check reduced_set_size - max_reduced_set_size = C.max_reduced_set_size - if reduced_set_size * X[0].size > max_reduced_set_size: - reduced_set_size = max(20, max_reduced_set_size // X[0].size) - - # Check cuda_idx - if not torch.cuda.is_available() or cuda_idx == -1: - cuda_idx = -1 - else: - num_cuda_devices = torch.cuda.device_count() - if cuda_idx is None or not (cuda_idx >= 0 and cuda_idx < num_cuda_devices): - cuda_idx = 0 - - # Generate rkme spec - rkme_spec = RKMETableSpecification(gamma=gamma, cuda_idx=cuda_idx) - rkme_spec.generate_stat_spec_from_data(X, reduced_set_size, step_size, steps, nonnegative_beta, reduce) - return rkme_spec - - -def generate_rkme_image_spec( - X: Union[np.ndarray, torch.Tensor], - reduced_set_size: int = 50, - step_size: float = 0.01, - steps: int = 100, - resize: bool = True, - nonnegative_beta: bool = True, - reduce: bool = True, - verbose: bool = True, - cuda_idx: int = None, -) -> RKMEImageSpecification: - """ - Interface for users to generate Reduced Kernel Mean Embedding (RKME) specification for Image. - Return a RKMEImageSpecification object, use .save() method to save as json file. - - Parameters - ---------- - X : np.ndarray, or torch.Tensor - Raw data in np.ndarray, or torch.Tensor format. - The shape of X: [N, C, H, W] - N: Number of images. - C: Number of channels. - H: Height of images. - W: Width of images.s - For example, if X has shape (100, 3, 32, 32), it means there are 100 samples, and each sample is a 3-channel (RGB) image of size 32x32. - reduced_set_size : int - Size of the construced reduced set. - step_size : float - Step size for gradient descent in the iterative optimization. - steps : int - Total rounds in the iterative optimization. - resize : bool - Whether to scale the image to the requested size, by default True. - nonnegative_beta : bool, optional - True if weights for the reduced set are intended to be kept non-negative, by default False. - reduce : bool, optional - Whether shrink original data to a smaller set, by default True - cuda_idx : int - A flag indicating whether use CUDA during RKME computation. -1 indicates CUDA not used. - None indicates that CUDA is automatically selected. - verbose : bool, optional - Whether to print training progress, by default True - - Returns - ------- - RKMEImageSpecification - A RKMEImageSpecification object - """ - - # Check cuda_idx - if not torch.cuda.is_available() or cuda_idx == -1: - cuda_idx = -1 - else: - num_cuda_devices = torch.cuda.device_count() - if cuda_idx is None or not (0 <= cuda_idx < num_cuda_devices): - cuda_idx = 0 - - # Generate rkme spec - rkme_image_spec = RKMEImageSpecification(cuda_idx=cuda_idx) - rkme_image_spec.generate_stat_spec_from_data( - X, reduced_set_size, step_size, steps, resize, nonnegative_beta, reduce, verbose - ) - return rkme_image_spec - - -def generate_rkme_text_spec( - X: List[str], - gamma: float = 0.1, - reduced_set_size: int = 100, - step_size: float = 0.1, - steps: int = 3, - nonnegative_beta: bool = True, - reduce: bool = True, - cuda_idx: int = None, -) -> RKMETextSpecification: - """ - Interface for users to generate Reduced Kernel Mean Embedding (RKME) specification for Text. - Return a RKMETextSpecification object, use .save() method to save as json file. - - Parameters - ---------- - X : List[str] - Raw data of text. - gamma : float - Bandwidth in gaussian kernel, by default 0.1. - reduced_set_size : int - Size of the construced reduced set. - step_size : float - Step size for gradient descent in the iterative optimization. - steps : int - Total rounds in the iterative optimization. - nonnegative_beta : bool, optional - True if weights for the reduced set are intended to be kept non-negative, by default False. - reduce : bool, optional - Whether shrink original data to a smaller set, by default True - cuda_idx : int - A flag indicating whether use CUDA during RKME computation. -1 indicates CUDA not used. - None indicates that CUDA is automatically selected. - - Returns - ------- - RKMETextSpecification - A RKMETextSpecification object - """ - # Check input type - if not isinstance(X, list) or not all(isinstance(item, str) for item in X): - raise TypeError("Input data must be a list of strings.") - - # Check cuda_idx - if not torch.cuda.is_available() or cuda_idx == -1: - cuda_idx = -1 - else: - num_cuda_devices = torch.cuda.device_count() - if cuda_idx is None or not (cuda_idx >= 0 and cuda_idx < num_cuda_devices): - cuda_idx = 0 - - # Generate rkme text spec - rkme_text_spec = RKMETextSpecification(gamma=gamma, cuda_idx=cuda_idx) - rkme_text_spec.generate_stat_spec_from_data(X, reduced_set_size, step_size, steps, nonnegative_beta, reduce) - return rkme_text_spec - - -def generate_stat_spec(X: np.ndarray) -> BaseStatSpecification: - """ - Interface for users to generate statistical specification. - Return a StatSpecification object, use .save() method to save as npy file. - - Parameters - ---------- - X : np.ndarray - Raw data in np.ndarray format. - Size of array: (n*d) - - Returns - ------- - StatSpecification - A StatSpecification object - """ - return None diff --git a/tests/test_learnware_client/test_all_learnware.py b/tests/test_learnware_client/test_all_learnware.py index bdeea39..c91222d 100644 --- a/tests/test_learnware_client/test_all_learnware.py +++ b/tests/test_learnware_client/test_all_learnware.py @@ -43,7 +43,7 @@ class TestAllLearnware(unittest.TestCase): failed_ids.append(idx) print(f"check learnware {idx} failed!!!") - print(f"failed learnware ids: {failed_ids}") + print(f"The currently failed learnware ids: {failed_ids}") if __name__ == "__main__": diff --git a/tests/test_market/test_easy.py b/tests/test_market/test_easy.py index bb03839..492c8a9 100644 --- a/tests/test_market/test_easy.py +++ b/tests/test_market/test_easy.py @@ -12,12 +12,13 @@ from shutil import copyfile, rmtree import learnware from learnware.market import instantiate_learnware_market, BaseUserInfo -import learnware.specification as specification +from learnware.specification import RKMETableSpecification, generate_rkme_spec +from learnware.reuse import JobSelectorReuser, AveragingReuser, EnsemblePruningReuser curr_root = os.path.dirname(os.path.abspath(__file__)) user_semantic = { - "Data": {"Values": ["Image"], "Type": "Class"}, + "Data": {"Values": ["Table"], "Type": "Class"}, "Task": { "Values": ["Classification"], "Type": "Class", @@ -26,12 +27,6 @@ user_semantic = { "Scenario": {"Values": ["Education"], "Type": "Tag"}, "Description": {"Values": "", "Type": "String"}, "Name": {"Values": "", "Type": "String"}, - "Output": { - "Dimension": 10, - "Description": { - "0": "the probability of the label is zero", - }, - }, } @@ -43,7 +38,7 @@ class TestMarket(unittest.TestCase): def _init_learnware_market(self): """initialize learnware market""" - easy_market = instantiate_learnware_market(market_id="sklearn_digits", name="easy", rebuild=True) + easy_market = instantiate_learnware_market(market_id="sklearn_digits_easy", name="easy", rebuild=True) return easy_market def test_prepare_learnware_randomly(self, learnware_num=5): @@ -62,7 +57,7 @@ class TestMarket(unittest.TestCase): joblib.dump(clf, os.path.join(dir_path, "svm.pkl")) - spec = specification.utils.generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0) + spec = generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0) spec.save(os.path.join(dir_path, "svm.json")) init_file = os.path.join(dir_path, "__init__.py") @@ -103,11 +98,21 @@ class TestMarket(unittest.TestCase): semantic_spec = copy.deepcopy(user_semantic) semantic_spec["Name"]["Values"] = "learnware_%d" % (idx) semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx) + semantic_spec["Input"] = { + "Dimension": 64, + "Description": { + f"{i}": f"The value in the grid {i // 8}{i % 8} of the image of hand-written digit." + for i in range(64) + }, + } + semantic_spec["Output"] = { + "Dimension": 10, + "Description": {f"{i}": "The probability for each digit for 0 to 9." for i in range(10)}, + } easy_market.add_learnware(zip_path, semantic_spec) print("Total Item:", len(easy_market)) assert len(easy_market) == self.learnware_num, f"The number of learnwares must be {self.learnware_num}!" - curr_inds = easy_market.get_learnware_ids() print("Available ids After Uploading Learnwares:", curr_inds) assert len(curr_inds) == self.learnware_num, f"The number of learnwares must be {self.learnware_num}!" @@ -128,31 +133,29 @@ class TestMarket(unittest.TestCase): easy_market = self.test_upload_delete_learnware(learnware_num, delete=False) print("Total Item:", len(easy_market)) assert len(easy_market) == self.learnware_num, f"The number of learnwares must be {self.learnware_num}!" + test_folder = os.path.join(curr_root, "test_semantics") + + # unzip -o -q zip_path -d unzip_dir + if os.path.exists(test_folder): + rmtree(test_folder) + os.makedirs(test_folder, exist_ok=True) + + with zipfile.ZipFile(self.zip_path_list[0], "r") as zip_obj: + zip_obj.extractall(path=test_folder) semantic_spec = copy.deepcopy(user_semantic) semantic_spec["Name"]["Values"] = f"learnware_{learnware_num - 1}" + semantic_spec["Description"]["Values"] = f"test_learnware_number_{learnware_num - 1}" user_info = BaseUserInfo(semantic_spec=semantic_spec) _, single_learnware_list, _, _ = easy_market.search_learnware(user_info) print("User info:", user_info.get_semantic_spec()) print(f"Search result:") - assert len(single_learnware_list) == 1, f"Exact semantic search failed!" for learnware in single_learnware_list: - semantic_spec1 = learnware.get_specification().get_semantic_spec() - print("Choose learnware:", learnware.id, semantic_spec1) - assert semantic_spec1["Name"]["Values"] == semantic_spec["Name"]["Values"], f"Exact semantic search failed!" + print("Choose learnware:", learnware.id, learnware.get_specification().get_semantic_spec()) - semantic_spec["Name"]["Values"] = "laernwaer" - user_info = BaseUserInfo(semantic_spec=semantic_spec) - _, single_learnware_list, _, _ = easy_market.search_learnware(user_info) - - print("User info:", user_info.get_semantic_spec()) - print(f"Search result:") - assert len(single_learnware_list) == self.learnware_num, f"Fuzzy semantic search failed!" - for learnware in single_learnware_list: - semantic_spec1 = learnware.get_specification().get_semantic_spec() - print("Choose learnware:", learnware.id, semantic_spec1) + rmtree(test_folder) # rm -r test_folder def test_stat_search(self, learnware_num=5): easy_market = self.test_upload_delete_learnware(learnware_num, delete=False) @@ -170,7 +173,7 @@ class TestMarket(unittest.TestCase): with zipfile.ZipFile(zip_path, "r") as zip_obj: zip_obj.extractall(path=unzip_dir) - user_spec = specification.rkme.RKMETableSpecification() + user_spec = RKMETableSpecification() user_spec.load(os.path.join(unzip_dir, "svm.json")) user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={"RKMETableSpecification": user_spec}) ( @@ -190,6 +193,36 @@ class TestMarket(unittest.TestCase): rmtree(test_folder) # rm -r test_folder + def test_learnware_reuse(self, learnware_num=5): + easy_market = self.test_upload_delete_learnware(learnware_num, delete=False) + print("Total Item:", len(easy_market)) + + X, y = load_digits(return_X_y=True) + train_X, data_X, train_y, data_y = train_test_split(X, y, test_size=0.3, shuffle=True) + + stat_spec = generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0) + user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={"RKMETableSpecification": stat_spec}) + + _, _, _, mixture_learnware_list = easy_market.search_learnware(user_info) + + # Based on user information, the learnware market returns a list of learnwares (learnware_list) + # Use jobselector reuser to reuse the searched learnwares to make prediction + reuse_job_selector = JobSelectorReuser(learnware_list=mixture_learnware_list) + job_selector_predict_y = reuse_job_selector.predict(user_data=data_X) + + # Use averaging ensemble reuser to reuse the searched learnwares to make prediction + reuse_ensemble = AveragingReuser(learnware_list=mixture_learnware_list, mode="vote_by_prob") + ensemble_predict_y = reuse_ensemble.predict(user_data=data_X) + + # Use ensemble pruning reuser to reuse the searched learnwares to make prediction + reuse_ensemble = EnsemblePruningReuser(learnware_list=mixture_learnware_list, mode="classification") + reuse_ensemble.fit(train_X[-200:], train_y[-200:]) + ensemble_pruning_predict_y = reuse_ensemble.predict(user_data=data_X) + + print("Job Selector Acc:", np.sum(np.argmax(job_selector_predict_y, axis=1) == data_y) / len(data_y)) + print("Averaging Reuser Acc:", np.sum(np.argmax(ensemble_predict_y, axis=1) == data_y) / len(data_y)) + print("Ensemble Pruning Reuser Acc:", np.sum(ensemble_pruning_predict_y == data_y) / len(data_y)) + def suite(): _suite = unittest.TestSuite() @@ -197,6 +230,7 @@ def suite(): _suite.addTest(TestMarket("test_upload_delete_learnware")) _suite.addTest(TestMarket("test_search_semantics")) _suite.addTest(TestMarket("test_stat_search")) + _suite.addTest(TestMarket("test_learnware_reuse")) return _suite diff --git a/tests/test_market/test_hetero_market/example_learnwares/config.py b/tests/test_market/test_hetero_market/example_learnwares/config.py new file mode 100644 index 0000000..6c8459a --- /dev/null +++ b/tests/test_market/test_hetero_market/example_learnwares/config.py @@ -0,0 +1 @@ +input_shape_list=[20, 30] # 20-input shape of example learnware 0, 30-input shape of example learnware 1 \ No newline at end of file diff --git a/tests/test_market/test_hetero_market/example_learnwares/example_learnware_0/__init__.py b/tests/test_market/test_hetero_market/example_learnwares/example_learnware_0/__init__.py new file mode 100644 index 0000000..e9c6cf0 --- /dev/null +++ b/tests/test_market/test_hetero_market/example_learnwares/example_learnware_0/__init__.py @@ -0,0 +1,22 @@ +from learnware.model import BaseModel +import numpy as np +import joblib +import os + + +class MyModel(BaseModel): + def __init__(self): + super(MyModel, self).__init__(input_shape=(20,), output_shape=(1,)) + dir_path = os.path.dirname(os.path.abspath(__file__)) + model_path=os.path.join(dir_path, "ridge.pkl") + model = joblib.load(model_path) + self.model=model + + def fit(self, X: np.ndarray, y: np.ndarray): + pass + + def predict(self, X: np.ndarray) -> np.ndarray: + return self.model.predict(X) + + def finetune(self, X: np.ndarray, y: np.ndarray): + pass \ No newline at end of file diff --git a/tests/test_market/test_hetero_market/example_learnwares/example_learnware_0/learnware.yaml b/tests/test_market/test_hetero_market/example_learnwares/example_learnware_0/learnware.yaml new file mode 100644 index 0000000..4a37a37 --- /dev/null +++ b/tests/test_market/test_hetero_market/example_learnwares/example_learnware_0/learnware.yaml @@ -0,0 +1,8 @@ +model: + class_name: MyModel + kwargs: {} +stat_specifications: + - module_path: learnware.specification + class_name: RKMETableSpecification + file_name: stat.json + kwargs: {} \ No newline at end of file diff --git a/tests/test_market/test_hetero_market/example_learnwares/example_learnware_0/requirements.txt b/tests/test_market/test_hetero_market/example_learnwares/example_learnware_0/requirements.txt new file mode 100644 index 0000000..1da1c5f --- /dev/null +++ b/tests/test_market/test_hetero_market/example_learnwares/example_learnware_0/requirements.txt @@ -0,0 +1 @@ +learnware == 0.1.0.999 \ No newline at end of file diff --git a/tests/test_market/test_hetero_market/example_learnwares/example_learnware_1/__init__.py b/tests/test_market/test_hetero_market/example_learnwares/example_learnware_1/__init__.py new file mode 100644 index 0000000..934e352 --- /dev/null +++ b/tests/test_market/test_hetero_market/example_learnwares/example_learnware_1/__init__.py @@ -0,0 +1,22 @@ +from learnware.model import BaseModel +import numpy as np +import joblib +import os + + +class MyModel(BaseModel): + def __init__(self): + super(MyModel, self).__init__(input_shape=(30,), output_shape=(1,)) + dir_path = os.path.dirname(os.path.abspath(__file__)) + model_path=os.path.join(dir_path, "ridge.pkl") + model = joblib.load(model_path) + self.model=model + + def fit(self, X: np.ndarray, y: np.ndarray): + pass + + def predict(self, X: np.ndarray) -> np.ndarray: + return self.model.predict(X) + + def finetune(self, X: np.ndarray, y: np.ndarray): + pass \ No newline at end of file diff --git a/tests/test_market/test_hetero_market/example_learnwares/example_learnware_1/learnware.yaml b/tests/test_market/test_hetero_market/example_learnwares/example_learnware_1/learnware.yaml new file mode 100644 index 0000000..4a37a37 --- /dev/null +++ b/tests/test_market/test_hetero_market/example_learnwares/example_learnware_1/learnware.yaml @@ -0,0 +1,8 @@ +model: + class_name: MyModel + kwargs: {} +stat_specifications: + - module_path: learnware.specification + class_name: RKMETableSpecification + file_name: stat.json + kwargs: {} \ No newline at end of file diff --git a/tests/test_market/test_hetero_market/example_learnwares/example_learnware_1/requirements.txt b/tests/test_market/test_hetero_market/example_learnwares/example_learnware_1/requirements.txt new file mode 100644 index 0000000..1da1c5f --- /dev/null +++ b/tests/test_market/test_hetero_market/example_learnwares/example_learnware_1/requirements.txt @@ -0,0 +1 @@ +learnware == 0.1.0.999 \ No newline at end of file diff --git a/tests/test_market/test_hetero_market/test_hetero.py b/tests/test_market/test_hetero_market/test_hetero.py new file mode 100644 index 0000000..5d08ea7 --- /dev/null +++ b/tests/test_market/test_hetero_market/test_hetero.py @@ -0,0 +1,236 @@ +import sys +import unittest +import os +import copy +import joblib +import zipfile +import numpy as np +from sklearn.linear_model import Ridge +from sklearn.datasets import make_regression +from sklearn.datasets import load_digits +from shutil import copyfile, rmtree +from multiprocessing import Pool +from learnware.client import LearnwareClient + +import learnware +from learnware.market import instantiate_learnware_market, BaseUserInfo +import learnware.specification as specification +from example_learnwares.config import input_shape_list + +curr_root = os.path.dirname(os.path.abspath(__file__)) + +user_semantic = { + "Data": {"Values": ["Image"], "Type": "Class"}, + "Task": { + "Values": ["Classification"], + "Type": "Class", + }, + "Library": {"Values": ["Scikit-learn"], "Type": "Class"}, + "Scenario": {"Values": ["Education"], "Type": "Tag"}, + "Description": {"Values": "", "Type": "String"}, + "Name": {"Values": "", "Type": "String"}, + "Output": { + "Dimension": 10, + "Description": { + "0": "the probability of the label is zero", + }, + }, +} + + +def check_learnware(learnware_name, dir_path=os.path.join(curr_root, "learnware_pool")): + print(f"Checking Learnware: {learnware_name}") + zip_file_path = os.path.join(dir_path, learnware_name) + client = LearnwareClient() + # if check_learnware doesn't raise an exception, return True, otherwise, return false + try: + client.check_learnware(zip_file_path) + return True + except Exception as e: + print(f"Learnware {learnware_name} failed the check: {e}") + return False + + +class TestMarket(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + np.random.seed(2023) + learnware.init() + + def _init_learnware_market(self): + """initialize learnware market""" + hetero_market = instantiate_learnware_market(market_id="hetero_toy", name="hetero", rebuild=True) + return hetero_market + + def test_prepare_learnware_randomly(self, learnware_num=5): + self.zip_path_list = [] + X, y = load_digits(return_X_y=True) + + for i in range(learnware_num): + dir_path = os.path.join(curr_root, "learnware_pool", "ridge_%d" % (i)) + os.makedirs(dir_path, exist_ok=True) + + print("Preparing Learnware: %d" % (i)) + + example_learnware_idx=i%2 + input_dim=input_shape_list[example_learnware_idx] + example_learnware_name="example_learnwares/example_learnware_%d" % (example_learnware_idx) + + X, y = make_regression(n_samples=5000, n_features=input_dim, noise=0.1, random_state=42) + + clf=Ridge(alpha=1.0) + clf.fit(X, y) + + joblib.dump(clf, os.path.join(dir_path, "ridge.pkl")) + + spec = specification.utils.generate_rkme_spec(X=X, gamma=0.1, cuda_idx=0) + spec.save(os.path.join(dir_path, "stat.json")) + + init_file = os.path.join(dir_path, "__init__.py") + copyfile( + os.path.join(curr_root, example_learnware_name, "__init__.py"), init_file + ) # cp example_init.py init_file + + yaml_file = os.path.join(dir_path, "learnware.yaml") + copyfile(os.path.join(curr_root, example_learnware_name, "learnware.yaml"), yaml_file) # cp example.yaml yaml_file + + env_file = os.path.join(dir_path, "requirements.txt") + copyfile(os.path.join(curr_root, example_learnware_name, "requirements.txt"), env_file) + + zip_file = dir_path + ".zip" + # zip -q -r -j zip_file dir_path + with zipfile.ZipFile(zip_file, "w") as zip_obj: + for foldername, subfolders, filenames in os.walk(dir_path): + for filename in filenames: + file_path = os.path.join(foldername, filename) + zip_info = zipfile.ZipInfo(filename) + zip_info.compress_type = zipfile.ZIP_STORED + with open(file_path, "rb") as file: + zip_obj.writestr(zip_info, file.read()) + + rmtree(dir_path) # rm -r dir_path + + def test_generated_learnwares(self): + curr_root = os.path.dirname(os.path.abspath(__file__)) + dir_path = os.path.join(curr_root, "learnware_pool") + + # Execute multi-process checking using Pool + with Pool() as pool: + results = pool.starmap(check_learnware, [(name, dir_path) for name in os.listdir(dir_path)]) + + # Use an assert statement to ensure that all checks return True + self.assertTrue(all(results), "Not all learnwares passed the check") + + def test_upload_delete_learnware(self, learnware_num=5, delete=True): + hetero_market = self._init_learnware_market() + self.test_prepare_learnware_randomly(learnware_num) + self.learnware_num = learnware_num + + print("Total Item:", len(hetero_market)) + assert len(hetero_market) == 0, f"The market should be empty!" + + for idx, zip_path in enumerate(self.zip_path_list): + semantic_spec = copy.deepcopy(user_semantic) + semantic_spec["Name"]["Values"] = "learnware_%d" % (idx) + semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx) + hetero_market.add_learnware(zip_path, semantic_spec) + + print("Total Item:", len(hetero_market)) + assert len(hetero_market) == self.learnware_num, f"The number of learnwares must be {self.learnware_num}!" + + curr_ids = hetero_market.get_learnware_ids() + print("Available ids After Uploading Learnwares:", curr_ids) + assert len(curr_ids) == self.learnware_num, f"The number of learnwares must be {self.learnware_num}!" + + if delete: + for learnware_id in curr_ids: + hetero_market.delete_learnware(learnware_id) + self.learnware_num -= 1 + assert len(hetero_market) == self.learnware_num, f"The number of learnwares must be {self.learnware_num}!" + + curr_ids = hetero_market.get_learnware_ids() + print("Available ids After Deleting Learnwares:", curr_ids) + assert len(curr_ids) == 0, f"The market should be empty!" + + return hetero_market + + # def test_search_semantics(self, learnware_num=5): + # easy_market = self.test_upload_delete_learnware(learnware_num, delete=False) + # print("Total Item:", len(easy_market)) + # assert len(easy_market) == self.learnware_num, f"The number of learnwares must be {self.learnware_num}!" + + # semantic_spec = copy.deepcopy(user_semantic) + # semantic_spec["Name"]["Values"] = f"learnware_{learnware_num - 1}" + + # user_info = BaseUserInfo(semantic_spec=semantic_spec) + # _, single_learnware_list, _, _ = easy_market.search_learnware(user_info) + + # print("User info:", user_info.get_semantic_spec()) + # print(f"Search result:") + # assert len(single_learnware_list) == 1, f"Exact semantic search failed!" + # for learnware in single_learnware_list: + # semantic_spec1 = learnware.get_specification().get_semantic_spec() + # print("Choose learnware:", learnware.id, semantic_spec1) + # assert semantic_spec1["Name"]["Values"] == semantic_spec["Name"]["Values"], f"Exact semantic search failed!" + + # semantic_spec["Name"]["Values"] = "laernwaer" + # user_info = BaseUserInfo(semantic_spec=semantic_spec) + # _, single_learnware_list, _, _ = easy_market.search_learnware(user_info) + + # print("User info:", user_info.get_semantic_spec()) + # print(f"Search result:") + # assert len(single_learnware_list) == self.learnware_num, f"Fuzzy semantic search failed!" + # for learnware in single_learnware_list: + # semantic_spec1 = learnware.get_specification().get_semantic_spec() + # print("Choose learnware:", learnware.id, semantic_spec1) + + # def test_stat_search(self, learnware_num=5): + # easy_market = self.test_upload_delete_learnware(learnware_num, delete=False) + # print("Total Item:", len(easy_market)) + + # test_folder = os.path.join(curr_root, "test_stat") + + # for idx, zip_path in enumerate(self.zip_path_list): + # unzip_dir = os.path.join(test_folder, f"{idx}") + + # # unzip -o -q zip_path -d unzip_dir + # if os.path.exists(unzip_dir): + # rmtree(unzip_dir) + # os.makedirs(unzip_dir, exist_ok=True) + # with zipfile.ZipFile(zip_path, "r") as zip_obj: + # zip_obj.extractall(path=unzip_dir) + + # user_spec = specification.rkme.RKMETableSpecification() + # user_spec.load(os.path.join(unzip_dir, "svm.json")) + # user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={"RKMETableSpecification": user_spec}) + # ( + # sorted_score_list, + # single_learnware_list, + # mixture_score, + # mixture_learnware_list, + # ) = easy_market.search_learnware(user_info) + + # assert len(single_learnware_list) == self.learnware_num, f"Statistical search failed!" + # print(f"search result of user{idx}:") + # for score, learnware in zip(sorted_score_list, single_learnware_list): + # print(f"score: {score}, learnware_id: {learnware.id}") + # print(f"mixture_score: {mixture_score}\n") + # mixture_id = " ".join([learnware.id for learnware in mixture_learnware_list]) + # print(f"mixture_learnware: {mixture_id}\n") + + # rmtree(test_folder) # rm -r test_folder + + +def suite(): + _suite = unittest.TestSuite() + _suite.addTest(TestMarket("test_prepare_learnware_randomly")) + _suite.addTest(TestMarket("test_generated_learnwares")) + # _suite.addTest(TestMarket("test_upload_delete_learnware")) + # _suite.addTest(TestMarket("test_search_semantics")) + # _suite.addTest(TestMarket("test_stat_search")) + return _suite + + +if __name__ == "__main__": + runner = unittest.TextTestRunner() + runner.run(suite()) diff --git a/tests/test_specification/test_rkme.py b/tests/test_specification/test_rkme.py index 143bf22..ba280b2 100644 --- a/tests/test_specification/test_rkme.py +++ b/tests/test_specification/test_rkme.py @@ -7,9 +7,8 @@ import unittest import tempfile import numpy as np -import learnware.specification as specification from learnware.specification import RKMETableSpecification, RKMEImageSpecification, RKMETextSpecification -from learnware.specification import generate_rkme_image_spec, generate_rkme_spec +from learnware.specification import generate_rkme_image_spec, generate_rkme_spec, generate_rkme_text_spec class TestRKME(unittest.TestCase): @@ -71,7 +70,7 @@ class TestRKME(unittest.TestCase): return text_list def _test_text_rkme(X): - rkme = specification.utils.generate_rkme_text_spec(X) + rkme = generate_rkme_text_spec(X) with tempfile.TemporaryDirectory(prefix="learnware_") as tempdir: rkme_path = os.path.join(tempdir, "rkme.json") diff --git a/tests/test_workflow/test_workflow.py b/tests/test_workflow/test_workflow.py index fea00d9..e9439a0 100644 --- a/tests/test_workflow/test_workflow.py +++ b/tests/test_workflow/test_workflow.py @@ -13,12 +13,12 @@ from shutil import copyfile, rmtree import learnware from learnware.market import EasyMarket, BaseUserInfo from learnware.reuse import JobSelectorReuser, AveragingReuser, EnsemblePruningReuser -import learnware.specification as specification +from learnware.specification import RKMETableSpecification, generate_rkme_spec curr_root = os.path.dirname(os.path.abspath(__file__)) user_semantic = { - "Data": {"Values": ["Table"], "Type": "Class"}, + "Data": {"Values": ["Image"], "Type": "Class"}, "Task": { "Values": ["Classification"], "Type": "Class", @@ -57,7 +57,7 @@ class TestAllWorkflow(unittest.TestCase): joblib.dump(clf, os.path.join(dir_path, "svm.pkl")) - spec = specification.utils.generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0) + spec = generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0) spec.save(os.path.join(dir_path, "svm.json")) init_file = os.path.join(dir_path, "__init__.py") @@ -96,11 +96,10 @@ class TestAllWorkflow(unittest.TestCase): semantic_spec = copy.deepcopy(user_semantic) semantic_spec["Name"]["Values"] = "learnware_%d" % (idx) semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx) - semantic_spec["Input"] = {"Dimension": 64} - semantic_spec["Input"].update( - {f"{i}": f"The value in the digit image with row is {i // 8} and col is {i % 8}." for i in range(64)} - ) - semantic_spec["Output"] = {"Dimension": 1, "Description": {"0": "The label of the hand-written digit."}} + semantic_spec["Output"] = { + "Dimension": 10, + "Description": {f"{i}": "The probability for each digit for 0 to 9." for i in range(10)}, + } easy_market.add_learnware(zip_path, semantic_spec) print("Total Item:", len(easy_market)) @@ -159,7 +158,7 @@ class TestAllWorkflow(unittest.TestCase): with zipfile.ZipFile(zip_path, "r") as zip_obj: zip_obj.extractall(path=unzip_dir) - user_spec = specification.RKMETableSpecification() + user_spec = RKMETableSpecification() user_spec.load(os.path.join(unzip_dir, "svm.json")) user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={"RKMETableSpecification": user_spec}) ( @@ -185,7 +184,7 @@ class TestAllWorkflow(unittest.TestCase): X, y = load_digits(return_X_y=True) train_X, data_X, train_y, data_y = train_test_split(X, y, test_size=0.3, shuffle=True) - stat_spec = specification.utils.generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0) + stat_spec = generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0) user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={"RKMETableSpecification": stat_spec}) _, _, _, mixture_learnware_list = easy_market.search_learnware(user_info)