From 8a31f1369453ca5378affdb081386b45ce42557d Mon Sep 17 00:00:00 2001 From: bxdd Date: Sat, 4 Nov 2023 03:36:02 +0800 Subject: [PATCH 01/27] [DOC] del annot --- learnware/market/easy2/checker.py | 1 - 1 file changed, 1 deletion(-) diff --git a/learnware/market/easy2/checker.py b/learnware/market/easy2/checker.py index f9ae5df..68aa02f 100644 --- a/learnware/market/easy2/checker.py +++ b/learnware/market/easy2/checker.py @@ -95,7 +95,6 @@ class EasyStatChecker(BaseChecker): # Check input shape input_shape = learnware_model.input_shape - ## WHY: why write this? if semantic_spec["Data"]["Values"][0] == "Table" and input_shape != ( int(semantic_spec["Input"]["Dimension"]), ): From bb8f2e8fd9d2147020c71e158bf50908c1fb81ee Mon Sep 17 00:00:00 2001 From: bxdd Date: Sat, 4 Nov 2023 04:06:07 +0800 Subject: [PATCH 02/27] [MNT] support raise error when init learnware --- learnware/learnware/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/learnware/learnware/__init__.py b/learnware/learnware/__init__.py index dcd3dc9..af6d45e 100644 --- a/learnware/learnware/__init__.py +++ b/learnware/learnware/__init__.py @@ -12,7 +12,7 @@ from ..config import C logger = get_module_logger("learnware.learnware") -def get_learnware_from_dirpath(id: str, semantic_spec: dict, learnware_dirpath) -> Learnware: +def get_learnware_from_dirpath(id: str, semantic_spec: dict, learnware_dirpath, raise_error=False) -> Learnware: """Get the learnware object from dirpath, and provide the manage interface tor Learnware class Parameters @@ -67,6 +67,8 @@ def get_learnware_from_dirpath(id: str, semantic_spec: dict, learnware_dirpath) learnware_spec.update_semantic_spec(copy.deepcopy(semantic_spec)) except Exception as e: + if raise_error: + raise e logger.warning(f"Load Learnware {id} failed! Due to {repr(e)}") return None From 8cdce7565caac1e67f8f703e02b08fb58001d7ad Mon Sep 17 00:00:00 2001 From: Gene Date: Sat, 4 Nov 2023 09:55:56 +0800 Subject: [PATCH 03/27] [MNT] modify details --- learnware/client/container.py | 5 ----- learnware/client/learnware_client.py | 12 ++++-------- learnware/client/utils.py | 1 + 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/learnware/client/container.py b/learnware/client/container.py index 6d67e0e..acc1499 100644 --- a/learnware/client/container.py +++ b/learnware/client/container.py @@ -556,11 +556,6 @@ class LearnwaresContainer: f"{len(self.learnware_list) - sum(results)} of {len(self.learnware_list)} learnwares init failed! This learnware will be ignored" ) - # if not self.cleanup and self.mode == "docker": - # _model_docker_container = self.learnware_containers[0].get_model() - # _model_docker_container.cleanup_flag = True - # atexit.register(_model_docker_container.remove_env) - return self def __exit__(self, exc_type, exc_val, exc_tb): diff --git a/learnware/client/learnware_client.py b/learnware/client/learnware_client.py index a5d20b3..aa95e7d 100644 --- a/learnware/client/learnware_client.py +++ b/learnware/client/learnware_client.py @@ -316,7 +316,7 @@ class LearnwareClient: tempdir = self.tempdir_list[-1].name zip_path = os.path.join(tempdir, f"{str(uuid.uuid4())}.zip") self.download_learnware(_learnware_id, zip_path) - return zip_path, _get_learnware_by_path(zip_path, tempdir=tempdir) + return _get_learnware_by_path(zip_path, tempdir=tempdir) def _get_learnware_by_path(_learnware_zippath, tempdir=None): if tempdir is None: @@ -346,16 +346,13 @@ class LearnwareClient: return learnware.get_learnware_from_dirpath(learnware_id, semantic_specification, tempdir) learnware_list = [] - zip_paths = [] if learnware_path is not None: - if isinstance(learnware_path, str): - zip_paths = [learnware_path] - elif isinstance(learnware_path, list): - zip_paths = learnware_path + zip_paths = [learnware_path] if isinstance(learnware_path, str) else learnware_path for zip_path in zip_paths: learnware_obj = _get_learnware_by_path(zip_path) learnware_list.append(learnware_obj) + elif learnware_id is not None: if isinstance(learnware_id, str): id_list = [learnware_id] @@ -363,8 +360,7 @@ class LearnwareClient: id_list = learnware_id for idx in id_list: - zip_path, learnware_obj = _get_learnware_by_id(idx) - zip_paths.append(zip_path) + learnware_obj = _get_learnware_by_id(idx) learnware_list.append(learnware_obj) if runnable_option is not None: diff --git a/learnware/client/utils.py b/learnware/client/utils.py index bb7b16f..fff1475 100644 --- a/learnware/client/utils.py +++ b/learnware/client/utils.py @@ -20,6 +20,7 @@ def system_execute(args, timeout=None): def remove_enviroment(conda_env): system_execute(args=["conda", "env", "remove", "-n", f"{conda_env}"]) + logger.info(f"The learnware conda env [{conda_env}] is removed.") def install_environment(learnware_dirpath, conda_env): From ede79b001e139f77189588fbdb97a0ed971aaaf4 Mon Sep 17 00:00:00 2001 From: Gene Date: Sat, 4 Nov 2023 16:10:02 +0800 Subject: [PATCH 04/27] [MNT] format code by black --- learnware/client/package_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/learnware/client/package_utils.py b/learnware/client/package_utils.py index 3c85767..30ce777 100644 --- a/learnware/client/package_utils.py +++ b/learnware/client/package_utils.py @@ -120,7 +120,7 @@ def filter_nonexist_conda_packages(packages: list) -> Tuple[List[str], List[str] if not any(package.startswith("python=") for package in exist_packages): exist_packages = ["python=3.8"] + exist_packages - + return exist_packages, nonexist_packages else: return packages, [] From 30167e65e19454898908f08d8051f886e87fd274 Mon Sep 17 00:00:00 2001 From: Gene Date: Sat, 4 Nov 2023 16:21:27 +0800 Subject: [PATCH 05/27] [FIX] fix bugs in get_learnware_ids --- learnware/market/easy2/organizer.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/learnware/market/easy2/organizer.py b/learnware/market/easy2/organizer.py index cef55fc..02998d8 100644 --- a/learnware/market/easy2/organizer.py +++ b/learnware/market/easy2/organizer.py @@ -306,10 +306,13 @@ class EasyOrganizer(BaseOrganizer): """ if check_status is None: filtered_ids = self.use_flags.keys() - elif check_status is True: - filtered_ids = [key for key, value in self.use_flags.items() if value == BaseChecker.USABLE_LEARWARE] - elif check_status is False: - filtered_ids = [key for key, value in self.use_flags.items() if value == BaseChecker.NONUSABLE_LEARNWARE] + elif check_status in [BaseChecker.NONUSABLE_LEARNWARE, BaseChecker.USABLE_LEARWARE]: + filtered_ids = [key for key, value in self.use_flags.items() if value == check_status] + else: + logger.warning( + f"check_status must be in [{BaseChecker.NONUSABLE_LEARNWARE}, {BaseChecker.USABLE_LEARWARE}]!" + ) + return None if top is None: return filtered_ids From 7b307846dfa85ce55a52e06e9e1cbfa2f31af4d4 Mon Sep 17 00:00:00 2001 From: bxdd Date: Sat, 4 Nov 2023 17:24:31 +0800 Subject: [PATCH 06/27] [FIX] fix parse spec type error --- learnware/market/easy2/checker.py | 2 +- learnware/market/easy2/searcher.py | 4 ++-- learnware/market/utils.py | 3 +-- learnware/reuse/job_selector.py | 4 ++-- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/learnware/market/easy2/checker.py b/learnware/market/easy2/checker.py index 68aa02f..1ae8aa7 100644 --- a/learnware/market/easy2/checker.py +++ b/learnware/market/easy2/checker.py @@ -101,7 +101,7 @@ class EasyStatChecker(BaseChecker): logger.warning("input shapes of model and semantic specifications are different") return self.INVALID_LEARNWARE - spec_type = parse_specification_type(learnware.get_specification()) + spec_type = parse_specification_type(learnware.get_specification().stat_spec) if spec_type is None: logger.warning(f"No valid specification is found in stat spec {spec_type}") return self.INVALID_LEARNWARE diff --git a/learnware/market/easy2/searcher.py b/learnware/market/easy2/searcher.py index 8feb5d9..f86c06c 100644 --- a/learnware/market/easy2/searcher.py +++ b/learnware/market/easy2/searcher.py @@ -565,7 +565,7 @@ class EasyStatSearcher(BaseSearcher): max_search_num: int = 5, search_method: str = "greedy", ) -> Tuple[List[float], List[Learnware], float, List[Learnware]]: - self.stat_spec_type = parse_specification_type(stat_spec=user_info.stat_info) + self.stat_spec_type = parse_specification_type(stat_specs=user_info.stat_info) if self.stat_spec_type is None: raise KeyError("No supported stat specification is given in the user info") @@ -646,7 +646,7 @@ class EasySearcher(BaseSearcher): if len(learnware_list) == 0: return [], [], 0.0, [] - if parse_specification_type(stat_spec=user_info.stat_info) is not None: + if parse_specification_type(stat_specs=user_info.stat_info) is not None: return self.stat_searcher(learnware_list, user_info, max_search_num, search_method) else: return None, learnware_list, 0.0, None diff --git a/learnware/market/utils.py b/learnware/market/utils.py index c0cc319..76d41b9 100644 --- a/learnware/market/utils.py +++ b/learnware/market/utils.py @@ -2,9 +2,8 @@ from ..specification import Specification def parse_specification_type( - stat_spec: Specification, spec_list=["RKMETableSpecification", "RKMETextSpecification", "RKMEImageSpecification"] + stat_specs: dict, spec_list=["RKMETableSpecification", "RKMETextSpecification", "RKMEImageSpecification"] ): - stat_specs = stat_spec.stat_spec for spec in spec_list: if spec in stat_specs: return spec diff --git a/learnware/reuse/job_selector.py b/learnware/reuse/job_selector.py index 7503b4a..a131fd5 100644 --- a/learnware/reuse/job_selector.py +++ b/learnware/reuse/job_selector.py @@ -49,7 +49,7 @@ class JobSelectorReuser(BaseReuser): """ raw_user_data = user_data if isinstance(user_data[0], str): - stat_spec_type = parse_specification_type(self.learnware_list[0].get_specification()) + stat_spec_type = parse_specification_type(self.learnware_list[0].get_specification().stat_spec) assert ( stat_spec_type == "RKMETextSpecification" ), "stat_spec_type must be 'RKMETextSpecification' when user data is the List of string." @@ -97,7 +97,7 @@ class JobSelectorReuser(BaseReuser): user_data_num = len(user_data) return np.array([0] * user_data_num) else: - stat_spec_type = parse_specification_type(self.learnware_list[0].get_specification()) + stat_spec_type = parse_specification_type(self.learnware_list[0].get_specification().stat_spec) learnware_rkme_spec_list = [ learnware.specification.get_stat_spec_by_name(stat_spec_type) for learnware in self.learnware_list ] From bafe1cc3233e1baa9064696ec630c53e13618c0f Mon Sep 17 00:00:00 2001 From: bxdd Date: Sat, 4 Nov 2023 18:03:03 +0800 Subject: [PATCH 07/27] [MNT] update checker log --- learnware/market/easy2/checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/learnware/market/easy2/checker.py b/learnware/market/easy2/checker.py index 1ae8aa7..61679a6 100644 --- a/learnware/market/easy2/checker.py +++ b/learnware/market/easy2/checker.py @@ -124,7 +124,7 @@ class EasyStatChecker(BaseChecker): outputs = outputs.reshape(-1, 1) if outputs.shape[1:] != learnware_model.output_shape: - logger.warning(f"The learnware [{learnware.id}] output dimention mismatch!") + logger.warning(f"The learnware [{learnware.id}] output dimention mismatch, where {outputs.shape[1:]} != {learnware_model.output_shape}.") return self.INVALID_LEARNWARE if semantic_spec["Task"]["Values"][0] in ("Classification", "Regression", "Feature Extraction"): From 9f7bf90eddcd55576af55ab191c007a133c901c0 Mon Sep 17 00:00:00 2001 From: bxdd Date: Sun, 5 Nov 2023 02:04:15 +0800 Subject: [PATCH 08/27] [FIX] fix easy check procedure --- learnware/client/container.py | 1 + learnware/learnware/__init__.py | 4 ++-- learnware/market/easy2/checker.py | 19 +++++++++---------- learnware/market/easy2/database_ops.py | 1 - 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/learnware/client/container.py b/learnware/client/container.py index acc1499..ea0db11 100644 --- a/learnware/client/container.py +++ b/learnware/client/container.py @@ -500,6 +500,7 @@ class LearnwaresContainer: learnwares: Union[List[Learnware], Learnware], cleanup=True, mode="conda", + ignore_error=True, ): """The initializaiton method for base reuser diff --git a/learnware/learnware/__init__.py b/learnware/learnware/__init__.py index af6d45e..738a55d 100644 --- a/learnware/learnware/__init__.py +++ b/learnware/learnware/__init__.py @@ -12,7 +12,7 @@ from ..config import C logger = get_module_logger("learnware.learnware") -def get_learnware_from_dirpath(id: str, semantic_spec: dict, learnware_dirpath, raise_error=False) -> Learnware: +def get_learnware_from_dirpath(id: str, semantic_spec: dict, learnware_dirpath, ignore_error=True) -> Learnware: """Get the learnware object from dirpath, and provide the manage interface tor Learnware class Parameters @@ -67,7 +67,7 @@ def get_learnware_from_dirpath(id: str, semantic_spec: dict, learnware_dirpath, learnware_spec.update_semantic_spec(copy.deepcopy(semantic_spec)) except Exception as e: - if raise_error: + if not ignore_error: raise e logger.warning(f"Load Learnware {id} failed! Due to {repr(e)}") return None diff --git a/learnware/market/easy2/checker.py b/learnware/market/easy2/checker.py index 61679a6..829af51 100644 --- a/learnware/market/easy2/checker.py +++ b/learnware/market/easy2/checker.py @@ -118,13 +118,12 @@ class EasyStatChecker(BaseChecker): inputs = np.random.randint(0, 255, size=(10, *input_shape)) else: raise ValueError(f"not supported spec type for spec_type = {spec_type}") - outputs = learnware.predict(inputs) + # Check output - if outputs.ndim == 1: - outputs = outputs.reshape(-1, 1) - - if outputs.shape[1:] != learnware_model.output_shape: - logger.warning(f"The learnware [{learnware.id}] output dimention mismatch, where {outputs.shape[1:]} != {learnware_model.output_shape}.") + try: + outputs = learnware.predict(inputs) + except Exception: + logger.warning(f"learnware {learnware} prediction method is not valid!") return self.INVALID_LEARNWARE if semantic_spec["Task"]["Values"][0] in ("Classification", "Regression", "Feature Extraction"): @@ -135,11 +134,11 @@ class EasyStatChecker(BaseChecker): logger.warning(f"The learnware [{learnware.id}] output must be np.ndarray or torch.Tensor!") return self.INVALID_LEARNWARE + if outputs.ndim == 1: + outputs = outputs.reshape(-1, 1) # Check output shape - if outputs[0].shape != learnware_model.output_shape or learnware_model.output_shape != int( - semantic_spec["Output"]["Dimension"] - ): - logger.warning(f"The learnware [{learnware.id}] output dimention mismatch!") + if outputs[0].shape != learnware_model.output_shape or learnware_model.output_shape != (int(semantic_spec["Output"]["Dimension"]), ): + logger.warning(f"The learnware [{learnware.id}] output dimention mismatch!, where pred_shape={outputs[0].shape}, model_shape={learnware_model.output_shape}, semantic_shape={(int(semantic_spec['Output']['Dimension']), )}") return self.INVALID_LEARNWARE except Exception as e: diff --git a/learnware/market/easy2/database_ops.py b/learnware/market/easy2/database_ops.py index 25f02e9..088695e 100644 --- a/learnware/market/easy2/database_ops.py +++ b/learnware/market/easy2/database_ops.py @@ -169,7 +169,6 @@ class DatabaseOperations(object): use_flags[id] = use_flag max_count = max(max_count, int(id)) pass - return learnware_list, zip_list, folder_list, use_flags, max_count + 1 pass From ccad9075ccff4689ca5ea94c29d97ac9dafaecd9 Mon Sep 17 00:00:00 2001 From: bxdd Date: Sun, 5 Nov 2023 02:21:01 +0800 Subject: [PATCH 09/27] [MNT] modify conda checker to return invalid when install learnware conda failed! --- learnware/client/container.py | 17 +++++++++++------ learnware/client/utils.py | 2 +- learnware/market/classes.py | 14 +++++++------- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/learnware/client/container.py b/learnware/client/container.py index ea0db11..e89dd32 100644 --- a/learnware/client/container.py +++ b/learnware/client/container.py @@ -520,6 +520,7 @@ class LearnwaresContainer: assert self.mode in {"conda", "docker"}, f"mode must be in ['conda', 'docker'], should not be {self.mode}" self.learnware_list = learnwares self.cleanup = cleanup + self.ignore_error = ignore_error def __enter__(self): if self.mode == "conda": @@ -549,7 +550,7 @@ class LearnwaresContainer: model_list = [_learnware.get_model() for _learnware in self.learnware_containers] with ThreadPoolExecutor(max_workers=max(os.cpu_count() // 2, 1)) as executor: - results = executor.map(self._initialize_model_container, model_list) + results = executor.map(self._initialize_model_container, model_list, [self.ignore_error]*len(model_list)) self.results = list(results) if sum(self.results) < len(self.learnware_list): @@ -568,7 +569,7 @@ class LearnwaresContainer: model_list = [_learnware.get_model() for _learnware in self.learnware_containers] with ThreadPoolExecutor(max_workers=max(os.cpu_count() // 2, 1)) as executor: - executor.map(self._destroy_model_container, model_list) + executor.map(self._destroy_model_container, model_list, [self.ignore_error]*len(model_list)) self.learnware_containers = None self.results = None @@ -577,20 +578,24 @@ class LearnwaresContainer: ModelDockerContainer._destroy_docker_container(self._docker_container) @staticmethod - def _initialize_model_container(model: ModelCondaContainer): + def _initialize_model_container(model: ModelCondaContainer, ignore_error=True): try: model.init_and_setup_env() except Exception as err: - logger.error(f"build env {model.conda_env} failed due to {err}") + if not ignore_error: + raise err + logger.warning(f"build env {model.conda_env} failed due to {err}") return False return True @staticmethod - def _destroy_model_container(model: ModelCondaContainer): + def _destroy_model_container(model: ModelCondaContainer, ignore_error=True): try: model.remove_env() except Exception as err: - logger.error(f"remove env {model.conda_env} failed due to {err}") + if not ignore_error: + raise err + logger.warning(f"remove env {model.conda_env} failed due to {err}") return False return True diff --git a/learnware/client/utils.py b/learnware/client/utils.py index fff1475..0c776f5 100644 --- a/learnware/client/utils.py +++ b/learnware/client/utils.py @@ -14,7 +14,7 @@ def system_execute(args, timeout=None): try: com_process.check_returncode() except subprocess.CalledProcessError as err: - logger.error(f"System Execute Error: {com_process.stderr.decode()}") + logger.warning(f"System Execute Error: {com_process.stderr.decode()}") raise err diff --git a/learnware/market/classes.py b/learnware/market/classes.py index 8fa9ab7..deee3d8 100644 --- a/learnware/market/classes.py +++ b/learnware/market/classes.py @@ -12,11 +12,11 @@ class CondaChecker(BaseChecker): super(CondaChecker, self).__init__(**kwargs) def __call__(self, learnware: Learnware) -> int: - with LearnwaresContainer(learnware) as env_container: - if not all(env_container.get_learnware_flags()): - logger.warning(f"Conda Checker failed due to installed learnware failed") - return BaseChecker.INVALID_LEARNWARE - learnwares = env_container.get_learnwares_with_container() - check_status = self.inner_checker(learnwares[0]) - + try: + with LearnwaresContainer(learnware, ignore_error=False) as env_container: + learnwares = env_container.get_learnwares_with_container() + check_status = self.inner_checker(learnwares[0]) + except Exception as e: + logger.warning(f"Conda Checker failed due to installed learnware failed and {e}") + return BaseChecker.INVALID_LEARNWARE return check_status From e516abc7f6fafbf3c257b97013a3fdd6ad706353 Mon Sep 17 00:00:00 2001 From: bxdd Date: Sun, 5 Nov 2023 02:21:21 +0800 Subject: [PATCH 10/27] [MNT] black format --- learnware/client/container.py | 4 ++-- learnware/market/easy2/checker.py | 10 +++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/learnware/client/container.py b/learnware/client/container.py index e89dd32..e7ae9ca 100644 --- a/learnware/client/container.py +++ b/learnware/client/container.py @@ -550,7 +550,7 @@ class LearnwaresContainer: model_list = [_learnware.get_model() for _learnware in self.learnware_containers] with ThreadPoolExecutor(max_workers=max(os.cpu_count() // 2, 1)) as executor: - results = executor.map(self._initialize_model_container, model_list, [self.ignore_error]*len(model_list)) + results = executor.map(self._initialize_model_container, model_list, [self.ignore_error] * len(model_list)) self.results = list(results) if sum(self.results) < len(self.learnware_list): @@ -569,7 +569,7 @@ class LearnwaresContainer: model_list = [_learnware.get_model() for _learnware in self.learnware_containers] with ThreadPoolExecutor(max_workers=max(os.cpu_count() // 2, 1)) as executor: - executor.map(self._destroy_model_container, model_list, [self.ignore_error]*len(model_list)) + executor.map(self._destroy_model_container, model_list, [self.ignore_error] * len(model_list)) self.learnware_containers = None self.results = None diff --git a/learnware/market/easy2/checker.py b/learnware/market/easy2/checker.py index 829af51..5e15271 100644 --- a/learnware/market/easy2/checker.py +++ b/learnware/market/easy2/checker.py @@ -118,7 +118,7 @@ class EasyStatChecker(BaseChecker): inputs = np.random.randint(0, 255, size=(10, *input_shape)) else: raise ValueError(f"not supported spec type for spec_type = {spec_type}") - + # Check output try: outputs = learnware.predict(inputs) @@ -137,8 +137,12 @@ class EasyStatChecker(BaseChecker): if outputs.ndim == 1: outputs = outputs.reshape(-1, 1) # Check output shape - if outputs[0].shape != learnware_model.output_shape or learnware_model.output_shape != (int(semantic_spec["Output"]["Dimension"]), ): - logger.warning(f"The learnware [{learnware.id}] output dimention mismatch!, where pred_shape={outputs[0].shape}, model_shape={learnware_model.output_shape}, semantic_shape={(int(semantic_spec['Output']['Dimension']), )}") + if outputs[0].shape != learnware_model.output_shape or learnware_model.output_shape != ( + int(semantic_spec["Output"]["Dimension"]), + ): + logger.warning( + f"The learnware [{learnware.id}] output dimention mismatch!, where pred_shape={outputs[0].shape}, model_shape={learnware_model.output_shape}, semantic_shape={(int(semantic_spec['Output']['Dimension']), )}" + ) return self.INVALID_LEARNWARE except Exception as e: From f7343bfadfa1bea9857240c2023535993b4532ea Mon Sep 17 00:00:00 2001 From: Gene Date: Sun, 5 Nov 2023 15:43:01 +0800 Subject: [PATCH 11/27] [MNT] modify details in log --- tests/test_learnware_client/test_all_learnware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_learnware_client/test_all_learnware.py b/tests/test_learnware_client/test_all_learnware.py index bdeea39..c91222d 100644 --- a/tests/test_learnware_client/test_all_learnware.py +++ b/tests/test_learnware_client/test_all_learnware.py @@ -43,7 +43,7 @@ class TestAllLearnware(unittest.TestCase): failed_ids.append(idx) print(f"check learnware {idx} failed!!!") - print(f"failed learnware ids: {failed_ids}") + print(f"The currently failed learnware ids: {failed_ids}") if __name__ == "__main__": From b6796f0f6e1dc7ae7773c7ba3cf20ccd1fa01903 Mon Sep 17 00:00:00 2001 From: Gene Date: Sun, 5 Nov 2023 17:39:20 +0800 Subject: [PATCH 12/27] [FIX] fix bugs in load_market --- learnware/market/easy2/database_ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/learnware/market/easy2/database_ops.py b/learnware/market/easy2/database_ops.py index 088695e..087e6fd 100644 --- a/learnware/market/easy2/database_ops.py +++ b/learnware/market/easy2/database_ops.py @@ -166,7 +166,7 @@ class DatabaseOperations(object): # assert new_learnware is not None zip_list[id] = zip_path folder_list[id] = folder_path - use_flags[id] = use_flag + use_flags[id] = int(use_flag) max_count = max(max_count, int(id)) pass return learnware_list, zip_list, folder_list, use_flags, max_count + 1 From 00a10f0ebf4ee93f52dd8859e2b457c325b6603d Mon Sep 17 00:00:00 2001 From: Gene Date: Sun, 5 Nov 2023 19:29:10 +0800 Subject: [PATCH 13/27] [MNT] modify get_learnware_path_by_ids to get_learnware_zip_path_by_ids --- learnware/market/base.py | 10 +++++----- learnware/market/easy.py | 2 +- learnware/market/easy2/organizer.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/learnware/market/base.py b/learnware/market/base.py index 8a2f445..3a5fa22 100644 --- a/learnware/market/base.py +++ b/learnware/market/base.py @@ -172,7 +172,7 @@ class LearnwareMarket: int The final learnware check_status. """ - zip_path = self.get_learnware_path_by_ids(id) if zip_path is None else zip_path + zip_path = self.get_learnware_zip_path_by_ids(id) if zip_path is None else zip_path semantic_spec = ( self.get_learnware_by_ids(id).get_specification().get_semantic_spec() if semantic_spec is None @@ -223,8 +223,8 @@ class LearnwareMarket: """ return self.learnware_organizer.get_learnwares(top, check_status, **kwargs) - def get_learnware_path_by_ids(self, ids: Union[str, List[str]], **kwargs) -> Union[Learnware, List[Learnware]]: - return self.learnware_organizer.get_learnware_path_by_ids(ids, **kwargs) + def get_learnware_zip_path_by_ids(self, ids: Union[str, List[str]], **kwargs) -> Union[Learnware, List[Learnware]]: + return self.learnware_organizer.get_learnware_zip_path_by_ids(ids, **kwargs) def get_learnware_by_ids(self, id: Union[str, List[str]], **kwargs) -> Union[Learnware, List[Learnware]]: return self.learnware_organizer.get_learnware_by_ids(id, **kwargs) @@ -331,7 +331,7 @@ class BaseOrganizer: """ raise NotImplementedError("get_learnware_by_ids is not implemented in BaseOrganizer") - def get_learnware_path_by_ids(self, ids: Union[str, List[str]]) -> Union[Learnware, List[Learnware]]: + def get_learnware_zip_path_by_ids(self, ids: Union[str, List[str]]) -> Union[Learnware, List[Learnware]]: """Get Zipped Learnware file by id Parameters @@ -347,7 +347,7 @@ class BaseOrganizer: Return the path for target learnware or list of path. None for Learnware NOT Found. """ - raise NotImplementedError("get_learnware_path_by_ids is not implemented in BaseOrganizer") + raise NotImplementedError("get_learnware_zip_path_by_ids is not implemented in BaseOrganizer") def get_learnware_ids(self, top: int = None, check_status: int = None) -> List[str]: """get the list of learnware ids diff --git a/learnware/market/easy.py b/learnware/market/easy.py index 098887b..b57e1c0 100644 --- a/learnware/market/easy.py +++ b/learnware/market/easy.py @@ -926,7 +926,7 @@ class EasyMarket(LearnwareMarket): logger.warning("Learnware ID '%s' NOT Found!" % (ids)) return None - def get_learnware_path_by_ids(self, ids: Union[str, List[str]]) -> Union[Learnware, List[Learnware]]: + def get_learnware_zip_path_by_ids(self, ids: Union[str, List[str]]) -> Union[Learnware, List[Learnware]]: """Get Zipped Learnware file by id Parameters diff --git a/learnware/market/easy2/organizer.py b/learnware/market/easy2/organizer.py index 02998d8..fa7054c 100644 --- a/learnware/market/easy2/organizer.py +++ b/learnware/market/easy2/organizer.py @@ -256,7 +256,7 @@ class EasyOrganizer(BaseOrganizer): logger.warning("Learnware ID '%s' NOT Found!" % (ids)) return None - def get_learnware_path_by_ids(self, ids: Union[str, List[str]]) -> Union[Learnware, List[Learnware]]: + def get_learnware_zip_path_by_ids(self, ids: Union[str, List[str]]) -> Union[Learnware, List[Learnware]]: """Get Zipped Learnware file by id Parameters From 64891ecd23ee558f988b1be7ebd36e036017e4c8 Mon Sep 17 00:00:00 2001 From: Gene Date: Sun, 5 Nov 2023 19:44:36 +0800 Subject: [PATCH 14/27] [MNT] add get_learnware_dir_path_by_ids in organizer --- learnware/market/base.py | 21 +++++++++++++++++++ learnware/market/easy2/organizer.py | 32 +++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/learnware/market/base.py b/learnware/market/base.py index 3a5fa22..9d96f2c 100644 --- a/learnware/market/base.py +++ b/learnware/market/base.py @@ -226,6 +226,9 @@ class LearnwareMarket: def get_learnware_zip_path_by_ids(self, ids: Union[str, List[str]], **kwargs) -> Union[Learnware, List[Learnware]]: return self.learnware_organizer.get_learnware_zip_path_by_ids(ids, **kwargs) + def get_learnware_dir_path_by_ids(self, ids: Union[str, List[str]], **kwargs) -> Union[Learnware, List[Learnware]]: + return self.learnware_organizer.get_learnware_dir_path_by_ids(ids, **kwargs) + def get_learnware_by_ids(self, id: Union[str, List[str]], **kwargs) -> Union[Learnware, List[Learnware]]: return self.learnware_organizer.get_learnware_by_ids(id, **kwargs) @@ -349,6 +352,24 @@ class BaseOrganizer: """ raise NotImplementedError("get_learnware_zip_path_by_ids is not implemented in BaseOrganizer") + def get_learnware_dir_path_by_ids(self, ids: Union[str, List[str]]) -> Union[Learnware, List[Learnware]]: + """Get Learnware dir path by id + + Parameters + ---------- + ids : Union[str, List[str]] + Give a id or a list of ids + str: id of targer learware + List[str]: A list of ids of target learnwares + + Returns + ------- + Union[Learnware, List[Learnware]] + Return the dir path for target learnware or list of path. + None for Learnware NOT Found. + """ + raise NotImplementedError("get_learnware_dir_path_by_ids is not implemented in BaseOrganizer") + def get_learnware_ids(self, top: int = None, check_status: int = None) -> List[str]: """get the list of learnware ids diff --git a/learnware/market/easy2/organizer.py b/learnware/market/easy2/organizer.py index fa7054c..6c0327d 100644 --- a/learnware/market/easy2/organizer.py +++ b/learnware/market/easy2/organizer.py @@ -288,6 +288,38 @@ class EasyOrganizer(BaseOrganizer): logger.warning("Learnware ID '%s' NOT Found!" % (ids)) return None + def get_learnware_dir_path_by_ids(self, ids: Union[str, List[str]]) -> Union[Learnware, List[Learnware]]: + """Get Learnware dir path by id + + Parameters + ---------- + ids : Union[str, List[str]] + Give a id or a list of ids + str: id of targer learware + List[str]: A list of ids of target learnwares + + Returns + ------- + Union[Learnware, List[Learnware]] + Return the dir path for target learnware or list of path. + None for Learnware NOT Found. + """ + if isinstance(ids, list): + ret = [] + for id in ids: + if id in self.learnware_folder_list: + ret.append(self.learnware_folder_list[id]) + else: + logger.warning("Learnware ID '%s' NOT Found!" % (id)) + ret.append(None) + return ret + else: + try: + return self.learnware_folder_list[ids] + except: + logger.warning("Learnware ID '%s' NOT Found!" % (ids)) + return None + def get_learnware_ids(self, top: int = None, check_status: int = None) -> List[str]: """Get learnware ids From df5b96d14bc1fce94b52ff183f0ff2d93055d503 Mon Sep 17 00:00:00 2001 From: Gene Date: Sun, 5 Nov 2023 21:33:02 +0800 Subject: [PATCH 15/27] [FIX] fix bugs about checker_names --- learnware/market/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/learnware/market/base.py b/learnware/market/base.py index 9d96f2c..9894c65 100644 --- a/learnware/market/base.py +++ b/learnware/market/base.py @@ -81,8 +81,6 @@ class LearnwareMarket: pending_learnware = get_learnware_from_dirpath( id="pending", semantic_spec=semantic_spec, learnware_dirpath=tempdir ) - checker_names = list(self.learnware_checker.keys()) if checker_names is None else checker_names - for name in checker_names: checker = self.learnware_checker[name] check_status = checker(pending_learnware) @@ -115,6 +113,7 @@ class LearnwareMarket: - str indicating model_id - int indicating the final learnware check_status """ + checker_names = list(self.learnware_checker.keys()) if checker_names is None else checker_names check_status = self.check_learnware(zip_path, semantic_spec, checker_names) return self.learnware_organizer.add_learnware( zip_path=zip_path, semantic_spec=semantic_spec, check_status=check_status, **kwargs @@ -178,6 +177,7 @@ class LearnwareMarket: if semantic_spec is None else semantic_spec ) + checker_names = list(self.learnware_checker.keys()) if checker_names is None else checker_names update_status = self.check_learnware(zip_path, semantic_spec, checker_names) check_status = ( update_status if check_status is None or update_status == BaseChecker.INVALID_LEARNWARE else check_status From 09eaf823651954d2703d6fd3931866366d883733 Mon Sep 17 00:00:00 2001 From: zouxiaochuan Date: Mon, 6 Nov 2023 14:13:28 +0800 Subject: [PATCH 16/27] [MNT]: invoke system with current environment --- learnware/client/package_utils.py | 3 ++- learnware/client/utils.py | 16 ++++++++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/learnware/client/package_utils.py b/learnware/client/package_utils.py index 30ce777..0f26955 100644 --- a/learnware/client/package_utils.py +++ b/learnware/client/package_utils.py @@ -4,6 +4,7 @@ import yaml import tempfile import subprocess from typing import List, Tuple +from . import utils from ..logger import get_module_logger @@ -15,7 +16,7 @@ def try_to_run(args, timeout=5, retry=5): sucess = False for i in range(retry): try: - subprocess.check_call(args=args, timeout=timeout, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + utils.system_execute(args=args, timeout=timeout, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) sucess = True break except subprocess.TimeoutExpired as e: diff --git a/learnware/client/utils.py b/learnware/client/utils.py index 0c776f5..1aa6651 100644 --- a/learnware/client/utils.py +++ b/learnware/client/utils.py @@ -9,8 +9,20 @@ from .package_utils import filter_nonexist_conda_packages_file, filter_nonexist_ logger = get_module_logger(module_name="client_utils") -def system_execute(args, timeout=None): - com_process = subprocess.run(args, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, timeout=timeout) +def system_execute(args, timeout=None, env=None, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE): + if env is None: + env = os.environ.copy() + pass + + if isinstance(args, str): + pass + else: + args = ' '.join(args) + pass + + com_process = subprocess.run( + args, stdout=stdout, stderr=stderr, timeout=timeout, env=env, shell=True) + try: com_process.check_returncode() except subprocess.CalledProcessError as err: From de8e685fb7df417e7e0bb482076fffac18e41426 Mon Sep 17 00:00:00 2001 From: Gene Date: Mon, 6 Nov 2023 20:01:06 +0800 Subject: [PATCH 17/27] [MNT] formate code by black --- examples/dataset_pfs_workflow/pfs/pfs_cross_transfer.py | 4 +++- learnware/client/utils.py | 9 ++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/examples/dataset_pfs_workflow/pfs/pfs_cross_transfer.py b/examples/dataset_pfs_workflow/pfs/pfs_cross_transfer.py index 93a3fa3..5f69127 100644 --- a/examples/dataset_pfs_workflow/pfs/pfs_cross_transfer.py +++ b/examples/dataset_pfs_workflow/pfs/pfs_cross_transfer.py @@ -85,7 +85,9 @@ def get_split_errs(algo): split = train_xs.shape[0] - proportion_list[tmp] model.fit( - train_xs[split:,], + train_xs[ + split:, + ], train_ys[split:], eval_set=[(val_xs, val_ys)], early_stopping_rounds=50, diff --git a/learnware/client/utils.py b/learnware/client/utils.py index 1aa6651..09e5142 100644 --- a/learnware/client/utils.py +++ b/learnware/client/utils.py @@ -13,15 +13,14 @@ def system_execute(args, timeout=None, env=None, stdout=subprocess.DEVNULL, stde if env is None: env = os.environ.copy() pass - + if isinstance(args, str): pass else: - args = ' '.join(args) + args = " ".join(args) pass - - com_process = subprocess.run( - args, stdout=stdout, stderr=stderr, timeout=timeout, env=env, shell=True) + + com_process = subprocess.run(args, stdout=stdout, stderr=stderr, timeout=timeout, env=env, shell=True) try: com_process.check_returncode() From 5588d79076d917494e21da4ba7994babdecf9a53 Mon Sep 17 00:00:00 2001 From: Peng Tan Date: Tue, 7 Nov 2023 13:45:04 +0800 Subject: [PATCH 18/27] [ENH] add test_prepare_learnware_randomly & test_generated_learnwares (multi-processing) --- learnware/market/module.py | 2 +- .../example_learnwares/config.py | 1 + .../example_learnware_0/__init__.py | 22 ++ .../example_learnware_0/learnware.yaml | 8 + .../example_learnware_0/requirements.txt | 1 + .../example_learnware_1/__init__.py | 22 ++ .../example_learnware_1/learnware.yaml | 8 + .../example_learnware_1/requirements.txt | 1 + .../test_hetero_market/test_hetero.py | 236 ++++++++++++++++++ 9 files changed, 300 insertions(+), 1 deletion(-) create mode 100644 tests/test_market/test_hetero_market/example_learnwares/config.py create mode 100644 tests/test_market/test_hetero_market/example_learnwares/example_learnware_0/__init__.py create mode 100644 tests/test_market/test_hetero_market/example_learnwares/example_learnware_0/learnware.yaml create mode 100644 tests/test_market/test_hetero_market/example_learnwares/example_learnware_0/requirements.txt create mode 100644 tests/test_market/test_hetero_market/example_learnwares/example_learnware_1/__init__.py create mode 100644 tests/test_market/test_hetero_market/example_learnwares/example_learnware_1/learnware.yaml create mode 100644 tests/test_market/test_hetero_market/example_learnwares/example_learnware_1/requirements.txt create mode 100644 tests/test_market/test_hetero_market/test_hetero.py diff --git a/learnware/market/module.py b/learnware/market/module.py index 43499ec..f60e06a 100644 --- a/learnware/market/module.py +++ b/learnware/market/module.py @@ -11,7 +11,7 @@ MARKET_CONFIG = { "hetero": { "organizer": HeteroMapTableOrganizer(), "searcher": HeteroMapTableSearcher(), - "checker_list": [] + "checker_list": [EasySemanticChecker(), EasyStatChecker()] } } diff --git a/tests/test_market/test_hetero_market/example_learnwares/config.py b/tests/test_market/test_hetero_market/example_learnwares/config.py new file mode 100644 index 0000000..6c8459a --- /dev/null +++ b/tests/test_market/test_hetero_market/example_learnwares/config.py @@ -0,0 +1 @@ +input_shape_list=[20, 30] # 20-input shape of example learnware 0, 30-input shape of example learnware 1 \ No newline at end of file diff --git a/tests/test_market/test_hetero_market/example_learnwares/example_learnware_0/__init__.py b/tests/test_market/test_hetero_market/example_learnwares/example_learnware_0/__init__.py new file mode 100644 index 0000000..e9c6cf0 --- /dev/null +++ b/tests/test_market/test_hetero_market/example_learnwares/example_learnware_0/__init__.py @@ -0,0 +1,22 @@ +from learnware.model import BaseModel +import numpy as np +import joblib +import os + + +class MyModel(BaseModel): + def __init__(self): + super(MyModel, self).__init__(input_shape=(20,), output_shape=(1,)) + dir_path = os.path.dirname(os.path.abspath(__file__)) + model_path=os.path.join(dir_path, "ridge.pkl") + model = joblib.load(model_path) + self.model=model + + def fit(self, X: np.ndarray, y: np.ndarray): + pass + + def predict(self, X: np.ndarray) -> np.ndarray: + return self.model.predict(X) + + def finetune(self, X: np.ndarray, y: np.ndarray): + pass \ No newline at end of file diff --git a/tests/test_market/test_hetero_market/example_learnwares/example_learnware_0/learnware.yaml b/tests/test_market/test_hetero_market/example_learnwares/example_learnware_0/learnware.yaml new file mode 100644 index 0000000..4a37a37 --- /dev/null +++ b/tests/test_market/test_hetero_market/example_learnwares/example_learnware_0/learnware.yaml @@ -0,0 +1,8 @@ +model: + class_name: MyModel + kwargs: {} +stat_specifications: + - module_path: learnware.specification + class_name: RKMETableSpecification + file_name: stat.json + kwargs: {} \ No newline at end of file diff --git a/tests/test_market/test_hetero_market/example_learnwares/example_learnware_0/requirements.txt b/tests/test_market/test_hetero_market/example_learnwares/example_learnware_0/requirements.txt new file mode 100644 index 0000000..1da1c5f --- /dev/null +++ b/tests/test_market/test_hetero_market/example_learnwares/example_learnware_0/requirements.txt @@ -0,0 +1 @@ +learnware == 0.1.0.999 \ No newline at end of file diff --git a/tests/test_market/test_hetero_market/example_learnwares/example_learnware_1/__init__.py b/tests/test_market/test_hetero_market/example_learnwares/example_learnware_1/__init__.py new file mode 100644 index 0000000..934e352 --- /dev/null +++ b/tests/test_market/test_hetero_market/example_learnwares/example_learnware_1/__init__.py @@ -0,0 +1,22 @@ +from learnware.model import BaseModel +import numpy as np +import joblib +import os + + +class MyModel(BaseModel): + def __init__(self): + super(MyModel, self).__init__(input_shape=(30,), output_shape=(1,)) + dir_path = os.path.dirname(os.path.abspath(__file__)) + model_path=os.path.join(dir_path, "ridge.pkl") + model = joblib.load(model_path) + self.model=model + + def fit(self, X: np.ndarray, y: np.ndarray): + pass + + def predict(self, X: np.ndarray) -> np.ndarray: + return self.model.predict(X) + + def finetune(self, X: np.ndarray, y: np.ndarray): + pass \ No newline at end of file diff --git a/tests/test_market/test_hetero_market/example_learnwares/example_learnware_1/learnware.yaml b/tests/test_market/test_hetero_market/example_learnwares/example_learnware_1/learnware.yaml new file mode 100644 index 0000000..4a37a37 --- /dev/null +++ b/tests/test_market/test_hetero_market/example_learnwares/example_learnware_1/learnware.yaml @@ -0,0 +1,8 @@ +model: + class_name: MyModel + kwargs: {} +stat_specifications: + - module_path: learnware.specification + class_name: RKMETableSpecification + file_name: stat.json + kwargs: {} \ No newline at end of file diff --git a/tests/test_market/test_hetero_market/example_learnwares/example_learnware_1/requirements.txt b/tests/test_market/test_hetero_market/example_learnwares/example_learnware_1/requirements.txt new file mode 100644 index 0000000..1da1c5f --- /dev/null +++ b/tests/test_market/test_hetero_market/example_learnwares/example_learnware_1/requirements.txt @@ -0,0 +1 @@ +learnware == 0.1.0.999 \ No newline at end of file diff --git a/tests/test_market/test_hetero_market/test_hetero.py b/tests/test_market/test_hetero_market/test_hetero.py new file mode 100644 index 0000000..32b4759 --- /dev/null +++ b/tests/test_market/test_hetero_market/test_hetero.py @@ -0,0 +1,236 @@ +import sys +import unittest +import os +import copy +import joblib +import zipfile +import numpy as np +from sklearn.linear_model import Ridge +from sklearn.datasets import make_regression +from sklearn.datasets import load_digits +from shutil import copyfile, rmtree +from multiprocessing import Pool +from learnware.client import LearnwareClient + +import learnware +from learnware.market import instantiate_learnware_market, BaseUserInfo +import learnware.specification as specification +from example_learnwares.config import input_shape_list + +curr_root = os.path.dirname(os.path.abspath(__file__)) + +user_semantic = { + "Data": {"Values": ["Image"], "Type": "Class"}, + "Task": { + "Values": ["Classification"], + "Type": "Class", + }, + "Library": {"Values": ["Scikit-learn"], "Type": "Class"}, + "Scenario": {"Values": ["Education"], "Type": "Tag"}, + "Description": {"Values": "", "Type": "String"}, + "Name": {"Values": "", "Type": "String"}, + "Output": { + "Dimension": 10, + "Description": { + "0": "the probability of the label is zero", + }, + }, +} + + +def check_learnware(learnware_name, dir_path=os.path.join(curr_root, "learnware_pool")): + print(f"Checking Learnware: {learnware_name}") + zip_file_path = os.path.join(dir_path, learnware_name) + client = LearnwareClient() + # if check_learnware doesn't raise an exception, return True, otherwise, return false + try: + client.check_learnware(zip_file_path) + return True + except Exception as e: + print(f"Learnware {learnware_name} failed the check: {e}") + return False + + +class TestMarket(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + np.random.seed(2023) + learnware.init() + + def _init_learnware_market(self): + """initialize learnware market""" + easy_market = instantiate_learnware_market(market_id="hetero_toy", name="hetero", rebuild=True) + return easy_market + + def test_prepare_learnware_randomly(self, learnware_num=5): + self.zip_path_list = [] + X, y = load_digits(return_X_y=True) + + for i in range(learnware_num): + dir_path = os.path.join(curr_root, "learnware_pool", "ridge_%d" % (i)) + os.makedirs(dir_path, exist_ok=True) + + print("Preparing Learnware: %d" % (i)) + + example_learnware_idx=i%2 + input_dim=input_shape_list[example_learnware_idx] + example_learnware_name="example_learnwares/example_learnware_%d" % (example_learnware_idx) + + X, y = make_regression(n_samples=5000, n_features=input_dim, noise=0.1, random_state=42) + + clf=Ridge(alpha=1.0) + clf.fit(X, y) + + joblib.dump(clf, os.path.join(dir_path, "ridge.pkl")) + + spec = specification.utils.generate_rkme_spec(X=X, gamma=0.1, cuda_idx=0) + spec.save(os.path.join(dir_path, "stat.json")) + + init_file = os.path.join(dir_path, "__init__.py") + copyfile( + os.path.join(curr_root, example_learnware_name, "__init__.py"), init_file + ) # cp example_init.py init_file + + yaml_file = os.path.join(dir_path, "learnware.yaml") + copyfile(os.path.join(curr_root, example_learnware_name, "learnware.yaml"), yaml_file) # cp example.yaml yaml_file + + env_file = os.path.join(dir_path, "requirements.txt") + copyfile(os.path.join(curr_root, example_learnware_name, "requirements.txt"), env_file) + + zip_file = dir_path + ".zip" + # zip -q -r -j zip_file dir_path + with zipfile.ZipFile(zip_file, "w") as zip_obj: + for foldername, subfolders, filenames in os.walk(dir_path): + for filename in filenames: + file_path = os.path.join(foldername, filename) + zip_info = zipfile.ZipInfo(filename) + zip_info.compress_type = zipfile.ZIP_STORED + with open(file_path, "rb") as file: + zip_obj.writestr(zip_info, file.read()) + + rmtree(dir_path) # rm -r dir_path + + def test_generated_learnwares(self): + curr_root = os.path.dirname(os.path.abspath(__file__)) + dir_path = os.path.join(curr_root, "learnware_pool") + + # Execute multi-process checking using Pool + with Pool() as pool: + results = pool.starmap(check_learnware, [(name, dir_path) for name in os.listdir(dir_path)]) + + # Use an assert statement to ensure that all checks return True + self.assertTrue(all(results), "Not all learnwares passed the check") + + # def test_upload_delete_learnware(self, learnware_num=5, delete=True): + # easy_market = self._init_learnware_market() + # self.test_prepare_learnware_randomly(learnware_num) + # self.learnware_num = learnware_num + + # print("Total Item:", len(easy_market)) + # assert len(easy_market) == 0, f"The market should be empty!" + + # for idx, zip_path in enumerate(self.zip_path_list): + # semantic_spec = copy.deepcopy(user_semantic) + # semantic_spec["Name"]["Values"] = "learnware_%d" % (idx) + # semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx) + # easy_market.add_learnware(zip_path, semantic_spec) + + # print("Total Item:", len(easy_market)) + # assert len(easy_market) == self.learnware_num, f"The number of learnwares must be {self.learnware_num}!" + + # curr_inds = easy_market.get_learnware_ids() + # print("Available ids After Uploading Learnwares:", curr_inds) + # assert len(curr_inds) == self.learnware_num, f"The number of learnwares must be {self.learnware_num}!" + + # if delete: + # for learnware_id in curr_inds: + # easy_market.delete_learnware(learnware_id) + # self.learnware_num -= 1 + # assert len(easy_market) == self.learnware_num, f"The number of learnwares must be {self.learnware_num}!" + + # curr_inds = easy_market.get_learnware_ids() + # print("Available ids After Deleting Learnwares:", curr_inds) + # assert len(curr_inds) == 0, f"The market should be empty!" + + # return easy_market + + # def test_search_semantics(self, learnware_num=5): + # easy_market = self.test_upload_delete_learnware(learnware_num, delete=False) + # print("Total Item:", len(easy_market)) + # assert len(easy_market) == self.learnware_num, f"The number of learnwares must be {self.learnware_num}!" + + # semantic_spec = copy.deepcopy(user_semantic) + # semantic_spec["Name"]["Values"] = f"learnware_{learnware_num - 1}" + + # user_info = BaseUserInfo(semantic_spec=semantic_spec) + # _, single_learnware_list, _, _ = easy_market.search_learnware(user_info) + + # print("User info:", user_info.get_semantic_spec()) + # print(f"Search result:") + # assert len(single_learnware_list) == 1, f"Exact semantic search failed!" + # for learnware in single_learnware_list: + # semantic_spec1 = learnware.get_specification().get_semantic_spec() + # print("Choose learnware:", learnware.id, semantic_spec1) + # assert semantic_spec1["Name"]["Values"] == semantic_spec["Name"]["Values"], f"Exact semantic search failed!" + + # semantic_spec["Name"]["Values"] = "laernwaer" + # user_info = BaseUserInfo(semantic_spec=semantic_spec) + # _, single_learnware_list, _, _ = easy_market.search_learnware(user_info) + + # print("User info:", user_info.get_semantic_spec()) + # print(f"Search result:") + # assert len(single_learnware_list) == self.learnware_num, f"Fuzzy semantic search failed!" + # for learnware in single_learnware_list: + # semantic_spec1 = learnware.get_specification().get_semantic_spec() + # print("Choose learnware:", learnware.id, semantic_spec1) + + # def test_stat_search(self, learnware_num=5): + # easy_market = self.test_upload_delete_learnware(learnware_num, delete=False) + # print("Total Item:", len(easy_market)) + + # test_folder = os.path.join(curr_root, "test_stat") + + # for idx, zip_path in enumerate(self.zip_path_list): + # unzip_dir = os.path.join(test_folder, f"{idx}") + + # # unzip -o -q zip_path -d unzip_dir + # if os.path.exists(unzip_dir): + # rmtree(unzip_dir) + # os.makedirs(unzip_dir, exist_ok=True) + # with zipfile.ZipFile(zip_path, "r") as zip_obj: + # zip_obj.extractall(path=unzip_dir) + + # user_spec = specification.rkme.RKMETableSpecification() + # user_spec.load(os.path.join(unzip_dir, "svm.json")) + # user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={"RKMETableSpecification": user_spec}) + # ( + # sorted_score_list, + # single_learnware_list, + # mixture_score, + # mixture_learnware_list, + # ) = easy_market.search_learnware(user_info) + + # assert len(single_learnware_list) == self.learnware_num, f"Statistical search failed!" + # print(f"search result of user{idx}:") + # for score, learnware in zip(sorted_score_list, single_learnware_list): + # print(f"score: {score}, learnware_id: {learnware.id}") + # print(f"mixture_score: {mixture_score}\n") + # mixture_id = " ".join([learnware.id for learnware in mixture_learnware_list]) + # print(f"mixture_learnware: {mixture_id}\n") + + # rmtree(test_folder) # rm -r test_folder + + +def suite(): + _suite = unittest.TestSuite() + _suite.addTest(TestMarket("test_prepare_learnware_randomly")) + _suite.addTest(TestMarket("test_generated_learnwares")) + # _suite.addTest(TestMarket("test_upload_delete_learnware")) + # _suite.addTest(TestMarket("test_search_semantics")) + # _suite.addTest(TestMarket("test_stat_search")) + return _suite + + +if __name__ == "__main__": + runner = unittest.TextTestRunner() + runner.run(suite()) From ce66805f41970ecaf3a053fff0a513a8eaef5554 Mon Sep 17 00:00:00 2001 From: Peng Tan Date: Tue, 7 Nov 2023 14:00:56 +0800 Subject: [PATCH 19/27] [MNT] modify variable names --- .../test_hetero_market/test_hetero.py | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/tests/test_market/test_hetero_market/test_hetero.py b/tests/test_market/test_hetero_market/test_hetero.py index 32b4759..5d08ea7 100644 --- a/tests/test_market/test_hetero_market/test_hetero.py +++ b/tests/test_market/test_hetero_market/test_hetero.py @@ -59,8 +59,8 @@ class TestMarket(unittest.TestCase): def _init_learnware_market(self): """initialize learnware market""" - easy_market = instantiate_learnware_market(market_id="hetero_toy", name="hetero", rebuild=True) - return easy_market + hetero_market = instantiate_learnware_market(market_id="hetero_toy", name="hetero", rebuild=True) + return hetero_market def test_prepare_learnware_randomly(self, learnware_num=5): self.zip_path_list = [] @@ -121,38 +121,38 @@ class TestMarket(unittest.TestCase): # Use an assert statement to ensure that all checks return True self.assertTrue(all(results), "Not all learnwares passed the check") - # def test_upload_delete_learnware(self, learnware_num=5, delete=True): - # easy_market = self._init_learnware_market() - # self.test_prepare_learnware_randomly(learnware_num) - # self.learnware_num = learnware_num + def test_upload_delete_learnware(self, learnware_num=5, delete=True): + hetero_market = self._init_learnware_market() + self.test_prepare_learnware_randomly(learnware_num) + self.learnware_num = learnware_num - # print("Total Item:", len(easy_market)) - # assert len(easy_market) == 0, f"The market should be empty!" + print("Total Item:", len(hetero_market)) + assert len(hetero_market) == 0, f"The market should be empty!" - # for idx, zip_path in enumerate(self.zip_path_list): - # semantic_spec = copy.deepcopy(user_semantic) - # semantic_spec["Name"]["Values"] = "learnware_%d" % (idx) - # semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx) - # easy_market.add_learnware(zip_path, semantic_spec) + for idx, zip_path in enumerate(self.zip_path_list): + semantic_spec = copy.deepcopy(user_semantic) + semantic_spec["Name"]["Values"] = "learnware_%d" % (idx) + semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx) + hetero_market.add_learnware(zip_path, semantic_spec) - # print("Total Item:", len(easy_market)) - # assert len(easy_market) == self.learnware_num, f"The number of learnwares must be {self.learnware_num}!" + print("Total Item:", len(hetero_market)) + assert len(hetero_market) == self.learnware_num, f"The number of learnwares must be {self.learnware_num}!" - # curr_inds = easy_market.get_learnware_ids() - # print("Available ids After Uploading Learnwares:", curr_inds) - # assert len(curr_inds) == self.learnware_num, f"The number of learnwares must be {self.learnware_num}!" + curr_ids = hetero_market.get_learnware_ids() + print("Available ids After Uploading Learnwares:", curr_ids) + assert len(curr_ids) == self.learnware_num, f"The number of learnwares must be {self.learnware_num}!" - # if delete: - # for learnware_id in curr_inds: - # easy_market.delete_learnware(learnware_id) - # self.learnware_num -= 1 - # assert len(easy_market) == self.learnware_num, f"The number of learnwares must be {self.learnware_num}!" + if delete: + for learnware_id in curr_ids: + hetero_market.delete_learnware(learnware_id) + self.learnware_num -= 1 + assert len(hetero_market) == self.learnware_num, f"The number of learnwares must be {self.learnware_num}!" - # curr_inds = easy_market.get_learnware_ids() - # print("Available ids After Deleting Learnwares:", curr_inds) - # assert len(curr_inds) == 0, f"The market should be empty!" + curr_ids = hetero_market.get_learnware_ids() + print("Available ids After Deleting Learnwares:", curr_ids) + assert len(curr_ids) == 0, f"The market should be empty!" - # return easy_market + return hetero_market # def test_search_semantics(self, learnware_num=5): # easy_market = self.test_upload_delete_learnware(learnware_num, delete=False) From c31f5f4dc87724784c9c7000fa32dd9fb7481cb3 Mon Sep 17 00:00:00 2001 From: bxdd Date: Tue, 7 Nov 2023 14:25:51 +0800 Subject: [PATCH 20/27] [FIX] fix cannot import generate_rkme bugs --- docs/workflow/submit.rst | 4 ++-- examples/dataset_m5_workflow/main.py | 7 +++---- examples/dataset_pfs_workflow/main.py | 8 +++----- examples/dataset_text_workflow/main.py | 13 ++++--------- examples/workflow_by_code/main.py | 7 +++---- learnware/reuse/job_selector.py | 2 +- tests/test_market/test_easy.py | 6 +++--- tests/test_specification/test_rkme.py | 5 ++--- tests/test_workflow/test_workflow.py | 8 ++++---- 9 files changed, 25 insertions(+), 35 deletions(-) diff --git a/docs/workflow/submit.rst b/docs/workflow/submit.rst index fe097c3..2d82936 100644 --- a/docs/workflow/submit.rst +++ b/docs/workflow/submit.rst @@ -80,10 +80,10 @@ the following code snippet offers guidance on how to construct and store the RKM .. code-block:: python - import learnware.specification as specification + from learnware.specification import generate_rkme_spec # generate rkme specification for digits dataset - spec = specification.utils.generate_rkme_spec(X=data_X) + spec = generate_rkme_spec(X=data_X) spec.save("stat.json") Significantly, the RKME generation process is entirely conducted on your local machine, without any involvement of cloud services, diff --git a/examples/dataset_m5_workflow/main.py b/examples/dataset_m5_workflow/main.py index 009b557..bfdbe71 100644 --- a/examples/dataset_m5_workflow/main.py +++ b/examples/dataset_m5_workflow/main.py @@ -9,9 +9,8 @@ from shutil import copyfile, rmtree import learnware from learnware.market import EasyMarket, BaseUserInfo from learnware.market import database_ops -from learnware.learnware import Learnware from learnware.reuse import JobSelectorReuser, AveragingReuser -import learnware.specification as specification +from learnware.specification import generate_rkme_spec from m5 import DataLoader from learnware.logger import get_module_logger @@ -88,7 +87,7 @@ class M5DatasetWorkflow: for idx in tqdm(idx_list): train_x, train_y, test_x, test_y = m5.get_idx_data(idx) st = time.time() - spec = specification.utils.generate_rkme_spec(X=train_x, gamma=0.1, cuda_idx=0) + spec = generate_rkme_spec(X=train_x, gamma=0.1, cuda_idx=0) ed = time.time() logger.info("Stat spec generated in %.3f s" % (ed - st)) @@ -140,7 +139,7 @@ class M5DatasetWorkflow: for idx in idx_list: train_x, train_y, test_x, test_y = m5.get_idx_data(idx) - user_spec = specification.utils.generate_rkme_spec(X=test_x, gamma=0.1, cuda_idx=0) + user_spec = generate_rkme_spec(X=test_x, gamma=0.1, cuda_idx=0) user_spec_path = f"./user_spec/user_{idx}.json" user_spec.save(user_spec_path) diff --git a/examples/dataset_pfs_workflow/main.py b/examples/dataset_pfs_workflow/main.py index b5cbdd8..abe80cd 100644 --- a/examples/dataset_pfs_workflow/main.py +++ b/examples/dataset_pfs_workflow/main.py @@ -8,10 +8,8 @@ from shutil import copyfile, rmtree import learnware from learnware.market import EasyMarket, BaseUserInfo -from learnware.market import database_ops -from learnware.learnware import Learnware from learnware.reuse import JobSelectorReuser, AveragingReuser -import learnware.specification as specification +from learnware.specification import generate_rkme_spec from pfs import Dataloader from learnware.logger import get_module_logger @@ -86,7 +84,7 @@ class PFSDatasetWorkflow: for idx in tqdm(idx_list): train_x, train_y, test_x, test_y = pfs.get_idx_data(idx) st = time.time() - spec = specification.utils.generate_rkme_spec(X=train_x, gamma=0.1, cuda_idx=0) + spec = generate_rkme_spec(X=train_x, gamma=0.1, cuda_idx=0) ed = time.time() logger.info("Stat spec generated in %.3f s" % (ed - st)) @@ -138,7 +136,7 @@ class PFSDatasetWorkflow: for idx in idx_list: train_x, train_y, test_x, test_y = pfs.get_idx_data(idx) - user_spec = specification.utils.generate_rkme_spec(X=test_x, gamma=0.1, cuda_idx=0) + user_spec = generate_rkme_spec(X=test_x, gamma=0.1, cuda_idx=0) user_spec_path = f"./user_spec/user_{idx}.json" user_spec.save(user_spec_path) diff --git a/examples/dataset_text_workflow/main.py b/examples/dataset_text_workflow/main.py index e7e1c38..406aad9 100644 --- a/examples/dataset_text_workflow/main.py +++ b/examples/dataset_text_workflow/main.py @@ -10,9 +10,7 @@ import time import pickle from learnware.market import instantiate_learnware_market, BaseUserInfo -from learnware.market import database_ops -from learnware.learnware import Learnware -import learnware.specification as specification +from learnware.specification import RKMETextSpecification from learnware.logger import get_module_logger from shutil import copyfile, rmtree @@ -99,8 +97,7 @@ def prepare_learnware(data_path, model_path, init_file_path, yaml_path, save_roo semantic_spec = semantic_specs[0] st = time.time() - # user_spec = specification.utils.generate_rkme_spec(X=X, gamma=0.1, cuda_idx=0) - user_spec = specification.RKMETextSpecification() + user_spec = RKMETextSpecification() user_spec.generate_stat_spec_from_data(X=X) ed = time.time() logger.info("Stat spec generated in %.3f s" % (ed - st)) @@ -163,10 +160,8 @@ def test_search(gamma=0.1, load_market=True): user_data = pickle.load(f) with open(user_label_path, "rb") as f: user_label = pickle.load(f) - # user_data = np.load(user_data_path) - # user_label = np.load(user_label_path) - # user_stat_spec = specification.utils.generate_rkme_spec(X=user_data, gamma=gamma, cuda_idx=0) - user_stat_spec = specification.RKMETextSpecification() + + user_stat_spec = RKMETextSpecification() user_stat_spec.generate_stat_spec_from_data(X=user_data) user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={"RKMETextSpecification": user_stat_spec}) logger.info("Searching Market for user: %d" % (i)) diff --git a/examples/workflow_by_code/main.py b/examples/workflow_by_code/main.py index 2f62db0..afedc41 100644 --- a/examples/workflow_by_code/main.py +++ b/examples/workflow_by_code/main.py @@ -12,8 +12,7 @@ from shutil import copyfile, rmtree import learnware from learnware.market import EasyMarket, BaseUserInfo from learnware.reuse import JobSelectorReuser, AveragingReuser -import learnware.specification as specification -from learnware.utils import get_module_by_module_path +from learnware.specification import generate_rkme_spec curr_root = os.path.dirname(os.path.abspath(__file__)) @@ -54,7 +53,7 @@ class LearnwareMarketWorkflow: joblib.dump(clf, os.path.join(dir_path, "svm.pkl")) - spec = specification.utils.generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0) + spec = generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0) spec.save(os.path.join(dir_path, "svm.json")) init_file = os.path.join(dir_path, "__init__.py") @@ -174,7 +173,7 @@ class LearnwareMarketWorkflow: X, y = load_digits(return_X_y=True) _, data_X, _, data_y = train_test_split(X, y, test_size=0.3, shuffle=True) - stat_spec = specification.utils.generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0) + stat_spec = generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0) user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={"RKMETableSpecification": stat_spec}) _, _, _, mixture_learnware_list = easy_market.search_learnware(user_info) diff --git a/learnware/reuse/job_selector.py b/learnware/reuse/job_selector.py index a131fd5..5e3a71f 100644 --- a/learnware/reuse/job_selector.py +++ b/learnware/reuse/job_selector.py @@ -11,7 +11,7 @@ from .base import BaseReuser from ..market.utils import parse_specification_type from ..learnware import Learnware from ..specification import RKMETableSpecification, RKMETextSpecification -from ..specification.utils import generate_rkme_spec +from ..specification import generate_rkme_spec from ..logger import get_module_logger logger = get_module_logger("job_selector_reuse") diff --git a/tests/test_market/test_easy.py b/tests/test_market/test_easy.py index bb03839..0ca779f 100644 --- a/tests/test_market/test_easy.py +++ b/tests/test_market/test_easy.py @@ -12,7 +12,7 @@ from shutil import copyfile, rmtree import learnware from learnware.market import instantiate_learnware_market, BaseUserInfo -import learnware.specification as specification +from learnware.specification import RKMETableSpecification, generate_rkme_spec curr_root = os.path.dirname(os.path.abspath(__file__)) @@ -62,7 +62,7 @@ class TestMarket(unittest.TestCase): joblib.dump(clf, os.path.join(dir_path, "svm.pkl")) - spec = specification.utils.generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0) + spec = generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0) spec.save(os.path.join(dir_path, "svm.json")) init_file = os.path.join(dir_path, "__init__.py") @@ -170,7 +170,7 @@ class TestMarket(unittest.TestCase): with zipfile.ZipFile(zip_path, "r") as zip_obj: zip_obj.extractall(path=unzip_dir) - user_spec = specification.rkme.RKMETableSpecification() + user_spec = RKMETableSpecification() user_spec.load(os.path.join(unzip_dir, "svm.json")) user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={"RKMETableSpecification": user_spec}) ( diff --git a/tests/test_specification/test_rkme.py b/tests/test_specification/test_rkme.py index 143bf22..ba280b2 100644 --- a/tests/test_specification/test_rkme.py +++ b/tests/test_specification/test_rkme.py @@ -7,9 +7,8 @@ import unittest import tempfile import numpy as np -import learnware.specification as specification from learnware.specification import RKMETableSpecification, RKMEImageSpecification, RKMETextSpecification -from learnware.specification import generate_rkme_image_spec, generate_rkme_spec +from learnware.specification import generate_rkme_image_spec, generate_rkme_spec, generate_rkme_text_spec class TestRKME(unittest.TestCase): @@ -71,7 +70,7 @@ class TestRKME(unittest.TestCase): return text_list def _test_text_rkme(X): - rkme = specification.utils.generate_rkme_text_spec(X) + rkme = generate_rkme_text_spec(X) with tempfile.TemporaryDirectory(prefix="learnware_") as tempdir: rkme_path = os.path.join(tempdir, "rkme.json") diff --git a/tests/test_workflow/test_workflow.py b/tests/test_workflow/test_workflow.py index fea00d9..3b3579c 100644 --- a/tests/test_workflow/test_workflow.py +++ b/tests/test_workflow/test_workflow.py @@ -13,7 +13,7 @@ from shutil import copyfile, rmtree import learnware from learnware.market import EasyMarket, BaseUserInfo from learnware.reuse import JobSelectorReuser, AveragingReuser, EnsemblePruningReuser -import learnware.specification as specification +from learnware.specification import generate_rkme_spec, RKMETableSpecification curr_root = os.path.dirname(os.path.abspath(__file__)) @@ -57,7 +57,7 @@ class TestAllWorkflow(unittest.TestCase): joblib.dump(clf, os.path.join(dir_path, "svm.pkl")) - spec = specification.utils.generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0) + spec = generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0) spec.save(os.path.join(dir_path, "svm.json")) init_file = os.path.join(dir_path, "__init__.py") @@ -159,7 +159,7 @@ class TestAllWorkflow(unittest.TestCase): with zipfile.ZipFile(zip_path, "r") as zip_obj: zip_obj.extractall(path=unzip_dir) - user_spec = specification.RKMETableSpecification() + user_spec = RKMETableSpecification() user_spec.load(os.path.join(unzip_dir, "svm.json")) user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={"RKMETableSpecification": user_spec}) ( @@ -185,7 +185,7 @@ class TestAllWorkflow(unittest.TestCase): X, y = load_digits(return_X_y=True) train_X, data_X, train_y, data_y = train_test_split(X, y, test_size=0.3, shuffle=True) - stat_spec = specification.utils.generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0) + stat_spec = generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0) user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={"RKMETableSpecification": stat_spec}) _, _, _, mixture_learnware_list = easy_market.search_learnware(user_info) From 4d24827533fa0e05a15756658f6cdc342bd68c13 Mon Sep 17 00:00:00 2001 From: bxdd Date: Tue, 7 Nov 2023 14:58:20 +0800 Subject: [PATCH 21/27] [MNT] add traceback --- learnware/market/base.py | 2 + learnware/market/classes.py | 2 + tests/test_market/test_easy.py | 64 +++++++++++++++++++++------- tests/test_workflow/test_workflow.py | 8 +--- 4 files changed, 54 insertions(+), 22 deletions(-) diff --git a/learnware/market/base.py b/learnware/market/base.py index 9894c65..1921323 100644 --- a/learnware/market/base.py +++ b/learnware/market/base.py @@ -1,5 +1,6 @@ from __future__ import annotations +import traceback import zipfile import tempfile from typing import Tuple, Any, List, Union @@ -90,6 +91,7 @@ class LearnwareMarket: return BaseChecker.INVALID_LEARNWARE return final_status except Exception as err: + traceback.print_exc( ) logger.warning(f"Check learnware failed! Due to {err}.") return BaseChecker.INVALID_LEARNWARE diff --git a/learnware/market/classes.py b/learnware/market/classes.py index deee3d8..1808b74 100644 --- a/learnware/market/classes.py +++ b/learnware/market/classes.py @@ -1,3 +1,4 @@ +import traceback from .base import BaseChecker from ..learnware import Learnware from ..client.container import LearnwaresContainer @@ -17,6 +18,7 @@ class CondaChecker(BaseChecker): learnwares = env_container.get_learnwares_with_container() check_status = self.inner_checker(learnwares[0]) except Exception as e: + traceback.print_exc( ) logger.warning(f"Conda Checker failed due to installed learnware failed and {e}") return BaseChecker.INVALID_LEARNWARE return check_status diff --git a/tests/test_market/test_easy.py b/tests/test_market/test_easy.py index 0ca779f..5d4fb02 100644 --- a/tests/test_market/test_easy.py +++ b/tests/test_market/test_easy.py @@ -13,11 +13,12 @@ from shutil import copyfile, rmtree import learnware from learnware.market import instantiate_learnware_market, BaseUserInfo from learnware.specification import RKMETableSpecification, generate_rkme_spec +from learnware.reuse import JobSelectorReuser, AveragingReuser, EnsemblePruningReuser curr_root = os.path.dirname(os.path.abspath(__file__)) user_semantic = { - "Data": {"Values": ["Image"], "Type": "Class"}, + "Data": {"Values": ["Table"], "Type": "Class"}, "Task": { "Values": ["Classification"], "Type": "Class", @@ -26,6 +27,9 @@ user_semantic = { "Scenario": {"Values": ["Education"], "Type": "Tag"}, "Description": {"Values": "", "Type": "String"}, "Name": {"Values": "", "Type": "String"}, + "Input": { + + }, "Output": { "Dimension": 10, "Description": { @@ -103,11 +107,11 @@ class TestMarket(unittest.TestCase): semantic_spec = copy.deepcopy(user_semantic) semantic_spec["Name"]["Values"] = "learnware_%d" % (idx) semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx) + semantic_spec["Output"] = {"Dimension": 1, "Description": {"0": "The label of the hand-written digit."}} easy_market.add_learnware(zip_path, semantic_spec) print("Total Item:", len(easy_market)) assert len(easy_market) == self.learnware_num, f"The number of learnwares must be {self.learnware_num}!" - curr_inds = easy_market.get_learnware_ids() print("Available ids After Uploading Learnwares:", curr_inds) assert len(curr_inds) == self.learnware_num, f"The number of learnwares must be {self.learnware_num}!" @@ -128,31 +132,29 @@ class TestMarket(unittest.TestCase): easy_market = self.test_upload_delete_learnware(learnware_num, delete=False) print("Total Item:", len(easy_market)) assert len(easy_market) == self.learnware_num, f"The number of learnwares must be {self.learnware_num}!" + test_folder = os.path.join(curr_root, "test_semantics") + + # unzip -o -q zip_path -d unzip_dir + if os.path.exists(test_folder): + rmtree(test_folder) + os.makedirs(test_folder, exist_ok=True) + + with zipfile.ZipFile(self.zip_path_list[0], "r") as zip_obj: + zip_obj.extractall(path=test_folder) semantic_spec = copy.deepcopy(user_semantic) semantic_spec["Name"]["Values"] = f"learnware_{learnware_num - 1}" + semantic_spec["Description"]["Values"] = f"test_learnware_number_{learnware_num - 1}" user_info = BaseUserInfo(semantic_spec=semantic_spec) _, single_learnware_list, _, _ = easy_market.search_learnware(user_info) print("User info:", user_info.get_semantic_spec()) print(f"Search result:") - assert len(single_learnware_list) == 1, f"Exact semantic search failed!" for learnware in single_learnware_list: - semantic_spec1 = learnware.get_specification().get_semantic_spec() - print("Choose learnware:", learnware.id, semantic_spec1) - assert semantic_spec1["Name"]["Values"] == semantic_spec["Name"]["Values"], f"Exact semantic search failed!" + print("Choose learnware:", learnware.id, learnware.get_specification().get_semantic_spec()) - semantic_spec["Name"]["Values"] = "laernwaer" - user_info = BaseUserInfo(semantic_spec=semantic_spec) - _, single_learnware_list, _, _ = easy_market.search_learnware(user_info) - - print("User info:", user_info.get_semantic_spec()) - print(f"Search result:") - assert len(single_learnware_list) == self.learnware_num, f"Fuzzy semantic search failed!" - for learnware in single_learnware_list: - semantic_spec1 = learnware.get_specification().get_semantic_spec() - print("Choose learnware:", learnware.id, semantic_spec1) + rmtree(test_folder) # rm -r test_folder def test_stat_search(self, learnware_num=5): easy_market = self.test_upload_delete_learnware(learnware_num, delete=False) @@ -190,6 +192,35 @@ class TestMarket(unittest.TestCase): rmtree(test_folder) # rm -r test_folder + def test_learnware_reuse(self, learnware_num=5): + easy_market = self.test_upload_delete_learnware(learnware_num, delete=False) + print("Total Item:", len(easy_market)) + + X, y = load_digits(return_X_y=True) + train_X, data_X, train_y, data_y = train_test_split(X, y, test_size=0.3, shuffle=True) + + stat_spec = generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0) + user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={"RKMETableSpecification": stat_spec}) + + _, _, _, mixture_learnware_list = easy_market.search_learnware(user_info) + + # Based on user information, the learnware market returns a list of learnwares (learnware_list) + # Use jobselector reuser to reuse the searched learnwares to make prediction + reuse_job_selector = JobSelectorReuser(learnware_list=mixture_learnware_list) + job_selector_predict_y = reuse_job_selector.predict(user_data=data_X) + + # Use averaging ensemble reuser to reuse the searched learnwares to make prediction + reuse_ensemble = AveragingReuser(learnware_list=mixture_learnware_list, mode="vote_by_prob") + ensemble_predict_y = reuse_ensemble.predict(user_data=data_X) + + # Use ensemble pruning reuser to reuse the searched learnwares to make prediction + reuse_ensemble = EnsemblePruningReuser(learnware_list=mixture_learnware_list, mode="classification") + reuse_ensemble.fit(train_X[-200:], train_y[-200:]) + ensemble_pruning_predict_y = reuse_ensemble.predict(user_data=data_X) + + print("Job Selector Acc:", np.sum(np.argmax(job_selector_predict_y, axis=1) == data_y) / len(data_y)) + print("Averaging Reuser Acc:", np.sum(np.argmax(ensemble_predict_y, axis=1) == data_y) / len(data_y)) + print("Ensemble Pruning Reuser Acc:", np.sum(ensemble_pruning_predict_y == data_y) / len(data_y)) def suite(): _suite = unittest.TestSuite() @@ -197,6 +228,7 @@ def suite(): _suite.addTest(TestMarket("test_upload_delete_learnware")) _suite.addTest(TestMarket("test_search_semantics")) _suite.addTest(TestMarket("test_stat_search")) + _suite.addTest(TestMarket("test_learnware_reuse")) return _suite diff --git a/tests/test_workflow/test_workflow.py b/tests/test_workflow/test_workflow.py index 3b3579c..aaeab40 100644 --- a/tests/test_workflow/test_workflow.py +++ b/tests/test_workflow/test_workflow.py @@ -13,12 +13,12 @@ from shutil import copyfile, rmtree import learnware from learnware.market import EasyMarket, BaseUserInfo from learnware.reuse import JobSelectorReuser, AveragingReuser, EnsemblePruningReuser -from learnware.specification import generate_rkme_spec, RKMETableSpecification +from learnware.specification import RKMETableSpecification, generate_rkme_spec curr_root = os.path.dirname(os.path.abspath(__file__)) user_semantic = { - "Data": {"Values": ["Table"], "Type": "Class"}, + "Data": {"Values": ["Image"], "Type": "Class"}, "Task": { "Values": ["Classification"], "Type": "Class", @@ -96,10 +96,6 @@ class TestAllWorkflow(unittest.TestCase): semantic_spec = copy.deepcopy(user_semantic) semantic_spec["Name"]["Values"] = "learnware_%d" % (idx) semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx) - semantic_spec["Input"] = {"Dimension": 64} - semantic_spec["Input"].update( - {f"{i}": f"The value in the digit image with row is {i // 8} and col is {i % 8}." for i in range(64)} - ) semantic_spec["Output"] = {"Dimension": 1, "Description": {"0": "The label of the hand-written digit."}} easy_market.add_learnware(zip_path, semantic_spec) From 4e66754de79e0f573bc992f369ed026700663f79 Mon Sep 17 00:00:00 2001 From: bxdd Date: Tue, 7 Nov 2023 15:16:55 +0800 Subject: [PATCH 22/27] [FIX] fix bug s in test_workflow --- learnware/market/easy2/organizer.py | 2 +- tests/test_market/test_easy.py | 14 +++----------- tests/test_workflow/test_workflow.py | 2 +- 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/learnware/market/easy2/organizer.py b/learnware/market/easy2/organizer.py index 6c0327d..f6122ca 100644 --- a/learnware/market/easy2/organizer.py +++ b/learnware/market/easy2/organizer.py @@ -337,7 +337,7 @@ class EasyOrganizer(BaseOrganizer): Learnware ids """ if check_status is None: - filtered_ids = self.use_flags.keys() + filtered_ids = list(self.use_flags.keys()) elif check_status in [BaseChecker.NONUSABLE_LEARNWARE, BaseChecker.USABLE_LEARWARE]: filtered_ids = [key for key, value in self.use_flags.items() if value == check_status] else: diff --git a/tests/test_market/test_easy.py b/tests/test_market/test_easy.py index 5d4fb02..973864d 100644 --- a/tests/test_market/test_easy.py +++ b/tests/test_market/test_easy.py @@ -27,15 +27,6 @@ user_semantic = { "Scenario": {"Values": ["Education"], "Type": "Tag"}, "Description": {"Values": "", "Type": "String"}, "Name": {"Values": "", "Type": "String"}, - "Input": { - - }, - "Output": { - "Dimension": 10, - "Description": { - "0": "the probability of the label is zero", - }, - }, } @@ -47,7 +38,7 @@ class TestMarket(unittest.TestCase): def _init_learnware_market(self): """initialize learnware market""" - easy_market = instantiate_learnware_market(market_id="sklearn_digits", name="easy", rebuild=True) + easy_market = instantiate_learnware_market(market_id="sklearn_digits_easy", name="easy", rebuild=True) return easy_market def test_prepare_learnware_randomly(self, learnware_num=5): @@ -107,7 +98,8 @@ class TestMarket(unittest.TestCase): semantic_spec = copy.deepcopy(user_semantic) semantic_spec["Name"]["Values"] = "learnware_%d" % (idx) semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx) - semantic_spec["Output"] = {"Dimension": 1, "Description": {"0": "The label of the hand-written digit."}} + semantic_spec["Input"] = {"Dimension": 64, "Description": {f"{i}": f"The value in the grid {i // 8}{i % 8} of the image of hand-written digit." for i in range(64)}} + semantic_spec["Output"] = {"Dimension": 10, "Description": {f"{i}": "The probability for each digit for 0 to 9." for i in range(10)}} easy_market.add_learnware(zip_path, semantic_spec) print("Total Item:", len(easy_market)) diff --git a/tests/test_workflow/test_workflow.py b/tests/test_workflow/test_workflow.py index aaeab40..d2b9c38 100644 --- a/tests/test_workflow/test_workflow.py +++ b/tests/test_workflow/test_workflow.py @@ -96,7 +96,7 @@ class TestAllWorkflow(unittest.TestCase): semantic_spec = copy.deepcopy(user_semantic) semantic_spec["Name"]["Values"] = "learnware_%d" % (idx) semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx) - semantic_spec["Output"] = {"Dimension": 1, "Description": {"0": "The label of the hand-written digit."}} + semantic_spec["Output"] = {"Dimension": 10, "Description": {f"{i}": "The probability for each digit for 0 to 9." for i in range(10)}} easy_market.add_learnware(zip_path, semantic_spec) print("Total Item:", len(easy_market)) From abe8489e0124f25789a3bac8638b7fa3bbf664d9 Mon Sep 17 00:00:00 2001 From: bxdd Date: Tue, 7 Nov 2023 15:19:07 +0800 Subject: [PATCH 23/27] [MNT] black foramt --- .../dataset_pfs_workflow/pfs/pfs_cross_transfer.py | 4 +--- learnware/market/base.py | 2 +- learnware/market/classes.py | 2 +- tests/test_market/test_easy.py | 14 ++++++++++++-- tests/test_workflow/test_workflow.py | 5 ++++- 5 files changed, 19 insertions(+), 8 deletions(-) diff --git a/examples/dataset_pfs_workflow/pfs/pfs_cross_transfer.py b/examples/dataset_pfs_workflow/pfs/pfs_cross_transfer.py index 5f69127..93a3fa3 100644 --- a/examples/dataset_pfs_workflow/pfs/pfs_cross_transfer.py +++ b/examples/dataset_pfs_workflow/pfs/pfs_cross_transfer.py @@ -85,9 +85,7 @@ def get_split_errs(algo): split = train_xs.shape[0] - proportion_list[tmp] model.fit( - train_xs[ - split:, - ], + train_xs[split:,], train_ys[split:], eval_set=[(val_xs, val_ys)], early_stopping_rounds=50, diff --git a/learnware/market/base.py b/learnware/market/base.py index 1921323..2349614 100644 --- a/learnware/market/base.py +++ b/learnware/market/base.py @@ -91,7 +91,7 @@ class LearnwareMarket: return BaseChecker.INVALID_LEARNWARE return final_status except Exception as err: - traceback.print_exc( ) + traceback.print_exc() logger.warning(f"Check learnware failed! Due to {err}.") return BaseChecker.INVALID_LEARNWARE diff --git a/learnware/market/classes.py b/learnware/market/classes.py index 1808b74..9c99555 100644 --- a/learnware/market/classes.py +++ b/learnware/market/classes.py @@ -18,7 +18,7 @@ class CondaChecker(BaseChecker): learnwares = env_container.get_learnwares_with_container() check_status = self.inner_checker(learnwares[0]) except Exception as e: - traceback.print_exc( ) + traceback.print_exc() logger.warning(f"Conda Checker failed due to installed learnware failed and {e}") return BaseChecker.INVALID_LEARNWARE return check_status diff --git a/tests/test_market/test_easy.py b/tests/test_market/test_easy.py index 973864d..492c8a9 100644 --- a/tests/test_market/test_easy.py +++ b/tests/test_market/test_easy.py @@ -98,8 +98,17 @@ class TestMarket(unittest.TestCase): semantic_spec = copy.deepcopy(user_semantic) semantic_spec["Name"]["Values"] = "learnware_%d" % (idx) semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx) - semantic_spec["Input"] = {"Dimension": 64, "Description": {f"{i}": f"The value in the grid {i // 8}{i % 8} of the image of hand-written digit." for i in range(64)}} - semantic_spec["Output"] = {"Dimension": 10, "Description": {f"{i}": "The probability for each digit for 0 to 9." for i in range(10)}} + semantic_spec["Input"] = { + "Dimension": 64, + "Description": { + f"{i}": f"The value in the grid {i // 8}{i % 8} of the image of hand-written digit." + for i in range(64) + }, + } + semantic_spec["Output"] = { + "Dimension": 10, + "Description": {f"{i}": "The probability for each digit for 0 to 9." for i in range(10)}, + } easy_market.add_learnware(zip_path, semantic_spec) print("Total Item:", len(easy_market)) @@ -214,6 +223,7 @@ class TestMarket(unittest.TestCase): print("Averaging Reuser Acc:", np.sum(np.argmax(ensemble_predict_y, axis=1) == data_y) / len(data_y)) print("Ensemble Pruning Reuser Acc:", np.sum(ensemble_pruning_predict_y == data_y) / len(data_y)) + def suite(): _suite = unittest.TestSuite() _suite.addTest(TestMarket("test_prepare_learnware_randomly")) diff --git a/tests/test_workflow/test_workflow.py b/tests/test_workflow/test_workflow.py index d2b9c38..e9439a0 100644 --- a/tests/test_workflow/test_workflow.py +++ b/tests/test_workflow/test_workflow.py @@ -96,7 +96,10 @@ class TestAllWorkflow(unittest.TestCase): semantic_spec = copy.deepcopy(user_semantic) semantic_spec["Name"]["Values"] = "learnware_%d" % (idx) semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx) - semantic_spec["Output"] = {"Dimension": 10, "Description": {f"{i}": "The probability for each digit for 0 to 9." for i in range(10)}} + semantic_spec["Output"] = { + "Dimension": 10, + "Description": {f"{i}": "The probability for each digit for 0 to 9." for i in range(10)}, + } easy_market.add_learnware(zip_path, semantic_spec) print("Total Item:", len(easy_market)) From 6573f01e6272c273aeb0aef927cfad77af75ab7a Mon Sep 17 00:00:00 2001 From: bxdd Date: Tue, 7 Nov 2023 15:28:38 +0800 Subject: [PATCH 24/27] [MNT] modify generate_stat_spec interface --- learnware/specification/utils.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/learnware/specification/utils.py b/learnware/specification/utils.py index 09d66c1..276fbb7 100644 --- a/learnware/specification/utils.py +++ b/learnware/specification/utils.py @@ -221,7 +221,7 @@ def generate_rkme_text_spec( return rkme_text_spec -def generate_stat_spec(X: np.ndarray) -> BaseStatSpecification: +def generate_stat_spec(type="table", *args, **kwargs) -> BaseStatSpecification: """ Interface for users to generate statistical specification. Return a StatSpecification object, use .save() method to save as npy file. @@ -237,4 +237,12 @@ def generate_stat_spec(X: np.ndarray) -> BaseStatSpecification: StatSpecification A StatSpecification object """ + if type == "table": + return generate_rkme_spec(*args, **kwargs) + elif type == "text": + return generate_rkme_text_spec(*args, **kwargs) + elif type == "image": + return generate_rkme_image_spec(*args, **kwargs) + else: + raise TypeError(f"type {type} is not supported!") return None From 53e9dc1bf4f587b1d498124d5580517b2537ecc1 Mon Sep 17 00:00:00 2001 From: bxdd Date: Tue, 7 Nov 2023 15:33:05 +0800 Subject: [PATCH 25/27] [MNT] refactor specifiation module --- learnware/specification/__init__.py | 2 +- learnware/specification/module.py | 223 ++++++++++++++++++++++++++++ learnware/specification/utils.py | 221 +-------------------------- 3 files changed, 225 insertions(+), 221 deletions(-) create mode 100644 learnware/specification/module.py diff --git a/learnware/specification/__init__.py b/learnware/specification/__init__.py index b27ef5b..4ecedc3 100644 --- a/learnware/specification/__init__.py +++ b/learnware/specification/__init__.py @@ -1,4 +1,4 @@ -from .utils import generate_stat_spec, generate_rkme_spec, generate_rkme_image_spec +from .module import generate_stat_spec, generate_rkme_spec, generate_rkme_image_spec, generate_rkme_text_spec from .base import Specification, BaseStatSpecification from .regular import ( RegularStatsSpecification, diff --git a/learnware/specification/module.py b/learnware/specification/module.py new file mode 100644 index 0000000..7ae6ded --- /dev/null +++ b/learnware/specification/module.py @@ -0,0 +1,223 @@ +import torch +import numpy as np +import pandas as pd +from typing import Union, List + +from .utils import convert_to_numpy +from .base import BaseStatSpecification +from .regular import RKMETableSpecification, RKMEImageSpecification, RKMETextSpecification +from ..config import C + + +def generate_rkme_spec( + X: Union[np.ndarray, pd.DataFrame, torch.Tensor], + gamma: float = 0.1, + reduced_set_size: int = 100, + step_size: float = 0.1, + steps: int = 3, + nonnegative_beta: bool = True, + reduce: bool = True, + cuda_idx: int = None, +) -> RKMETableSpecification: + """ + Interface for users to generate Reduced Kernel Mean Embedding (RKME) specification. + Return a RKMETableSpecification object, use .save() method to save as json file. + + Parameters + ---------- + X : np.ndarray, pd.DataFrame, or torch.Tensor + Raw data in np.ndarray, pd.DataFrame, or torch.Tensor format. + The shape of X: + First dimension represents the number of samples (data points). + The remaining dimensions represent the dimensions (features) of each sample. + For example, if X has shape (100, 3), it means there are 100 samples, and each sample has 3 features. + gamma : float + Bandwidth in gaussian kernel, by default 0.1. + reduced_set_size : int + Size of the construced reduced set. + step_size : float + Step size for gradient descent in the iterative optimization. + steps : int + Total rounds in the iterative optimization. + nonnegative_beta : bool, optional + True if weights for the reduced set are intended to be kept non-negative, by default False. + reduce : bool, optional + Whether shrink original data to a smaller set, by default True + cuda_idx : int + A flag indicating whether use CUDA during RKME computation. -1 indicates CUDA not used. + None indicates that CUDA is automatically selected. + + Returns + ------- + RKMETableSpecification + A RKMETableSpecification object + """ + # Convert data type + X = convert_to_numpy(X) + X = np.ascontiguousarray(X).astype(np.float32) + + # Check reduced_set_size + max_reduced_set_size = C.max_reduced_set_size + if reduced_set_size * X[0].size > max_reduced_set_size: + reduced_set_size = max(20, max_reduced_set_size // X[0].size) + + # Check cuda_idx + if not torch.cuda.is_available() or cuda_idx == -1: + cuda_idx = -1 + else: + num_cuda_devices = torch.cuda.device_count() + if cuda_idx is None or not (cuda_idx >= 0 and cuda_idx < num_cuda_devices): + cuda_idx = 0 + + # Generate rkme spec + rkme_spec = RKMETableSpecification(gamma=gamma, cuda_idx=cuda_idx) + rkme_spec.generate_stat_spec_from_data(X, reduced_set_size, step_size, steps, nonnegative_beta, reduce) + return rkme_spec + + +def generate_rkme_image_spec( + X: Union[np.ndarray, torch.Tensor], + reduced_set_size: int = 50, + step_size: float = 0.01, + steps: int = 100, + resize: bool = True, + nonnegative_beta: bool = True, + reduce: bool = True, + verbose: bool = True, + cuda_idx: int = None, +) -> RKMEImageSpecification: + """ + Interface for users to generate Reduced Kernel Mean Embedding (RKME) specification for Image. + Return a RKMEImageSpecification object, use .save() method to save as json file. + + Parameters + ---------- + X : np.ndarray, or torch.Tensor + Raw data in np.ndarray, or torch.Tensor format. + The shape of X: [N, C, H, W] + N: Number of images. + C: Number of channels. + H: Height of images. + W: Width of images.s + For example, if X has shape (100, 3, 32, 32), it means there are 100 samples, and each sample is a 3-channel (RGB) image of size 32x32. + reduced_set_size : int + Size of the construced reduced set. + step_size : float + Step size for gradient descent in the iterative optimization. + steps : int + Total rounds in the iterative optimization. + resize : bool + Whether to scale the image to the requested size, by default True. + nonnegative_beta : bool, optional + True if weights for the reduced set are intended to be kept non-negative, by default False. + reduce : bool, optional + Whether shrink original data to a smaller set, by default True + cuda_idx : int + A flag indicating whether use CUDA during RKME computation. -1 indicates CUDA not used. + None indicates that CUDA is automatically selected. + verbose : bool, optional + Whether to print training progress, by default True + + Returns + ------- + RKMEImageSpecification + A RKMEImageSpecification object + """ + + # Check cuda_idx + if not torch.cuda.is_available() or cuda_idx == -1: + cuda_idx = -1 + else: + num_cuda_devices = torch.cuda.device_count() + if cuda_idx is None or not (0 <= cuda_idx < num_cuda_devices): + cuda_idx = 0 + + # Generate rkme spec + rkme_image_spec = RKMEImageSpecification(cuda_idx=cuda_idx) + rkme_image_spec.generate_stat_spec_from_data( + X, reduced_set_size, step_size, steps, resize, nonnegative_beta, reduce, verbose + ) + return rkme_image_spec + + +def generate_rkme_text_spec( + X: List[str], + gamma: float = 0.1, + reduced_set_size: int = 100, + step_size: float = 0.1, + steps: int = 3, + nonnegative_beta: bool = True, + reduce: bool = True, + cuda_idx: int = None, +) -> RKMETextSpecification: + """ + Interface for users to generate Reduced Kernel Mean Embedding (RKME) specification for Text. + Return a RKMETextSpecification object, use .save() method to save as json file. + + Parameters + ---------- + X : List[str] + Raw data of text. + gamma : float + Bandwidth in gaussian kernel, by default 0.1. + reduced_set_size : int + Size of the construced reduced set. + step_size : float + Step size for gradient descent in the iterative optimization. + steps : int + Total rounds in the iterative optimization. + nonnegative_beta : bool, optional + True if weights for the reduced set are intended to be kept non-negative, by default False. + reduce : bool, optional + Whether shrink original data to a smaller set, by default True + cuda_idx : int + A flag indicating whether use CUDA during RKME computation. -1 indicates CUDA not used. + None indicates that CUDA is automatically selected. + + Returns + ------- + RKMETextSpecification + A RKMETextSpecification object + """ + # Check input type + if not isinstance(X, list) or not all(isinstance(item, str) for item in X): + raise TypeError("Input data must be a list of strings.") + + # Check cuda_idx + if not torch.cuda.is_available() or cuda_idx == -1: + cuda_idx = -1 + else: + num_cuda_devices = torch.cuda.device_count() + if cuda_idx is None or not (cuda_idx >= 0 and cuda_idx < num_cuda_devices): + cuda_idx = 0 + + # Generate rkme text spec + rkme_text_spec = RKMETextSpecification(gamma=gamma, cuda_idx=cuda_idx) + rkme_text_spec.generate_stat_spec_from_data(X, reduced_set_size, step_size, steps, nonnegative_beta, reduce) + return rkme_text_spec + + +def generate_stat_spec(type="table", *args, **kwargs) -> BaseStatSpecification: + """ + Interface for users to generate statistical specification. + Return a StatSpecification object, use .save() method to save as npy file. + + Parameters + ---------- + X : np.ndarray + Raw data in np.ndarray format. + Size of array: (n*d) + + Returns + ------- + StatSpecification + A StatSpecification object + """ + if type == "table": + return generate_rkme_spec(*args, **kwargs) + elif type == "text": + return generate_rkme_text_spec(*args, **kwargs) + elif type == "image": + return generate_rkme_image_spec(*args, **kwargs) + else: + raise TypeError(f"type {type} is not supported!") diff --git a/learnware/specification/utils.py b/learnware/specification/utils.py index 276fbb7..fdf9fc0 100644 --- a/learnware/specification/utils.py +++ b/learnware/specification/utils.py @@ -1,11 +1,7 @@ import torch import numpy as np import pandas as pd -from typing import Union, List - -from .base import BaseStatSpecification -from .regular import RKMETableSpecification, RKMEImageSpecification, RKMETextSpecification -from ..config import C +from typing import Union def convert_to_numpy(data: Union[np.ndarray, pd.DataFrame, torch.Tensor]): @@ -31,218 +27,3 @@ def convert_to_numpy(data: Union[np.ndarray, pd.DataFrame, torch.Tensor]): raise TypeError( "Unsupported data format. Please provide a NumPy array, a Pandas DataFrame, or a PyTorch Tensor." ) - - -def generate_rkme_spec( - X: Union[np.ndarray, pd.DataFrame, torch.Tensor], - gamma: float = 0.1, - reduced_set_size: int = 100, - step_size: float = 0.1, - steps: int = 3, - nonnegative_beta: bool = True, - reduce: bool = True, - cuda_idx: int = None, -) -> RKMETableSpecification: - """ - Interface for users to generate Reduced Kernel Mean Embedding (RKME) specification. - Return a RKMETableSpecification object, use .save() method to save as json file. - - Parameters - ---------- - X : np.ndarray, pd.DataFrame, or torch.Tensor - Raw data in np.ndarray, pd.DataFrame, or torch.Tensor format. - The shape of X: - First dimension represents the number of samples (data points). - The remaining dimensions represent the dimensions (features) of each sample. - For example, if X has shape (100, 3), it means there are 100 samples, and each sample has 3 features. - gamma : float - Bandwidth in gaussian kernel, by default 0.1. - reduced_set_size : int - Size of the construced reduced set. - step_size : float - Step size for gradient descent in the iterative optimization. - steps : int - Total rounds in the iterative optimization. - nonnegative_beta : bool, optional - True if weights for the reduced set are intended to be kept non-negative, by default False. - reduce : bool, optional - Whether shrink original data to a smaller set, by default True - cuda_idx : int - A flag indicating whether use CUDA during RKME computation. -1 indicates CUDA not used. - None indicates that CUDA is automatically selected. - - Returns - ------- - RKMETableSpecification - A RKMETableSpecification object - """ - # Convert data type - X = convert_to_numpy(X) - X = np.ascontiguousarray(X).astype(np.float32) - - # Check reduced_set_size - max_reduced_set_size = C.max_reduced_set_size - if reduced_set_size * X[0].size > max_reduced_set_size: - reduced_set_size = max(20, max_reduced_set_size // X[0].size) - - # Check cuda_idx - if not torch.cuda.is_available() or cuda_idx == -1: - cuda_idx = -1 - else: - num_cuda_devices = torch.cuda.device_count() - if cuda_idx is None or not (cuda_idx >= 0 and cuda_idx < num_cuda_devices): - cuda_idx = 0 - - # Generate rkme spec - rkme_spec = RKMETableSpecification(gamma=gamma, cuda_idx=cuda_idx) - rkme_spec.generate_stat_spec_from_data(X, reduced_set_size, step_size, steps, nonnegative_beta, reduce) - return rkme_spec - - -def generate_rkme_image_spec( - X: Union[np.ndarray, torch.Tensor], - reduced_set_size: int = 50, - step_size: float = 0.01, - steps: int = 100, - resize: bool = True, - nonnegative_beta: bool = True, - reduce: bool = True, - verbose: bool = True, - cuda_idx: int = None, -) -> RKMEImageSpecification: - """ - Interface for users to generate Reduced Kernel Mean Embedding (RKME) specification for Image. - Return a RKMEImageSpecification object, use .save() method to save as json file. - - Parameters - ---------- - X : np.ndarray, or torch.Tensor - Raw data in np.ndarray, or torch.Tensor format. - The shape of X: [N, C, H, W] - N: Number of images. - C: Number of channels. - H: Height of images. - W: Width of images.s - For example, if X has shape (100, 3, 32, 32), it means there are 100 samples, and each sample is a 3-channel (RGB) image of size 32x32. - reduced_set_size : int - Size of the construced reduced set. - step_size : float - Step size for gradient descent in the iterative optimization. - steps : int - Total rounds in the iterative optimization. - resize : bool - Whether to scale the image to the requested size, by default True. - nonnegative_beta : bool, optional - True if weights for the reduced set are intended to be kept non-negative, by default False. - reduce : bool, optional - Whether shrink original data to a smaller set, by default True - cuda_idx : int - A flag indicating whether use CUDA during RKME computation. -1 indicates CUDA not used. - None indicates that CUDA is automatically selected. - verbose : bool, optional - Whether to print training progress, by default True - - Returns - ------- - RKMEImageSpecification - A RKMEImageSpecification object - """ - - # Check cuda_idx - if not torch.cuda.is_available() or cuda_idx == -1: - cuda_idx = -1 - else: - num_cuda_devices = torch.cuda.device_count() - if cuda_idx is None or not (0 <= cuda_idx < num_cuda_devices): - cuda_idx = 0 - - # Generate rkme spec - rkme_image_spec = RKMEImageSpecification(cuda_idx=cuda_idx) - rkme_image_spec.generate_stat_spec_from_data( - X, reduced_set_size, step_size, steps, resize, nonnegative_beta, reduce, verbose - ) - return rkme_image_spec - - -def generate_rkme_text_spec( - X: List[str], - gamma: float = 0.1, - reduced_set_size: int = 100, - step_size: float = 0.1, - steps: int = 3, - nonnegative_beta: bool = True, - reduce: bool = True, - cuda_idx: int = None, -) -> RKMETextSpecification: - """ - Interface for users to generate Reduced Kernel Mean Embedding (RKME) specification for Text. - Return a RKMETextSpecification object, use .save() method to save as json file. - - Parameters - ---------- - X : List[str] - Raw data of text. - gamma : float - Bandwidth in gaussian kernel, by default 0.1. - reduced_set_size : int - Size of the construced reduced set. - step_size : float - Step size for gradient descent in the iterative optimization. - steps : int - Total rounds in the iterative optimization. - nonnegative_beta : bool, optional - True if weights for the reduced set are intended to be kept non-negative, by default False. - reduce : bool, optional - Whether shrink original data to a smaller set, by default True - cuda_idx : int - A flag indicating whether use CUDA during RKME computation. -1 indicates CUDA not used. - None indicates that CUDA is automatically selected. - - Returns - ------- - RKMETextSpecification - A RKMETextSpecification object - """ - # Check input type - if not isinstance(X, list) or not all(isinstance(item, str) for item in X): - raise TypeError("Input data must be a list of strings.") - - # Check cuda_idx - if not torch.cuda.is_available() or cuda_idx == -1: - cuda_idx = -1 - else: - num_cuda_devices = torch.cuda.device_count() - if cuda_idx is None or not (cuda_idx >= 0 and cuda_idx < num_cuda_devices): - cuda_idx = 0 - - # Generate rkme text spec - rkme_text_spec = RKMETextSpecification(gamma=gamma, cuda_idx=cuda_idx) - rkme_text_spec.generate_stat_spec_from_data(X, reduced_set_size, step_size, steps, nonnegative_beta, reduce) - return rkme_text_spec - - -def generate_stat_spec(type="table", *args, **kwargs) -> BaseStatSpecification: - """ - Interface for users to generate statistical specification. - Return a StatSpecification object, use .save() method to save as npy file. - - Parameters - ---------- - X : np.ndarray - Raw data in np.ndarray format. - Size of array: (n*d) - - Returns - ------- - StatSpecification - A StatSpecification object - """ - if type == "table": - return generate_rkme_spec(*args, **kwargs) - elif type == "text": - return generate_rkme_text_spec(*args, **kwargs) - elif type == "image": - return generate_rkme_image_spec(*args, **kwargs) - else: - raise TypeError(f"type {type} is not supported!") - return None From 806cdd3134c4de79e9dcc716a3e515a461631e01 Mon Sep 17 00:00:00 2001 From: bxdd Date: Tue, 7 Nov 2023 15:45:45 +0800 Subject: [PATCH 26/27] [FIX] fix bugs in example --- examples/workflow_by_code/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/workflow_by_code/main.py b/examples/workflow_by_code/main.py index afedc41..39b1b6c 100644 --- a/examples/workflow_by_code/main.py +++ b/examples/workflow_by_code/main.py @@ -12,7 +12,7 @@ from shutil import copyfile, rmtree import learnware from learnware.market import EasyMarket, BaseUserInfo from learnware.reuse import JobSelectorReuser, AveragingReuser -from learnware.specification import generate_rkme_spec +from learnware.specification import generate_rkme_spec, RKMETableSpecification curr_root = os.path.dirname(os.path.abspath(__file__)) @@ -147,7 +147,7 @@ class LearnwareMarketWorkflow: with zipfile.ZipFile(zip_path, "r") as zip_obj: zip_obj.extractall(path=unzip_dir) - user_spec = specification.RKMETableSpecification() + user_spec = RKMETableSpecification() user_spec.load(os.path.join(unzip_dir, "svm.json")) user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={"RKMETableSpecification": user_spec}) ( From 4af5bd1c216ea8db5cb04d83c130b02d600f7318 Mon Sep 17 00:00:00 2001 From: bxdd Date: Tue, 7 Nov 2023 15:48:18 +0800 Subject: [PATCH 27/27] [FIX] fix typo --- learnware/market/easy2/checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/learnware/market/easy2/checker.py b/learnware/market/easy2/checker.py index 5e15271..589f3e2 100644 --- a/learnware/market/easy2/checker.py +++ b/learnware/market/easy2/checker.py @@ -141,7 +141,7 @@ class EasyStatChecker(BaseChecker): int(semantic_spec["Output"]["Dimension"]), ): logger.warning( - f"The learnware [{learnware.id}] output dimention mismatch!, where pred_shape={outputs[0].shape}, model_shape={learnware_model.output_shape}, semantic_shape={(int(semantic_spec['Output']['Dimension']), )}" + f"The learnware [{learnware.id}] output dimension mismatch!, where pred_shape={outputs[0].shape}, model_shape={learnware_model.output_shape}, semantic_shape={(int(semantic_spec['Output']['Dimension']), )}" ) return self.INVALID_LEARNWARE