From abfcbb1ab77b7af066b3e0da7643ee4e00cd95a5 Mon Sep 17 00:00:00 2001 From: bxdd Date: Thu, 12 Oct 2023 16:33:43 +0800 Subject: [PATCH 1/2] [ENH] add reuse method in learnware client --- learnware/client/container.py | 12 ++---------- learnware/client/learnware_client.py | 13 +++++++++++++ learnware/learnware/base.py | 7 ++++++- learnware/learnware/reuse.py | 6 +++--- 4 files changed, 24 insertions(+), 14 deletions(-) diff --git a/learnware/client/container.py b/learnware/client/container.py index 15236b0..8dcb195 100644 --- a/learnware/client/container.py +++ b/learnware/client/container.py @@ -129,19 +129,11 @@ class LearnwaresContainer: @staticmethod def _initialize_model_container(model: ModelEnvContainer): - try: - model.init_env_and_metadata() - except Exception as e: - logger.warning(f"fail to initialize model container, due to {e}") - pass + model.init_env_and_metadata() @staticmethod def _destroy_model_container(model: ModelEnvContainer): - try: - model.remove_env() - except Exception as e: - logger.warning(f"fail to destroy model container, due to {e}") - pass + model.remove_env() def __enter__(self): model_list = [_learnware.get_model() for _learnware in self.learnware_list] diff --git a/learnware/client/learnware_client.py b/learnware/client/learnware_client.py index 723267f..645d1ff 100644 --- a/learnware/client/learnware_client.py +++ b/learnware/client/learnware_client.py @@ -1,4 +1,5 @@ import os +import numpy as np import yaml import json import zipfile @@ -7,13 +8,16 @@ import requests import tempfile from enum import Enum from tqdm import tqdm +from typing import List from ..config import C from .. import learnware from . import package_utils +from .container import LearnwaresContainer from ..market.easy import EasyMarket from ..logger import get_module_logger from ..specification import Specification +from ..learnware import BaseReuser, Learnware CHUNK_SIZE = 1024 * 1024 logger = get_module_logger(module_name="LearnwareClient") @@ -427,3 +431,12 @@ class LearnwareClient: logger.info("test ok") pass + + def reuse_learnware(self, input_array: np.ndarray, learnware_list: List[Learnware], learnware_zippaths: List[str], reuser: BaseReuser): + logger.info(f"reuse learnare list {learnware_list} with reuser {reuser}") + with LearnwaresContainer(learnware_list, learnware_zippaths) as env_container: + learnware_list = env_container.get_learnware_list_with_container() + reuser.reset(learnware_list=learnware_list) + result = reuser.predict(input_array) + + return result \ No newline at end of file diff --git a/learnware/learnware/base.py b/learnware/learnware/base.py index af08fcf..4662369 100644 --- a/learnware/learnware/base.py +++ b/learnware/learnware/base.py @@ -77,7 +77,7 @@ class Learnware: class BaseReuser: """Providing the interfaces to reuse the learnwares which is searched by learnware""" - def __init__(self, learnware_list: List[Learnware]): + def __init__(self, learnware_list: List[Learnware] = None): """The initializaiton method for base reuser Parameters @@ -87,6 +87,11 @@ class BaseReuser: """ self.learnware_list = learnware_list + def reset(self, **kwargs): + for _k, _v in kwargs.items(): + if hasattr(_k): + setattr(_k, _v) + def predict(self, user_data: np.ndarray) -> np.ndarray: """Give the final prediction for user data with reused learnware diff --git a/learnware/learnware/reuse.py b/learnware/learnware/reuse.py index b618002..0091e6b 100644 --- a/learnware/learnware/reuse.py +++ b/learnware/learnware/reuse.py @@ -21,7 +21,7 @@ logger = get_module_logger("Reuser") class JobSelectorReuser(BaseReuser): """Baseline Multiple Learnware Reuser using Job Selector Method""" - def __init__(self, learnware_list: List[Learnware], herding_num: int = 1000, use_herding: bool = True): + def __init__(self, learnware_list: List[Learnware] = None, herding_num: int = 1000, use_herding: bool = True): """The initialization method for job selector reuser Parameters @@ -265,7 +265,7 @@ class JobSelectorReuser(BaseReuser): class AveragingReuser(BaseReuser): """Baseline Multiple Learnware Reuser using Ensemble Method""" - def __init__(self, learnware_list: List[Learnware], mode: str): + def __init__(self, learnware_list: List[Learnware] = None, mode: str = 'mean'): """The initialization method for averaging ensemble reuser Parameters @@ -330,7 +330,7 @@ class EnsemblePruningReuser(BaseReuser): References: [1] Yu-Chang Wu, Yi-Xiao He, Chao Qian, and Zhi-Hua Zhou. Multi-objective Evolutionary Ensemble Pruning Guided by Margin Distribution. In: Proceedings of the 17th International Conference on Parallel Problem Solving from Nature (PPSN'22), Dortmund, Germany, 2022. """ - def __init__(self, learnware_list: List[Learnware], mode: str): + def __init__(self, learnware_list: List[Learnware] = None, mode: str = 'classification'): """The initialization method for ensemble pruning reuser Parameters From bf43b582cccedd247f37cf1f596cdcc3112922b5 Mon Sep 17 00:00:00 2001 From: bxdd Date: Thu, 12 Oct 2023 16:36:03 +0800 Subject: [PATCH 2/2] [MNT] black format --- learnware/client/learnware_client.py | 12 +++++++++--- learnware/learnware/base.py | 2 +- learnware/learnware/reuse.py | 4 ++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/learnware/client/learnware_client.py b/learnware/client/learnware_client.py index 645d1ff..2982cdf 100644 --- a/learnware/client/learnware_client.py +++ b/learnware/client/learnware_client.py @@ -432,11 +432,17 @@ class LearnwareClient: logger.info("test ok") pass - def reuse_learnware(self, input_array: np.ndarray, learnware_list: List[Learnware], learnware_zippaths: List[str], reuser: BaseReuser): + def reuse_learnware( + self, + input_array: np.ndarray, + learnware_list: List[Learnware], + learnware_zippaths: List[str], + reuser: BaseReuser, + ): logger.info(f"reuse learnare list {learnware_list} with reuser {reuser}") with LearnwaresContainer(learnware_list, learnware_zippaths) as env_container: learnware_list = env_container.get_learnware_list_with_container() reuser.reset(learnware_list=learnware_list) result = reuser.predict(input_array) - - return result \ No newline at end of file + + return result diff --git a/learnware/learnware/base.py b/learnware/learnware/base.py index 4662369..7f309fd 100644 --- a/learnware/learnware/base.py +++ b/learnware/learnware/base.py @@ -91,7 +91,7 @@ class BaseReuser: for _k, _v in kwargs.items(): if hasattr(_k): setattr(_k, _v) - + def predict(self, user_data: np.ndarray) -> np.ndarray: """Give the final prediction for user data with reused learnware diff --git a/learnware/learnware/reuse.py b/learnware/learnware/reuse.py index 0091e6b..565ba89 100644 --- a/learnware/learnware/reuse.py +++ b/learnware/learnware/reuse.py @@ -265,7 +265,7 @@ class JobSelectorReuser(BaseReuser): class AveragingReuser(BaseReuser): """Baseline Multiple Learnware Reuser using Ensemble Method""" - def __init__(self, learnware_list: List[Learnware] = None, mode: str = 'mean'): + def __init__(self, learnware_list: List[Learnware] = None, mode: str = "mean"): """The initialization method for averaging ensemble reuser Parameters @@ -330,7 +330,7 @@ class EnsemblePruningReuser(BaseReuser): References: [1] Yu-Chang Wu, Yi-Xiao He, Chao Qian, and Zhi-Hua Zhou. Multi-objective Evolutionary Ensemble Pruning Guided by Margin Distribution. In: Proceedings of the 17th International Conference on Parallel Problem Solving from Nature (PPSN'22), Dortmund, Germany, 2022. """ - def __init__(self, learnware_list: List[Learnware] = None, mode: str = 'classification'): + def __init__(self, learnware_list: List[Learnware] = None, mode: str = "classification"): """The initialization method for ensemble pruning reuser Parameters