diff --git a/docs/references/api.rst b/docs/references/api.rst index ebd276f..a2f723b 100644 --- a/docs/references/api.rst +++ b/docs/references/api.rst @@ -11,7 +11,7 @@ Here you can find all ``learnware`` interfaces. Market ==================== -.. autoclass:: learnware.market.BaseMarket +.. autoclass:: learnware.market.LearnwareMarket :members: .. autoclass:: learnware.market.EasyMarket diff --git a/learnware/market/__init__.py b/learnware/market/__init__.py index 1620a5e..bf18990 100644 --- a/learnware/market/__init__.py +++ b/learnware/market/__init__.py @@ -1,5 +1,5 @@ from .anchor import AnchoredUserInfo, AnchoredMarket -from .base import BaseUserInfo, BaseMarket +from .base import BaseUserInfo, LearnwareMarket from .evolve_anchor import EvolvedAnchoredMarket from .evolve import EvolvedMarket from .easy import EasyMarket diff --git a/learnware/market/anchor.py b/learnware/market/anchor.py index 79d5443..bd912f3 100644 --- a/learnware/market/anchor.py +++ b/learnware/market/anchor.py @@ -2,7 +2,7 @@ import os from typing import Tuple, Any, List, Union, Dict from ..learnware import Learnware -from .base import BaseMarket, BaseUserInfo +from .base import LearnwareMarket, BaseUserInfo class AnchoredUserInfo(BaseUserInfo): @@ -42,12 +42,12 @@ class AnchoredUserInfo(BaseUserInfo): self.stat_info[name] = item -class AnchoredMarket(BaseMarket): - """Add the anchor design to the BaseMarket +class AnchoredMarket(LearnwareMarket): + """Add the anchor design to the LearnwareMarket Parameters ---------- - BaseMarket : _type_ + LearnwareMarket : _type_ Basic market version """ diff --git a/learnware/market/base.py b/learnware/market/base.py index 5f3dc56..12c8f9f 100644 --- a/learnware/market/base.py +++ b/learnware/market/base.py @@ -43,7 +43,7 @@ class BaseUserInfo: return self.stat_info.get(name, None) -class BaseMarket: +class LearnwareMarket: """Base interface for market, it provide the interface of search/add/detele/update learnwares""" def __init__( @@ -55,9 +55,9 @@ class BaseMarket: ): self.market_id = market_id self.learnware_organizer = LearnwareOrganizer() if organizer is None else organizer - self.learnware_organizer.reset(market_id=market_id) self.learnware_checker = LearnwareChecker() if checker is None else checker self.learnware_checker.reset(organizer=self.learnware_organizer) + self.learnware_organizer.reset(market_id=market_id, checker=self.learnware_checker) self.learnware_searcher = LearnwareSearcher() if searcher is None else searcher self.learnware_searcher.reset(organizer=self.learnware_organizer) @@ -128,26 +128,7 @@ class BaseMarket: - second is a list of matched learnwares """ - return self.learnware_searcher(user_info, *args, **kwargs) - - def get_learnware_by_ids(self, id: Union[str, List[str]]) -> Union[Learnware, List[Learnware]]: - """ - Get Learnware from market by id - - Parameters - ---------- - id : Union[str, List[str]] - Given one id or a list of ids as target. - - Returns - ------- - Union[Learnware, List[Learnware]] - Return a Learnware object or a list of Learnware objects based on the type of input param. - - - The returned items are search results. - - 'None' indicating the target id not found. - """ - return self.learnware_organizer.get_learnware_by_ids(id) + return self.learnware_searcher(user_info, **kwargs) def delete_learnware(self, id: str, *args, **kwargs) -> bool: """Delete a learnware from market @@ -191,19 +172,35 @@ class BaseMarket: """ raise NotImplementedError("get semantic spec list is not implemented") + + def get_learnware_by_ids(self, id: Union[str, List[str]]) -> Union[Learnware, List[Learnware]]: + """ + Get Learnware from market by id + Parameters + ---------- + id : Union[str, List[str]] + Given one id or a list of ids as target. - def get_learnware_ids(self) -> List[str]: - raise NotImplementedError("get_learnware_ids is not implemented") - - + Returns + ------- + Union[Learnware, List[Learnware]] + Return a Learnware object or a list of Learnware objects based on the type of input param. + + - The returned items are search results. + - 'None' indicating the target id not found. + """ + return self.learnware_organizer.get_learnware_by_ids(id) + + def class LearnwareOrganizer: - def __init__(self, market_id): - self.market_id = market_id + def __init__(self, market_id, organizer: 'LearnwareOrganizer' = None): + self.reset(market_id=market_id, organizer=organizer) - def reset(self, market_id): + def reset(self, market_id, organizer: 'LearnwareOrganizer' ): self.market_id = market_id + self.organizer = organizer def reload_market(self) -> bool: """Reload the learnware organizer when server restared. @@ -327,6 +324,12 @@ class LearnwareOrganizer: """ raise NotImplementedError("get_learnware_path_by_ids is not implemented") + def get_learnware_ids(self, top:int = None): + if top is None: + return list(self.learnware_list.keys()) + else: + return list(self.learnware_list.keys())[:top] + class LearnwareSearcher: def __init__(self, organizer: LearnwareOrganizer = None): self.learnware_oganizer = organizer diff --git a/learnware/market/easy.py b/learnware/market/easy.py index 26f8fb0..c577a82 100644 --- a/learnware/market/easy.py +++ b/learnware/market/easy.py @@ -11,7 +11,7 @@ from cvxopt import solvers, matrix from shutil import copyfile, rmtree from typing import Tuple, Any, List, Union, Dict -from .base import BaseMarket, BaseUserInfo +from .base import LearnwareMarket, BaseUserInfo from .database_ops import DatabaseOperations from .. import utils @@ -24,8 +24,8 @@ from ..specification import RKMEStatSpecification, Specification logger = get_module_logger("market", "INFO") -class EasyMarket(BaseMarket): - """EasyMarket provide an easy and simple implementation for BaseMarket +class EasyMarket(LearnwareMarket): + """EasyMarket provide an easy and simple implementation for LearnwareMarket - EasyMarket stores learnwares with file system and database - EasyMarket search the learnwares with the match of semantical tag and the statistical RKME - EasyMarket does not support the search between heterogeneous features learnwars diff --git a/learnware/market/easy/organizer.py b/learnware/market/easy/organizer.py index 02e1db7..d3a70f8 100644 --- a/learnware/market/easy/organizer.py +++ b/learnware/market/easy/organizer.py @@ -11,7 +11,7 @@ from cvxopt import solvers, matrix from shutil import copyfile, rmtree from typing import Tuple, Any, List, Union, Dict -from ..base import BaseMarket, BaseUserInfo +from ..base import LearnwareMarket, BaseUserInfo from ..database_ops import DatabaseOperations from ... import utils @@ -20,7 +20,7 @@ from ...logger import get_module_logger from ...learnware import Learnware, get_learnware_from_dirpath from ...specification import RKMEStatSpecification, Specification -from ..base import LearnwareOrganizer +from ..base import LearnwareOrganizer, LearnwareChecker from ...logger import get_module_logger logger = get_module_logger("easy_organizer") @@ -28,9 +28,9 @@ logger = get_module_logger("easy_organizer") class EasyOrganizer(LearnwareOrganizer): - def reset(self, market_id): + def reset(self, market_id, rebuild=False): self.market_id = market_id - self.reload_market() + self.reload_market(rebuild=rebuild) def reload_market(self, rebuild=False) -> bool: """Reload the learnware organizer when server restared. @@ -64,5 +64,221 @@ class EasyOrganizer(LearnwareOrganizer): os.makedirs(self.learnware_zip_pool_path, exist_ok=True) os.makedirs(self.learnware_folder_pool_path, exist_ok=True) self.learnware_list, self.learnware_zip_list, self.learnware_folder_list, self.count = self.dbops.load_market() + + + def add_learnware(self, zip_path: str, semantic_spec: dict, learnware_id: str = None, check: bool = False) -> Tuple[str, bool]: + """Add a learnware into the market. + + .. note:: + + Given a prediction of a certain time, all signals before this time will be prepared well. + + + Parameters + ---------- + zip_path : str + Filepath for learnware model, a zipped file. + semantic_spec : dict + semantic_spec for new learnware, in dictionary format. + + Returns + ------- + Tuple[str, int] + - str indicating model_id + - int indicating what the flag of learnware is added. + + """ + semantic_spec = copy.deepcopy(semantic_spec) + + if not os.path.exists(zip_path): + logger.warning("Zip Path NOT Found! Fail to add learnware.") + return None, self.INVALID_LEARNWARE + + try: + if len(semantic_spec["Data"]["Values"]) == 0: + logger.warning("Illegal semantic specification, please choose Data.") + return None, self.INVALID_LEARNWARE + if len(semantic_spec["Task"]["Values"]) == 0: + logger.warning("Illegal semantic specification, please choose Task.") + return None, self.INVALID_LEARNWARE + if len(semantic_spec["Library"]["Values"]) == 0: + logger.warning("Illegal semantic specification, please choose Device.") + return None, self.INVALID_LEARNWARE + if len(semantic_spec["Name"]["Values"]) == 0: + logger.warning("Illegal semantic specification, please provide Name.") + return None, self.INVALID_LEARNWARE + if len(semantic_spec["Description"]["Values"]) == 0 and len(semantic_spec["Scenario"]["Values"]) == 0: + logger.warning("Illegal semantic specification, please provide Scenario or Description.") + return None, self.INVALID_LEARNWARE + if ( + semantic_spec["Data"]["Type"] != "Class" + or semantic_spec["Task"]["Type"] != "Class" + or semantic_spec["Library"]["Type"] != "Class" + or semantic_spec["Scenario"]["Type"] != "Tag" + or semantic_spec["Name"]["Type"] != "String" + or semantic_spec["Description"]["Type"] != "String" + ): + logger.warning("Illegal semantic specification, please provide the right type.") + return None, self.INVALID_LEARNWARE + except: + print(semantic_spec) + logger.warning("Illegal semantic specification, some keys are missing.") + return None, self.INVALID_LEARNWARE + + logger.info("Get new learnware from %s" % (zip_path)) + if learnware_id is not None: + id = learnware_id + else: + id = "%08d" % (self.count) + target_zip_dir = os.path.join(self.learnware_zip_pool_path, "%s.zip" % (id)) + target_folder_dir = os.path.join(self.learnware_folder_pool_path, id) + copyfile(zip_path, target_zip_dir) + + with zipfile.ZipFile(target_zip_dir, "r") as z_file: + z_file.extractall(target_folder_dir) + logger.info("Learnware move to %s, and unzip to %s" % (target_zip_dir, target_folder_dir)) + + try: + new_learnware = get_learnware_from_dirpath( + id=id, semantic_spec=semantic_spec, learnware_dirpath=target_folder_dir + ) + except: + try: + os.remove(target_zip_dir) + rmtree(target_folder_dir) + except: + pass + return None, self.INVALID_LEARNWARE + + if new_learnware is None: + return None, self.INVALID_LEARNWARE + + if check and self.checker + + self.dbops.add_learnware( + id=id, + semantic_spec=semantic_spec, + zip_path=target_zip_dir, + folder_path=target_folder_dir, + use_flag=LearnwareChecker.USABLE_LEARWARE, + ) + + self.learnware_list[id] = new_learnware + self.learnware_zip_list[id] = target_zip_dir + self.learnware_folder_list[id] = target_folder_dir + self.count += 1 + return id, LearnwareChecker.USABLE_LEARWARE + + def add_learnware(self, zip_path: str, semantic_spec: dict) -> Tuple[str, bool]: + """Add a learnware into the market. + + .. note:: + + Given a prediction of a certain time, all signals before this time will be prepared well. + + + Parameters + ---------- + zip_path : str + Filepath for learnware model, a zipped file. + semantic_spec : dict + semantic_spec for new learnware, in dictionary format. + + Returns + ------- + Tuple[str, int] + - str indicating model_id + - int indicating what the flag of learnware is added. + + """ + semantic_spec = copy.deepcopy(semantic_spec) + + if not os.path.exists(zip_path): + logger.warning("Zip Path NOT Found! Fail to add learnware.") + return None, self.INVALID_LEARNWARE + + try: + if len(semantic_spec["Data"]["Values"]) == 0: + logger.warning("Illegal semantic specification, please choose Data.") + return None, self.INVALID_LEARNWARE + if len(semantic_spec["Task"]["Values"]) == 0: + logger.warning("Illegal semantic specification, please choose Task.") + return None, self.INVALID_LEARNWARE + if len(semantic_spec["Library"]["Values"]) == 0: + logger.warning("Illegal semantic specification, please choose Device.") + return None, self.INVALID_LEARNWARE + if len(semantic_spec["Name"]["Values"]) == 0: + logger.warning("Illegal semantic specification, please provide Name.") + return None, self.INVALID_LEARNWARE + if len(semantic_spec["Description"]["Values"]) == 0 and len(semantic_spec["Scenario"]["Values"]) == 0: + logger.warning("Illegal semantic specification, please provide Scenario or Description.") + return None, self.INVALID_LEARNWARE + if ( + semantic_spec["Data"]["Type"] != "Class" + or semantic_spec["Task"]["Type"] != "Class" + or semantic_spec["Library"]["Type"] != "Class" + or semantic_spec["Scenario"]["Type"] != "Tag" + or semantic_spec["Name"]["Type"] != "String" + or semantic_spec["Description"]["Type"] != "String" + ): + logger.warning("Illegal semantic specification, please provide the right type.") + return None, self.INVALID_LEARNWARE + except: + logger.info(f"Semantic specification: {semantic_spec}") + logger.warning("Illegal semantic specification, some keys are missing.") + return None, self.INVALID_LEARNWARE + + logger.info("Get new learnware from %s" % (zip_path)) + id = "%08d" % (self.count) + target_zip_dir = os.path.join(self.learnware_zip_pool_path, "%s.zip" % (id)) + target_folder_dir = os.path.join(self.learnware_folder_pool_path, id) + copyfile(zip_path, target_zip_dir) + + with zipfile.ZipFile(target_zip_dir, "r") as z_file: + z_file.extractall(target_folder_dir) + logger.info("Learnware move to %s, and unzip to %s" % (target_zip_dir, target_folder_dir)) + + try: + new_learnware = get_learnware_from_dirpath( + id=id, semantic_spec=semantic_spec, learnware_dirpath=target_folder_dir + ) + except: + try: + os.remove(target_zip_dir) + rmtree(target_folder_dir) + except: + pass + return None, self.INVALID_LEARNWARE + + if new_learnware is None: + return None, self.INVALID_LEARNWARE + + check_flag = self.check_learnware(new_learnware) + + self.dbops.add_learnware( + id=id, + semantic_spec=semantic_spec, + zip_path=target_zip_dir, + folder_path=target_folder_dir, + use_flag=check_flag, + ) + + self.learnware_list[id] = new_learnware + self.learnware_zip_list[id] = target_zip_dir + self.learnware_folder_list[id] = target_folder_dir + self.count += 1 + return id, check_flag + + + def get_learnware_ids(self, top:int = None): + if top is None: + return list(self.learnware_list.keys()) + else: + return list(self.learnware_list.keys())[:top] - \ No newline at end of file + + def get_learnwares(self, top:int = None): + if top is None: + return list(self.learnware_list.values()) + else: + return list(self.learnware_list.values())[:top] \ No newline at end of file diff --git a/learnware/market/evolve.py b/learnware/market/evolve.py index 8912700..e9e5cc3 100644 --- a/learnware/market/evolve.py +++ b/learnware/market/evolve.py @@ -1,16 +1,16 @@ from typing import Tuple, Any, List, Union, Dict -from .base import BaseMarket +from .base import LearnwareMarket from ..learnware import Learnware from ..specification import BaseStatSpecification -class EvolvedMarket(BaseMarket): +class EvolvedMarket(LearnwareMarket): """Organize learnwares and enable them to continuously evolve Parameters ---------- - BaseMarket : _type_ + LearnwareMarket : _type_ Basic market version """