diff --git a/docs/references/api.rst b/docs/references/api.rst index de63395..1b58405 100644 --- a/docs/references/api.rst +++ b/docs/references/api.rst @@ -17,10 +17,10 @@ Market .. autoclass:: learnware.market.AnchoredMarket :members: -.. autoclass:: learnware.market.BaseMarket +.. autoclass:: learnware.market.EvolvedMarket :members: -.. autoclass:: learnware.market.BaseReuse +.. autoclass:: learnware.market.BaseUserInfo :members: Learnware @@ -29,6 +29,8 @@ Learnware .. autoclass:: learnware.learnware.Learnware :members: +.. autoclass:: learnware.learnware.BaseReuse + :members: Specification ==================== @@ -36,7 +38,7 @@ Specification .. autoclass:: learnware.specification.Specification :members: -.. autoclass:: learnware.specification.StatSpecification +.. autoclass:: learnware.specification.BaseStatSpecification :members: .. autoclass:: learnware.specification.RKMESpecification diff --git a/learnware/market/base.py b/learnware/market/base.py index d66c22d..d2e1432 100644 --- a/learnware/market/base.py +++ b/learnware/market/base.py @@ -48,60 +48,7 @@ class BaseMarket: learnmarket = BaseMarket() """ - def __init__(self): - """Initializing an empty market""" - self.learnware_list = {} # id: Learnware - self.count = 0 - self.semantic_spec_list = self._init_semantic_spec_list() - - def _init_semantic_spec_list(self): - return { - "Data": { - "Values": ["Tabular", "Image", "Video", "Text", "Audio"], - "Type": "Class", # Choose only one class - }, - "Task": { - "Values": [ - "Classification", - "Regression", - "Clustering", - "Feature Extraction", - "Generation", - "Segmentation", - "Object Detection", - ], - "Type": "Class", # Choose only one class - }, - "Device": { - "Values": ["CPU", "GPU"], - "Type": "Tag", # Choose one or more tags - }, - "Scenario": { - "Values": [ - "Business", - "Financial", - "Health", - "Politics", - "Computer", - "Internet", - "Traffic", - "Nature", - "Fashion", - "Industry", - "Agriculture", - "Education", - "Entertainment", - "Architecture", - ], - "Type": "Tag", # Choose one or more tags - }, - "Description": { - "Values": str, - "Type": "Description", - }, - } - - def reload_market(self, market_path: str, semantic_spec_list_path: str, load_mode: str = "database") -> bool: + def reload_market(self, market_path: str, semantic_spec_list_path: str) -> bool: """Reload the market when server restared. Parameters @@ -110,32 +57,14 @@ class BaseMarket: Directory for market data. '_IP_:_port_' for loading from database. semantic_spec_list_path : str Directory for available semantic_spec. Should be a json file. - load_mode : str, optional - Type of reload source. Currently, only 'database' is available. Defaults to 'database', by default "database" Returns ------- bool A flag indicating whether the market is reload successfully. - - Raises - ------ - NotImplementedError - Reload method NOT implemented. Currently, only loading from database is supported. - FileNotFoundError - Loading source/semantic_spec_list NOT found. Check whether the source and semantic_spec_list are available. - """ - if load_mode == "database": - pass - else: - # May Support other loading methods in the future - raise NotImplementedError("reload_market from {} is NOT implemented".format(load_mode)) - - raise FileNotFoundError("Reload source NOT Found!") - - return True + raise NotImplementedError("reload market is Not Implemented") def check_learnware(self, learnware: Learnware) -> bool: """Check the utility of a learnware @@ -186,9 +115,7 @@ class BaseMarket: file for model or statistical specification not found """ - if (not os.path.exists(model_path)) or (not os.path.exists(stat_spec_path)): - raise FileNotFoundError("Model or Stat_spec NOT Found.") - return str(self.count), True + raise NotImplementedError("add learnware is Not Implemented") def search_learnware(self, user_info: BaseUserInfo) -> Tuple[Any, List[Learnware]]: """Search Learnware based on user_info @@ -207,30 +134,7 @@ class BaseMarket: - second is a list of matched learnwares """ - def search_by_semantic_spec(): - def match_semantic_spec(semantic_spec1, semantic_spec2): - if semantic_spec1.keys() != semantic_spec2.keys(): - raise Exception("semantic_spec key error".format(semantic_spec1.keys(), semantic_spec2.keys())) - for key in semantic_spec1.keys(): - if semantic_spec1[key]["Type"] == "Class": - if semantic_spec1[key]["Values"] != semantic_spec2[key]["Values"]: - return False - elif semantic_spec1[key]["Type"] == "Tag": - if not (set(semantic_spec1[key]["Values"]) & set(semantic_spec2[key]["Values"])): - return False - return True - - match_learnwares = [] - for learnware in self.learnware_list: - learnware_semantic_spec = learnware.get_specification().get_semantic_spec() - user_semantic_spec = user_info.get_semantic_spec() - if match_semantic_spec(learnware_semantic_spec, user_semantic_spec): - match_learnwares.append(learnware) - return match_learnwares - - match_learnwares = search_by_semantic_spec() - - pass + raise NotImplementedError("search learnware is Not Implemented") def get_learnware_by_ids(self, id: Union[str, List[str]]) -> Union[Learnware, List[Learnware]]: """ @@ -249,7 +153,7 @@ class BaseMarket: - The returned items are search results. - 'None' indicating the target id not found. """ - return None + raise NotImplementedError("search learnware is Not Implemented") def delete_learnware(self, id: str) -> bool: """Delete a learnware from market @@ -269,11 +173,7 @@ class BaseMarket: Exception Raise an excpetion when given id is NOT found in learnware list """ - if not id in self.learnware_list: - raise Exception("Learnware id:{} NOT Found!".format(id)) - - self.learnware_list.pop(id) - return True + raise NotImplementedError("delete learnware is Not Implemented") def update_learnware(self, id: str) -> bool: """ @@ -285,7 +185,7 @@ class BaseMarket: id : str id of target learnware. """ - return True + raise NotImplementedError("update learnware is Not Implemented") def get_semantic_spec_list(self) -> dict: """Return all semantic specifications available @@ -296,4 +196,4 @@ class BaseMarket: All emantic specifications in dictionary format """ - return self.semantic_spec_list + raise NotImplementedError("get semantic spec list is not implemented") diff --git a/learnware/market/serial.py b/learnware/market/serial.py new file mode 100644 index 0000000..40e2f11 --- /dev/null +++ b/learnware/market/serial.py @@ -0,0 +1,138 @@ +import os +import numpy as np +import pandas as pd +from typing import Tuple, Any, List, Union, Dict + +from .base import BaseMarket, BaseUserInfo +from ..learnware import Learnware + + +class SerialMarket(BaseMarket): + def __init__(self): + """Initializing an empty market""" + self.learnware_list = {} # id: Learnware + self.count = 0 + self.semantic_spec_list = self._init_semantic_spec_list() + + def _init_semantic_spec_list(self): + return { + "Data": { + "Values": ["Tabular", "Image", "Video", "Text", "Audio"], + "Type": "Class", # Choose only one class + }, + "Task": { + "Values": [ + "Classification", + "Regression", + "Clustering", + "Feature Extraction", + "Generation", + "Segmentation", + "Object Detection", + ], + "Type": "Class", # Choose only one class + }, + "Device": { + "Values": ["CPU", "GPU"], + "Type": "Tag", # Choose one or more tags + }, + "Scenario": { + "Values": [ + "Business", + "Financial", + "Health", + "Politics", + "Computer", + "Internet", + "Traffic", + "Nature", + "Fashion", + "Industry", + "Agriculture", + "Education", + "Entertainment", + "Architecture", + ], + "Type": "Tag", # Choose one or more tags + }, + "Description": { + "Values": str, + "Type": "Description", + }, + } + + def reload_market(self, market_path: str, semantic_spec_list_path: str) -> bool: + raise NotImplementedError("reload market is Not Implemented") + + def add_learnware( + self, learnware_name: str, model_path: str, stat_spec_path: str, semantic_spec: dict, desc: str + ) -> Tuple[str, bool]: + """Add a learnware into the market. + + .. note:: + + Given a prediction of a certain time, all signals before this time will be prepared well. + + + Parameters + ---------- + learnware_name : str + Name of new learnware. + model_path : str + Filepath for learnware model, a zipped file. + stat_spec_path : str + Filepath for statistical specification, a '.npy' file. + How to pass parameters requires further discussion. + semantic_spec : dict + semantic_spec for new learnware, in dictionary format. + desc : str + Brief desciption for new learnware. + + Returns + ------- + Tuple[str, bool] + str indicating model_id, bool indicating whether the learnware is added successfully. + + Raises + ------ + FileNotFoundError + file for model or statistical specification not found + + """ + if (not os.path.exists(model_path)) or (not os.path.exists(stat_spec_path)): + raise FileNotFoundError("Model or Stat_spec NOT Found.") + return str(self.count), True + + def search_learnware(self, user_info: BaseUserInfo) -> Tuple[Any, List[Learnware]]: + def search_by_semantic_spec(): + def match_semantic_spec(semantic_spec1, semantic_spec2): + if semantic_spec1.keys() != semantic_spec2.keys(): + raise Exception("semantic_spec key error".format(semantic_spec1.keys(), semantic_spec2.keys())) + for key in semantic_spec1.keys(): + if semantic_spec1[key]["Type"] == "Class": + if semantic_spec1[key]["Values"] != semantic_spec2[key]["Values"]: + return False + elif semantic_spec1[key]["Type"] == "Tag": + if not (set(semantic_spec1[key]["Values"]) & set(semantic_spec2[key]["Values"])): + return False + return True + + match_learnwares = [] + for learnware in self.learnware_list: + learnware_semantic_spec = learnware.get_specification().get_semantic_spec() + user_semantic_spec = user_info.get_semantic_spec() + if match_semantic_spec(learnware_semantic_spec, user_semantic_spec): + match_learnwares.append(learnware) + return match_learnwares + + match_learnwares = search_by_semantic_spec() + + def delete_learnware(self, id: str) -> bool: + if not id in self.learnware_list: + raise Exception("Learnware id:{} NOT Found!".format(id)) + + self.learnware_list.pop(id) + return True + + def get_semantic_spec_list(self) -> dict: + return self.semantic_spec_list diff --git a/learnware/specification/rkme.py b/learnware/specification/rkme.py index eda959a..2179af9 100644 --- a/learnware/specification/rkme.py +++ b/learnware/specification/rkme.py @@ -1,7 +1,8 @@ from __future__ import annotations import os -import mkl + +# import mkl import copy import torch import faiss @@ -14,7 +15,7 @@ from typing import Tuple, Any, List, Union, Dict from .base import BaseStatSpecification -mkl.get_max_threads() +# mkl.get_max_threads() def setup_seed(seed):