[MNT] rename basemarket to learnware market

docs/references/api.rst

@@ -11,7 +11,7 @@ Here you can find all ``learnware`` interfaces.
 Market
 ====================
 
-.. autoclass:: learnware.market.BaseMarket
+.. autoclass:: learnware.market.LearnwareMarket
     :members:
 
 .. autoclass:: learnware.market.EasyMarket

learnware/market/__init__.py

@@ -1,5 +1,5 @@
 from .anchor import AnchoredUserInfo, AnchoredMarket
-from .base import BaseUserInfo, BaseMarket
+from .base import BaseUserInfo, LearnwareMarket
 from .evolve_anchor import EvolvedAnchoredMarket
 from .evolve import EvolvedMarket
 from .easy import EasyMarket

learnware/market/anchor.py

@@ -2,7 +2,7 @@ import os
 from typing import Tuple, Any, List, Union, Dict
 
 from ..learnware import Learnware
-from .base import BaseMarket, BaseUserInfo
+from .base import LearnwareMarket, BaseUserInfo
 
 
 class AnchoredUserInfo(BaseUserInfo):
@@ -42,12 +42,12 @@ class AnchoredUserInfo(BaseUserInfo):
         self.stat_info[name] = item
 
 
-class AnchoredMarket(BaseMarket):
-    """Add the anchor design to the BaseMarket
+class AnchoredMarket(LearnwareMarket):
+    """Add the anchor design to the LearnwareMarket
 
     Parameters
     ----------
-    BaseMarket : _type_
+    LearnwareMarket : _type_
         Basic market version
     """
 

learnware/market/base.py

@@ -43,7 +43,7 @@ class BaseUserInfo:
         return self.stat_info.get(name, None)
 
 
-class BaseMarket:
+class LearnwareMarket:
     """Base interface for market, it provide the interface of search/add/detele/update learnwares"""
 
     def __init__(
@@ -55,9 +55,9 @@ class BaseMarket:
     ):
         self.market_id = market_id
         self.learnware_organizer = LearnwareOrganizer() if organizer is None else organizer
-        self.learnware_organizer.reset(market_id=market_id)
         self.learnware_checker = LearnwareChecker() if checker is None else checker
         self.learnware_checker.reset(organizer=self.learnware_organizer)
+        self.learnware_organizer.reset(market_id=market_id, checker=self.learnware_checker)
         self.learnware_searcher = LearnwareSearcher() if searcher is None else searcher
         self.learnware_searcher.reset(organizer=self.learnware_organizer)
 
@@ -128,26 +128,7 @@ class BaseMarket:
             - second is a list of matched learnwares
         """
 
-        return self.learnware_searcher(user_info, *args, **kwargs)
-
-    def get_learnware_by_ids(self, id: Union[str, List[str]]) -> Union[Learnware, List[Learnware]]:
-        """
-            Get Learnware from market by id
-
-        Parameters
-        ----------
-        id : Union[str, List[str]]
-            Given one id or a list of ids as target.
-
-        Returns
-        -------
-        Union[Learnware, List[Learnware]]
-            Return a Learnware object or a list of Learnware objects based on the type of input param.
-
-            - The returned items are search results.
-            - 'None' indicating the target id not found.
-        """
-        return self.learnware_organizer.get_learnware_by_ids(id)
+        return self.learnware_searcher(user_info, **kwargs)
 
     def delete_learnware(self, id: str, *args, **kwargs) -> bool:
         """Delete a learnware from market
@@ -191,19 +172,35 @@ class BaseMarket:
 
         """
         raise NotImplementedError("get semantic spec list is not implemented")
+    
+    def get_learnware_by_ids(self, id: Union[str, List[str]]) -> Union[Learnware, List[Learnware]]:
+        """
+            Get Learnware from market by id
 
+        Parameters
+        ----------
+        id : Union[str, List[str]]
+            Given one id or a list of ids as target.
 
-    def get_learnware_ids(self) -> List[str]:
-        raise NotImplementedError("get_learnware_ids is not implemented")
-    
-    
+        Returns
+        -------
+        Union[Learnware, List[Learnware]]
+            Return a Learnware object or a list of Learnware objects based on the type of input param.
+
+            - The returned items are search results.
+            - 'None' indicating the target id not found.
+        """
+        return self.learnware_organizer.get_learnware_by_ids(id)
+
+    def 
         
 class LearnwareOrganizer:
-    def __init__(self, market_id):
-        self.market_id = market_id
+    def __init__(self, market_id, organizer: 'LearnwareOrganizer' = None):
+        self.reset(market_id=market_id, organizer=organizer)
         
-    def reset(self, market_id):
+    def reset(self, market_id, organizer: 'LearnwareOrganizer' ):
         self.market_id = market_id
+        self.organizer = organizer
     
     def reload_market(self) -> bool:
         """Reload the learnware organizer when server restared.
@@ -327,6 +324,12 @@ class LearnwareOrganizer:
         """
         raise NotImplementedError("get_learnware_path_by_ids is not implemented")
     
+    def get_learnware_ids(self, top:int = None):
+        if top is None:
+            return list(self.learnware_list.keys())
+        else:
+            return list(self.learnware_list.keys())[:top]
+
 class LearnwareSearcher:
     def __init__(self, organizer: LearnwareOrganizer = None):
         self.learnware_oganizer = organizer

learnware/market/easy.py

@@ -11,7 +11,7 @@ from cvxopt import solvers, matrix
 from shutil import copyfile, rmtree
 from typing import Tuple, Any, List, Union, Dict
 
-from .base import BaseMarket, BaseUserInfo
+from .base import LearnwareMarket, BaseUserInfo
 from .database_ops import DatabaseOperations
 
 from .. import utils
@@ -24,8 +24,8 @@ from ..specification import RKMEStatSpecification, Specification
 logger = get_module_logger("market", "INFO")
 
 
-class EasyMarket(BaseMarket):
-    """EasyMarket provide an easy and simple implementation for BaseMarket
+class EasyMarket(LearnwareMarket):
+    """EasyMarket provide an easy and simple implementation for LearnwareMarket
     - EasyMarket stores learnwares with file system and database
     - EasyMarket search the learnwares with the match of semantical tag and the statistical RKME
     - EasyMarket does not support the search between heterogeneous features learnwars

learnware/market/easy/organizer.py

@@ -11,7 +11,7 @@ from cvxopt import solvers, matrix
 from shutil import copyfile, rmtree
 from typing import Tuple, Any, List, Union, Dict
 
-from ..base import BaseMarket, BaseUserInfo
+from ..base import LearnwareMarket, BaseUserInfo
 from ..database_ops import DatabaseOperations
 
 from ... import utils
@@ -20,7 +20,7 @@ from ...logger import get_module_logger
 from ...learnware import Learnware, get_learnware_from_dirpath
 from ...specification import RKMEStatSpecification, Specification
 
-from ..base import LearnwareOrganizer
+from ..base import LearnwareOrganizer, LearnwareChecker
 from ...logger import get_module_logger
 
 logger = get_module_logger("easy_organizer")
@@ -28,9 +28,9 @@ logger = get_module_logger("easy_organizer")
 
 class EasyOrganizer(LearnwareOrganizer):
     
-    def reset(self, market_id):
+    def reset(self, market_id, rebuild=False):
         self.market_id = market_id
-        self.reload_market()
+        self.reload_market(rebuild=rebuild)
     
     def reload_market(self, rebuild=False) -> bool:
         """Reload the learnware organizer when server restared.
@@ -64,5 +64,221 @@ class EasyOrganizer(LearnwareOrganizer):
         os.makedirs(self.learnware_zip_pool_path, exist_ok=True)
         os.makedirs(self.learnware_folder_pool_path, exist_ok=True)
         self.learnware_list, self.learnware_zip_list, self.learnware_folder_list, self.count = self.dbops.load_market()
+    
+    
+    def add_learnware(self, zip_path: str, semantic_spec: dict, learnware_id: str = None, check: bool = False) -> Tuple[str, bool]:
+        """Add a learnware into the market.
+
+        .. note::
+
+            Given a prediction of a certain time, all signals before this time will be prepared well.
+
+
+        Parameters
+        ----------
+        zip_path : str
+            Filepath for learnware model, a zipped file.
+        semantic_spec : dict
+            semantic_spec for new learnware, in dictionary format.
+
+        Returns
+        -------
+        Tuple[str, int]
+            - str indicating model_id
+            - int indicating what the flag of learnware is added.
+
+        """
+        semantic_spec = copy.deepcopy(semantic_spec)
+
+        if not os.path.exists(zip_path):
+            logger.warning("Zip Path NOT Found! Fail to add learnware.")
+            return None, self.INVALID_LEARNWARE
+
+        try:
+            if len(semantic_spec["Data"]["Values"]) == 0:
+                logger.warning("Illegal semantic specification, please choose Data.")
+                return None, self.INVALID_LEARNWARE
+            if len(semantic_spec["Task"]["Values"]) == 0:
+                logger.warning("Illegal semantic specification, please choose Task.")
+                return None, self.INVALID_LEARNWARE
+            if len(semantic_spec["Library"]["Values"]) == 0:
+                logger.warning("Illegal semantic specification, please choose Device.")
+                return None, self.INVALID_LEARNWARE
+            if len(semantic_spec["Name"]["Values"]) == 0:
+                logger.warning("Illegal semantic specification, please provide Name.")
+                return None, self.INVALID_LEARNWARE
+            if len(semantic_spec["Description"]["Values"]) == 0 and len(semantic_spec["Scenario"]["Values"]) == 0:
+                logger.warning("Illegal semantic specification, please provide Scenario or Description.")
+                return None, self.INVALID_LEARNWARE
+            if (
+                semantic_spec["Data"]["Type"] != "Class"
+                or semantic_spec["Task"]["Type"] != "Class"
+                or semantic_spec["Library"]["Type"] != "Class"
+                or semantic_spec["Scenario"]["Type"] != "Tag"
+                or semantic_spec["Name"]["Type"] != "String"
+                or semantic_spec["Description"]["Type"] != "String"
+            ):
+                logger.warning("Illegal semantic specification, please provide the right type.")
+                return None, self.INVALID_LEARNWARE
+        except:
+            print(semantic_spec)
+            logger.warning("Illegal semantic specification, some keys are missing.")
+            return None, self.INVALID_LEARNWARE
+
+        logger.info("Get new learnware from %s" % (zip_path))
+        if learnware_id is not None:
+            id = learnware_id
+        else:
+            id = "%08d" % (self.count)
+        target_zip_dir = os.path.join(self.learnware_zip_pool_path, "%s.zip" % (id))
+        target_folder_dir = os.path.join(self.learnware_folder_pool_path, id)
+        copyfile(zip_path, target_zip_dir)
+
+        with zipfile.ZipFile(target_zip_dir, "r") as z_file:
+            z_file.extractall(target_folder_dir)
+        logger.info("Learnware move to %s, and unzip to %s" % (target_zip_dir, target_folder_dir))
+
+        try:
+            new_learnware = get_learnware_from_dirpath(
+                id=id, semantic_spec=semantic_spec, learnware_dirpath=target_folder_dir
+            )
+        except:
+            try:
+                os.remove(target_zip_dir)
+                rmtree(target_folder_dir)
+            except:
+                pass
+            return None, self.INVALID_LEARNWARE
+
+        if new_learnware is None:
+            return None, self.INVALID_LEARNWARE
+
+        if check and self.checker
+        
+        self.dbops.add_learnware(
+            id=id,
+            semantic_spec=semantic_spec,
+            zip_path=target_zip_dir,
+            folder_path=target_folder_dir,
+            use_flag=LearnwareChecker.USABLE_LEARWARE,
+        )
+
+        self.learnware_list[id] = new_learnware
+        self.learnware_zip_list[id] = target_zip_dir
+        self.learnware_folder_list[id] = target_folder_dir
+        self.count += 1
+        return id, LearnwareChecker.USABLE_LEARWARE
+    
+    def add_learnware(self, zip_path: str, semantic_spec: dict) -> Tuple[str, bool]:
+        """Add a learnware into the market.
+
+        .. note::
+
+            Given a prediction of a certain time, all signals before this time will be prepared well.
+
+
+        Parameters
+        ----------
+        zip_path : str
+            Filepath for learnware model, a zipped file.
+        semantic_spec : dict
+            semantic_spec for new learnware, in dictionary format.
+
+        Returns
+        -------
+        Tuple[str, int]
+            - str indicating model_id
+            - int indicating what the flag of learnware is added.
+
+        """
+        semantic_spec = copy.deepcopy(semantic_spec)
+
+        if not os.path.exists(zip_path):
+            logger.warning("Zip Path NOT Found! Fail to add learnware.")
+            return None, self.INVALID_LEARNWARE
+
+        try:
+            if len(semantic_spec["Data"]["Values"]) == 0:
+                logger.warning("Illegal semantic specification, please choose Data.")
+                return None, self.INVALID_LEARNWARE
+            if len(semantic_spec["Task"]["Values"]) == 0:
+                logger.warning("Illegal semantic specification, please choose Task.")
+                return None, self.INVALID_LEARNWARE
+            if len(semantic_spec["Library"]["Values"]) == 0:
+                logger.warning("Illegal semantic specification, please choose Device.")
+                return None, self.INVALID_LEARNWARE
+            if len(semantic_spec["Name"]["Values"]) == 0:
+                logger.warning("Illegal semantic specification, please provide Name.")
+                return None, self.INVALID_LEARNWARE
+            if len(semantic_spec["Description"]["Values"]) == 0 and len(semantic_spec["Scenario"]["Values"]) == 0:
+                logger.warning("Illegal semantic specification, please provide Scenario or Description.")
+                return None, self.INVALID_LEARNWARE
+            if (
+                semantic_spec["Data"]["Type"] != "Class"
+                or semantic_spec["Task"]["Type"] != "Class"
+                or semantic_spec["Library"]["Type"] != "Class"
+                or semantic_spec["Scenario"]["Type"] != "Tag"
+                or semantic_spec["Name"]["Type"] != "String"
+                or semantic_spec["Description"]["Type"] != "String"
+            ):
+                logger.warning("Illegal semantic specification, please provide the right type.")
+                return None, self.INVALID_LEARNWARE
+        except:
+            logger.info(f"Semantic specification: {semantic_spec}")
+            logger.warning("Illegal semantic specification, some keys are missing.")
+            return None, self.INVALID_LEARNWARE
+
+        logger.info("Get new learnware from %s" % (zip_path))
+        id = "%08d" % (self.count)
+        target_zip_dir = os.path.join(self.learnware_zip_pool_path, "%s.zip" % (id))
+        target_folder_dir = os.path.join(self.learnware_folder_pool_path, id)
+        copyfile(zip_path, target_zip_dir)
+
+        with zipfile.ZipFile(target_zip_dir, "r") as z_file:
+            z_file.extractall(target_folder_dir)
+        logger.info("Learnware move to %s, and unzip to %s" % (target_zip_dir, target_folder_dir))
+
+        try:
+            new_learnware = get_learnware_from_dirpath(
+                id=id, semantic_spec=semantic_spec, learnware_dirpath=target_folder_dir
+            )
+        except:
+            try:
+                os.remove(target_zip_dir)
+                rmtree(target_folder_dir)
+            except:
+                pass
+            return None, self.INVALID_LEARNWARE
+
+        if new_learnware is None:
+            return None, self.INVALID_LEARNWARE
+
+        check_flag = self.check_learnware(new_learnware)
+
+        self.dbops.add_learnware(
+            id=id,
+            semantic_spec=semantic_spec,
+            zip_path=target_zip_dir,
+            folder_path=target_folder_dir,
+            use_flag=check_flag,
+        )
+
+        self.learnware_list[id] = new_learnware
+        self.learnware_zip_list[id] = target_zip_dir
+        self.learnware_folder_list[id] = target_folder_dir
+        self.count += 1
+        return id, check_flag
+    
+
+    def get_learnware_ids(self, top:int = None):
+        if top is None:
+            return list(self.learnware_list.keys())
+        else:
+            return list(self.learnware_list.keys())[:top]
         
-        
+    
+    def get_learnwares(self, top:int = None):
+        if top is None:
+            return list(self.learnware_list.values())
+        else:
+            return list(self.learnware_list.values())[:top]

learnware/market/evolve.py

@@ -1,16 +1,16 @@
 from typing import Tuple, Any, List, Union, Dict
 
-from .base import BaseMarket
+from .base import LearnwareMarket
 from ..learnware import Learnware
 from ..specification import BaseStatSpecification
 
 
-class EvolvedMarket(BaseMarket):
+class EvolvedMarket(LearnwareMarket):
     """Organize learnwares and enable them to continuously evolve
 
     Parameters
     ----------
-    BaseMarket : _type_
+    LearnwareMarket : _type_
         Basic market version
     """