[ENH] market with database operations

.gitignore

@@ -24,6 +24,7 @@ dist/
 .vscode
 *.egg-info/
 *.db
+*.json
 
 # special software
 .pytest_cache/

examples/example_market_db/example_db.py

@@ -1,3 +1,49 @@
-import learnware.market.database_ops as db_ops
+from learnware.market import EasyMarket
+from learnware.market import database_ops
+from learnware.learnware import Learnware
+import learnware.specification as specification
+from learnware.utils import get_module_by_module_path
 
-db_ops.load_market_from_db()
+from sklearn import svm
+import joblib
+import numpy as np
+import os
+
+# database_ops.load_market_from_db()
+
+def prepare_learnware(learnware_num = 10):
+    
+    for i in range(learnware_num):
+        print("Preparing Learnware: %d"%(i))
+        data_X = np.random.randn(5000, 20)
+        data_y = np.random.randn(5000)
+        data_y = np.where(data_y > 0, 1, 0)
+
+        clf = svm.SVC()
+        clf.fit(data_X, data_y)
+        joblib.dump(clf, "./svm/svm_%d.pkl"%(i))
+
+        spec = specification.utils.generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0)
+        spec.save("./svm/spec_%d.json"%(i))
+
+def test_market():
+    easy_market = EasyMarket()
+    print('Total Item:', len(easy_market))
+    root_path = '/home/chenzx/code/learnware_project/learnware-market/examples/example_market_db/svm'
+    os.makedirs(root_path, exist_ok=True)
+    test_learnware_num = 10
+    prepare_learnware(test_learnware_num)
+    for i in range(test_learnware_num):
+        model_path = os.path.join(root_path, "svm_%d.pkl"%(i))
+        stat_spec_path = os.path.join(root_path, "spec_%d.json"%(i))
+        easy_market.add_learnware('learnware_%d'%(i), model_path, stat_spec_path, {"desc":"test_learnware_number_%d"%(i)})
+    print('Total Item:', len(easy_market))
+    curr_inds = easy_market._get_ids()
+    print("Available ids:", curr_inds)
+    easy_market.delete_learnware(curr_inds[4])
+    easy_market.delete_learnware(curr_inds[8])
+    curr_inds = easy_market._get_ids()
+    print("Available ids:", curr_inds)
+
+if __name__ == '__main__':
+    test_market()

learnware/learnware/base.py

@@ -14,7 +14,7 @@ class Learnware:
         self.model = self._import_model(model)
         self.specification = specification
 
-    def _import_model(self, model: Union[BaseModel, dict]) -> BaseModel:
+    def _import_model(self, model: Union[BaseModel, dict, str]) -> BaseModel:
         """_summary_
 
         Parameters
@@ -41,6 +41,8 @@ class Learnware:
         elif isinstance(model, dict):
             model_module = get_module_by_module_path(model["module_path"])
             return getattr(model_module, model["class_name"])()
+        elif isinstance(model, str):
+            return model    # For test purpose
         else:
             raise TypeError("model must be BaseModel or dict")
 

learnware/market/base.py

@@ -40,7 +40,7 @@ class BaseUserInfo:
         return self.stat_info.get(name, None)
 
 
-class BaseMarket:
+class BaseMarket():
     """Market for Learnware
 
     .. code-block:: python

learnware/market/database_ops.py

@@ -7,7 +7,7 @@ import json
 
 ROOT_PATH = os.path.dirname(os.path.abspath(__file__))
 DB_PATH = os.path.join(ROOT_PATH, "market.db")
-LOGGER = get_module_logger("market")
+LOGGER = get_module_logger("db")
 
 
 def init_empty_db(func):
@@ -29,16 +29,15 @@ def init_empty_db(func):
             )
             LOGGER.info("Database Built!")
         kwargs["cur"] = cur
-        kwargs["conn"] = conn
-        func(*args, **kwargs)
+        item = func(*args, **kwargs)
         conn.commit()
         conn.close()
+        return item
 
     return wrapper
 
-
 @init_empty_db
-def add_learnware_to_db(id: str, name: str, model_path: str, stat_spec_path: str, semantic_spec: dict, cur, conn):
+def add_learnware_to_db(id: str, name: str, model_path: str, stat_spec_path: str, semantic_spec: dict, cur):
     semantic_spec_str = json.dumps(semantic_spec)
     stat_spec_path_dict = {"RKME": stat_spec_path}
     stat_spec_str = json.dumps(stat_spec_path_dict)
@@ -48,16 +47,12 @@ def add_learnware_to_db(id: str, name: str, model_path: str, stat_spec_path: str
         % (id, name, semantic_spec_str, model_path, stat_spec_str)
     )
 
-
 @init_empty_db
-def delete_learnware_from_db(id: str, cur, conn):
-    cur.execute("DELETE from LEARNWARE where ID=%;")
-    conn.commit()
-    LOGGER.info("%d item has been deleted from table 'LEARNWARE'" % (conn.total_changes))
-
+def delete_learnware_from_db(id: str, cur):
+    cur.execute("DELETE from LEARNWARE where ID='%s';"%(id))
 
 @init_empty_db
-def load_market_from_db(cur, conn):
+def load_market_from_db(cur):
     LOGGER.info("Reload from Database")
     cursor = cur.execute("SELECT id, name, semantic_spec, model_path, stat_spec_path from LEARNWARE")
 
@@ -70,9 +65,11 @@ def load_market_from_db(cur, conn):
         stat_spec_dict = {}
         for stat_spec_name in stat_spec_path_dict:
             new_stat_spec = RKMEStatSpecification()
-            new_stat_spec.load(stat_spec_dict[stat_spec_name])
+            new_stat_spec.load(stat_spec_path_dict[stat_spec_name])
             stat_spec_dict[stat_spec_name] = new_stat_spec
-        model_dict = {"model_path": model_path, "class_name": "BaseModel"}
+        # Commented for test purpose. Uncomment when Learnware class is implemented.
+        # model_dict = {"module_path": model_path, "class_name": "BaseModel"}
+        model_dict = model_path
         specification = Specification(semantic_spec=semantic_spec_dict, stat_spec=stat_spec_dict)
         new_learnware = Learnware(id=id, name=name, model=model_dict, specification=specification)
         learnware_list[id] = new_learnware

learnware/market/easy.py

@@ -8,7 +8,9 @@ from .base import BaseMarket, BaseUserInfo
 from ..learnware import Learnware
 from ..specification import RKMEStatSpecification, Specification
 from .database_ops import load_market_from_db, add_learnware_to_db, delete_learnware_from_db
+from ..logger import get_module_logger
 
+LOGGER = get_module_logger('market', 'INFO')
 
 class EasyMarket(BaseMarket):
     def __init__(self):
@@ -16,6 +18,8 @@ class EasyMarket(BaseMarket):
         self.learnware_list = {}  # id: Learnware
         self.count = 0
         self.semantic_spec_list = self._init_semantic_spec_list()
+        self.reload_market()
+        LOGGER.info('Market Initialized!')
 
     def _init_semantic_spec_list(self):
         # TODO: Load from json
@@ -111,12 +115,14 @@ class EasyMarket(BaseMarket):
         rkme_stat_spec.load(stat_spec_path)
         stat_spec = {"RKME": rkme_stat_spec}
         specification = Specification(semantic_spec=semantic_spec, stat_spec=stat_spec)
-        # specification.update_stat_spec("RKME", rkme_stat_spec)
-        model_dict = {"model_path": model_path, "class_name": "BaseModel"}
+        # Commented for test purpose. Uncomment when Learnware class is implemented.
+        # model_dict = {"module_path": model_path, "class_name": "BaseModel"}
+        model_dict = model_path
         new_learnware = Learnware(id=id, name=learnware_name, model=model_dict, specification=specification)
         self.learnware_list[id] = new_learnware
         self.count += 1
-
+        add_learnware_to_db(id, name=learnware_name, model_path=model_path, stat_spec_path=stat_spec_path, semantic_spec=semantic_spec)
+        
         return id, True
 
     def _calculate_rkme_spec_mixture_weight(
@@ -325,10 +331,20 @@ class EasyMarket(BaseMarket):
 
     def delete_learnware(self, id: str) -> bool:
         if not id in self.learnware_list:
-            raise Exception("Learnware id:{} NOT Found!".format(id))
+            raise Exception("Learnware id:'{}' NOT Found!".format(id))
 
         self.learnware_list.pop(id)
+        delete_learnware_from_db(id)
         return True
 
     def get_semantic_spec_list(self) -> dict:
         return self.semantic_spec_list
+    
+    def __len__(self):
+        return len(self.learnware_list.keys())
+    
+    def _get_ids(self, top=None):
+        if top is None:
+            return list(self.learnware_list.keys())
+        else:
+            return list(self.learnware_list.keys())[:top]