diff --git a/.gitignore b/.gitignore index b44d262..6e5f597 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ dist/ .vscode *.egg-info/ *.db +*.json # special software .pytest_cache/ diff --git a/examples/example_market_db/example_db.py b/examples/example_market_db/example_db.py index 886518b..7f0f753 100644 --- a/examples/example_market_db/example_db.py +++ b/examples/example_market_db/example_db.py @@ -1,3 +1,49 @@ -import learnware.market.database_ops as db_ops +from learnware.market import EasyMarket +from learnware.market import database_ops +from learnware.learnware import Learnware +import learnware.specification as specification +from learnware.utils import get_module_by_module_path -db_ops.load_market_from_db() +from sklearn import svm +import joblib +import numpy as np +import os + +# database_ops.load_market_from_db() + +def prepare_learnware(learnware_num = 10): + + for i in range(learnware_num): + print("Preparing Learnware: %d"%(i)) + data_X = np.random.randn(5000, 20) + data_y = np.random.randn(5000) + data_y = np.where(data_y > 0, 1, 0) + + clf = svm.SVC() + clf.fit(data_X, data_y) + joblib.dump(clf, "./svm/svm_%d.pkl"%(i)) + + spec = specification.utils.generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0) + spec.save("./svm/spec_%d.json"%(i)) + +def test_market(): + easy_market = EasyMarket() + print('Total Item:', len(easy_market)) + root_path = '/home/chenzx/code/learnware_project/learnware-market/examples/example_market_db/svm' + os.makedirs(root_path, exist_ok=True) + test_learnware_num = 10 + prepare_learnware(test_learnware_num) + for i in range(test_learnware_num): + model_path = os.path.join(root_path, "svm_%d.pkl"%(i)) + stat_spec_path = os.path.join(root_path, "spec_%d.json"%(i)) + easy_market.add_learnware('learnware_%d'%(i), model_path, stat_spec_path, {"desc":"test_learnware_number_%d"%(i)}) + print('Total Item:', len(easy_market)) + curr_inds = easy_market._get_ids() + print("Available ids:", curr_inds) + easy_market.delete_learnware(curr_inds[4]) + easy_market.delete_learnware(curr_inds[8]) + curr_inds = easy_market._get_ids() + print("Available ids:", curr_inds) + +if __name__ == '__main__': + test_market() \ No newline at end of file diff --git a/learnware/learnware/base.py b/learnware/learnware/base.py index 3be20b9..abf7410 100644 --- a/learnware/learnware/base.py +++ b/learnware/learnware/base.py @@ -14,7 +14,7 @@ class Learnware: self.model = self._import_model(model) self.specification = specification - def _import_model(self, model: Union[BaseModel, dict]) -> BaseModel: + def _import_model(self, model: Union[BaseModel, dict, str]) -> BaseModel: """_summary_ Parameters @@ -41,6 +41,8 @@ class Learnware: elif isinstance(model, dict): model_module = get_module_by_module_path(model["module_path"]) return getattr(model_module, model["class_name"])() + elif isinstance(model, str): + return model # For test purpose else: raise TypeError("model must be BaseModel or dict") diff --git a/learnware/market/base.py b/learnware/market/base.py index 06de774..a118f43 100644 --- a/learnware/market/base.py +++ b/learnware/market/base.py @@ -40,7 +40,7 @@ class BaseUserInfo: return self.stat_info.get(name, None) -class BaseMarket: +class BaseMarket(): """Market for Learnware .. code-block:: python diff --git a/learnware/market/database_ops.py b/learnware/market/database_ops.py index a59a36e..009d9f3 100644 --- a/learnware/market/database_ops.py +++ b/learnware/market/database_ops.py @@ -7,7 +7,7 @@ import json ROOT_PATH = os.path.dirname(os.path.abspath(__file__)) DB_PATH = os.path.join(ROOT_PATH, "market.db") -LOGGER = get_module_logger("market") +LOGGER = get_module_logger("db") def init_empty_db(func): @@ -29,16 +29,15 @@ def init_empty_db(func): ) LOGGER.info("Database Built!") kwargs["cur"] = cur - kwargs["conn"] = conn - func(*args, **kwargs) + item = func(*args, **kwargs) conn.commit() conn.close() + return item return wrapper - @init_empty_db -def add_learnware_to_db(id: str, name: str, model_path: str, stat_spec_path: str, semantic_spec: dict, cur, conn): +def add_learnware_to_db(id: str, name: str, model_path: str, stat_spec_path: str, semantic_spec: dict, cur): semantic_spec_str = json.dumps(semantic_spec) stat_spec_path_dict = {"RKME": stat_spec_path} stat_spec_str = json.dumps(stat_spec_path_dict) @@ -48,16 +47,12 @@ def add_learnware_to_db(id: str, name: str, model_path: str, stat_spec_path: str % (id, name, semantic_spec_str, model_path, stat_spec_str) ) - @init_empty_db -def delete_learnware_from_db(id: str, cur, conn): - cur.execute("DELETE from LEARNWARE where ID=%;") - conn.commit() - LOGGER.info("%d item has been deleted from table 'LEARNWARE'" % (conn.total_changes)) - +def delete_learnware_from_db(id: str, cur): + cur.execute("DELETE from LEARNWARE where ID='%s';"%(id)) @init_empty_db -def load_market_from_db(cur, conn): +def load_market_from_db(cur): LOGGER.info("Reload from Database") cursor = cur.execute("SELECT id, name, semantic_spec, model_path, stat_spec_path from LEARNWARE") @@ -70,9 +65,11 @@ def load_market_from_db(cur, conn): stat_spec_dict = {} for stat_spec_name in stat_spec_path_dict: new_stat_spec = RKMEStatSpecification() - new_stat_spec.load(stat_spec_dict[stat_spec_name]) + new_stat_spec.load(stat_spec_path_dict[stat_spec_name]) stat_spec_dict[stat_spec_name] = new_stat_spec - model_dict = {"model_path": model_path, "class_name": "BaseModel"} + # Commented for test purpose. Uncomment when Learnware class is implemented. + # model_dict = {"module_path": model_path, "class_name": "BaseModel"} + model_dict = model_path specification = Specification(semantic_spec=semantic_spec_dict, stat_spec=stat_spec_dict) new_learnware = Learnware(id=id, name=name, model=model_dict, specification=specification) learnware_list[id] = new_learnware diff --git a/learnware/market/easy.py b/learnware/market/easy.py index 0f51a56..796331e 100644 --- a/learnware/market/easy.py +++ b/learnware/market/easy.py @@ -8,7 +8,9 @@ from .base import BaseMarket, BaseUserInfo from ..learnware import Learnware from ..specification import RKMEStatSpecification, Specification from .database_ops import load_market_from_db, add_learnware_to_db, delete_learnware_from_db +from ..logger import get_module_logger +LOGGER = get_module_logger('market', 'INFO') class EasyMarket(BaseMarket): def __init__(self): @@ -16,6 +18,8 @@ class EasyMarket(BaseMarket): self.learnware_list = {} # id: Learnware self.count = 0 self.semantic_spec_list = self._init_semantic_spec_list() + self.reload_market() + LOGGER.info('Market Initialized!') def _init_semantic_spec_list(self): # TODO: Load from json @@ -111,12 +115,14 @@ class EasyMarket(BaseMarket): rkme_stat_spec.load(stat_spec_path) stat_spec = {"RKME": rkme_stat_spec} specification = Specification(semantic_spec=semantic_spec, stat_spec=stat_spec) - # specification.update_stat_spec("RKME", rkme_stat_spec) - model_dict = {"model_path": model_path, "class_name": "BaseModel"} + # Commented for test purpose. Uncomment when Learnware class is implemented. + # model_dict = {"module_path": model_path, "class_name": "BaseModel"} + model_dict = model_path new_learnware = Learnware(id=id, name=learnware_name, model=model_dict, specification=specification) self.learnware_list[id] = new_learnware self.count += 1 - + add_learnware_to_db(id, name=learnware_name, model_path=model_path, stat_spec_path=stat_spec_path, semantic_spec=semantic_spec) + return id, True def _calculate_rkme_spec_mixture_weight( @@ -325,10 +331,20 @@ class EasyMarket(BaseMarket): def delete_learnware(self, id: str) -> bool: if not id in self.learnware_list: - raise Exception("Learnware id:{} NOT Found!".format(id)) + raise Exception("Learnware id:'{}' NOT Found!".format(id)) self.learnware_list.pop(id) + delete_learnware_from_db(id) return True def get_semantic_spec_list(self) -> dict: return self.semantic_spec_list + + def __len__(self): + return len(self.learnware_list.keys()) + + def _get_ids(self, top=None): + if top is None: + return list(self.learnware_list.keys()) + else: + return list(self.learnware_list.keys())[:top] \ No newline at end of file