[ENH] Modify data storage path, which is relative to user path

examples/example_market_db/example_db.py

@@ -107,6 +107,7 @@ def test_market():
         semantic_spec["Name"]["Values"] = "learnware_%d" % (idx)
         semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx)
         easy_market.add_learnware(zip_path, semantic_spec)
+    return
     print("Total Item:", len(easy_market))
     curr_inds = easy_market._get_ids()
     print("Available ids:", curr_inds)
@@ -171,7 +172,7 @@ def test_stat_search():
 
 
 if __name__ == "__main__":
-    learnware_num = 10
+    learnware_num = 5
     prepare_learnware(learnware_num)
     test_market()
     test_stat_search()

examples/workflow_by_code/main.py

@@ -0,0 +1,58 @@
+import os
+import fire
+
+
+class LearnwareMarketWorkflow:
+    curr_root = os.path.dirname(os.path.abspath(__file__))
+
+    semantic_specs = [
+        {
+            "Data": {"Values": ["Tabular"], "Type": "Class"},
+            "Task": {
+                "Values": ["Classification"],
+                "Type": "Class",
+            },
+            "Device": {"Values": ["GPU"], "Type": "Tag"},
+            "Scenario": {"Values": ["Nature"], "Type": "Tag"},
+            "Description": {"Values": "", "Type": "Description"},
+            "Name": {"Values": "learnware_1", "Type": "Name"},
+        },
+        {
+            "Data": {"Values": ["Tabular"], "Type": "Class"},
+            "Task": {
+                "Values": ["Classification"],
+                "Type": "Class",
+            },
+            "Device": {"Values": ["GPU"], "Type": "Tag"},
+            "Scenario": {"Values": ["Business", "Nature"], "Type": "Tag"},
+            "Description": {"Values": "", "Type": "Description"},
+            "Name": {"Values": "learnware_2", "Type": "Name"},
+        },
+        {
+            "Data": {"Values": ["Tabular"], "Type": "Class"},
+            "Task": {
+                "Values": ["Classification"],
+                "Type": "Class",
+            },
+            "Device": {"Values": ["GPU"], "Type": "Tag"},
+            "Scenario": {"Values": ["Business"], "Type": "Tag"},
+            "Description": {"Values": "", "Type": "Description"},
+            "Name": {"Values": "learnware_3", "Type": "Name"},
+        },
+    ]
+
+    user_senmantic = {
+        "Data": {"Values": ["Tabular"], "Type": "Class"},
+        "Task": {
+            "Values": ["Classification"],
+            "Type": "Class",
+        },
+        "Device": {"Values": ["GPU"], "Type": "Tag"},
+        "Scenario": {"Values": ["Business"], "Type": "Tag"},
+        "Description": {"Values": "", "Type": "Description"},
+        "Name": {"Values": "", "Type": "Name"},
+    }
+
+
+if __name__ == "__main__":
+    fire.Fire(LearnwareMarketWorkflow)

learnware/__init__.py

@@ -1,12 +1,22 @@
 __version__ = "0.0.1.99"
 
+import os
 from .logger import get_module_logger
 
 
 def init(**kwargs):
     from .config import C
 
+    C.reset()
     C.update(**kwargs)
 
     logger = get_module_logger("Initialization")
+
+    ## make dirs
+    os.makedirs(C.root_path, exist_ok=True)
+    os.makedirs(C.database_path, exist_ok=True)
+    os.makedirs(C.learnware_pool_path, exist_ok=True)
+    os.makedirs(C.learnware_zip_pool_path, exist_ok=True)
+    os.makedirs(C.learnware_folder_pool_path, exist_ok=True)
+
     logger.info(f"init learnware market with {kwargs}")

learnware/config.py

@@ -48,13 +48,22 @@ class Config:
         self.__dict__["_config"].update(*args, **kwargs)
 
 
-ROOT_DIRPATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+ROOT_DIRPATH = os.path.join(os.path.expanduser("~"), ".learnware")
 SPEC_DIRPATH = None
+
 LEARNWARE_POOL_PATH = os.path.join(ROOT_DIRPATH, "learnware_pool")
 LEARNWARE_ZIP_POOL_PATH = os.path.join(LEARNWARE_POOL_PATH, "zips")
 LEARNWARE_FOLDER_POOL_PATH = os.path.join(LEARNWARE_POOL_PATH, "learnwares")
+
+DATABASE_PATH = os.path.join(ROOT_DIRPATH, "database")
+
+
+# TODO: Delete them later
+os.makedirs(ROOT_DIRPATH, exist_ok=True)
+os.makedirs(LEARNWARE_POOL_PATH, exist_ok=True)
 os.makedirs(LEARNWARE_ZIP_POOL_PATH, exist_ok=True)
 os.makedirs(LEARNWARE_FOLDER_POOL_PATH, exist_ok=True)
+os.makedirs(DATABASE_PATH, exist_ok=True)
 
 semantic_config = {
     "Data": {
@@ -109,11 +118,16 @@ semantic_config = {
 _DEFAULT_CONFIG = {
     "root_path": ROOT_DIRPATH,
     "logging_level": logging.INFO,
-    "specification_path": SPEC_DIRPATH,
+    "logging_outfile": None,
     "semantic_specs": semantic_config,
     "learnware_pool_path": LEARNWARE_POOL_PATH,
     "learnware_zip_pool_path": LEARNWARE_ZIP_POOL_PATH,
     "learnware_folder_pool_path": LEARNWARE_FOLDER_POOL_PATH,
+    "learnware_folder_config": {
+        "yaml_file": "learnware.yaml",
+        "module_file": "__init__.py",
+    },
+    "database_path": DATABASE_PATH,
 }
 
 C = Config(_DEFAULT_CONFIG)

learnware/learnware/__init__.py

@@ -6,6 +6,7 @@ from .utils import get_stat_spec_from_config, get_model_from_config
 from ..specification import Specification
 from ..utils import read_yaml_to_dict
 from ..logger import get_module_logger
+from ..config import C
 
 logger = get_module_logger("learnware.learnware")
 
@@ -44,7 +45,7 @@ def get_learnware_from_dirpath(id: str, semantic_spec: dict, learnware_dirpath:
 
     if learnware_dirpath is not None:
         try:
-            yaml_config = read_yaml_to_dict(os.path.join(learnware_dirpath, "learnware.yaml"))
+            yaml_config = read_yaml_to_dict(os.path.join(learnware_dirpath, C.learnware_folder_config["yaml_file"]))
         except FileNotFoundError:
             yaml_config = {}
 
@@ -56,7 +57,9 @@ def get_learnware_from_dirpath(id: str, semantic_spec: dict, learnware_dirpath:
         learnware_config["stat_specifications"] = yaml_config["stat_specifications"].copy()
 
     if "module_path" not in learnware_config["model"]:
-        learnware_config["model"]["module_path"] = os.path.join(learnware_dirpath, "__init__.py")
+        learnware_config["model"]["module_path"] = os.path.join(
+            learnware_dirpath, C.learnware_folder_config["module_file"]
+        )
 
     try:
         learnware_spec = Specification()

learnware/logger.py

@@ -12,6 +12,8 @@ def get_module_logger(module_name: str, level: int = None, outfile: str = None)
         Logic module name.
     level : int, optional
         Logging level, by default None
+    outfile : str, optional
+        The output filepath, by default None
 
     Returns
     -------
@@ -20,6 +22,8 @@ def get_module_logger(module_name: str, level: int = None, outfile: str = None)
     """
     if level is None:
         level = C.logging_level
+    if outfile is None:
+        outfile = C.logging_outfile
 
     # Get logger.
     console_handler = logging.StreamHandler()

learnware/market/database_ops.py

@@ -5,22 +5,20 @@ from copy import deepcopy
 
 from ..logger import get_module_logger
 from ..learnware import get_learnware_from_dirpath
+from ..config import C
 
-
-ROOT_PATH = os.path.dirname(os.path.abspath(__file__))
-DB_PATH = os.path.join(ROOT_PATH, "market.db")
-LOGGER = get_module_logger("db")
+logger = get_module_logger("database_ops")
 
 
 def init_empty_db(func):
     def wrapper(*args, **kwargs):
-        conn = sqlite3.connect(DB_PATH)
+        conn = sqlite3.connect(os.path.join(C.database_path, "market.db"))
         cur = conn.cursor()
         listOfTables = cur.execute(
             """SELECT name FROM sqlite_master WHERE type='table' AND name='LEARNWARE'; """
         ).fetchall()
-        if listOfTables == []:
-            LOGGER.info("Initializing Database in %s..." % (DB_PATH))
+        if len(listOfTables) == 0:
+            logger.info("Initializing Database in %s..." % (os.path.join(C.database_path, "market.db")))
             cur.execute(
                 """CREATE TABLE LEARNWARE
             (ID CHAR(10) PRIMARY KEY     NOT NULL,
@@ -28,7 +26,7 @@ def init_empty_db(func):
             ZIP_PATH     TEXT NOT NULL,
             FOLDER_PATH         TEXT NOT NULL);"""
             )
-            LOGGER.info("Database Built!")
+            logger.info("Database Built!")
         kwargs["cur"] = cur
         item = func(*args, **kwargs)
         conn.commit()
@@ -46,7 +44,7 @@ def init_empty_db(func):
 # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 @init_empty_db
 def clear_learnware_table(cur):
-    LOGGER.warning("!!! Drop Learnware Table !!!")
+    logger.warning("!!! Drop Learnware Table !!!")
     cur.execute("DROP TABLE LEARNWARE")
 
 
@@ -67,7 +65,7 @@ def delete_learnware_from_db(id: str, cur):
 
 @init_empty_db
 def load_market_from_db(cur):
-    LOGGER.info("Reload from Database")
+    logger.info("Reload from Database")
     cursor = cur.execute("SELECT id, semantic_spec, zip_path, FOLDER_PATH from LEARNWARE")
 
     learnware_list = {}
@@ -86,5 +84,5 @@ def load_market_from_db(cur):
         folder_list[id] = folder_path
         max_count = max(max_count, int(id))
 
-    LOGGER.info("Market Reloaded from DB.")
+    logger.info("Market Reloaded from DB.")
     return learnware_list, zip_list, folder_list, max_count + 1

learnware/market/easy.py

@@ -82,7 +82,8 @@ class EasyMarket(BaseMarket):
         Returns
         -------
         Tuple[str, bool]
-            str indicating model_id, bool indicating whether the learnware is added successfully.
+            - str indicating model_id
+            - bool indicating whether the learnware is added successfully.
 
         """
         if not os.path.exists(zip_path):
@@ -104,6 +105,7 @@ class EasyMarket(BaseMarket):
             )
         except:
             new_learnware = None
+
         if new_learnware is None:
             try:
                 os.remove(target_zip_dir)

learnware/utils.py

@@ -12,13 +12,6 @@ from .logger import get_module_logger
 logger = get_module_logger("utils")
 
 
-def make_dir_by_path(dirpath):
-    if not os.path.exists(dirpath):
-        os.makedirs(dirpath)
-    else:
-        logger.warning(f"Directorty {dirpath} has been exited, ignore mkdir")
-
-
 def get_module_by_module_path(module_path: Union[str, ModuleType]):
     if module_path is None:
         raise ModuleNotFoundError("None is passed in as parameters as module_path")

setup.py

@@ -44,6 +44,7 @@ REQUIRED = [
     "scikit-learn>=1.2.2",
     "joblib>=1.2.0",
     "pyyaml>=6.0",
+    "fire>=0.5.0",
 ]
 
 here = os.path.abspath(os.path.dirname(__file__))