From 94161b9e46e07fb689cd5304cd0caf0452ff8613 Mon Sep 17 00:00:00 2001 From: bxdd Date: Tue, 11 Apr 2023 02:27:38 +0800 Subject: [PATCH] [ENH] Modify data storage path, which is relative to user path --- examples/example_market_db/example_db.py | 3 +- examples/workflow_by_code/main.py | 58 ++++++++++++++++++++++++ learnware/__init__.py | 10 ++++ learnware/config.py | 18 +++++++- learnware/learnware/__init__.py | 7 ++- learnware/logger.py | 4 ++ learnware/market/database_ops.py | 20 ++++---- learnware/market/easy.py | 4 +- learnware/utils.py | 7 --- setup.py | 1 + 10 files changed, 108 insertions(+), 24 deletions(-) create mode 100644 examples/workflow_by_code/main.py diff --git a/examples/example_market_db/example_db.py b/examples/example_market_db/example_db.py index 32aeb07..5e63b40 100644 --- a/examples/example_market_db/example_db.py +++ b/examples/example_market_db/example_db.py @@ -107,6 +107,7 @@ def test_market(): semantic_spec["Name"]["Values"] = "learnware_%d" % (idx) semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx) easy_market.add_learnware(zip_path, semantic_spec) + return print("Total Item:", len(easy_market)) curr_inds = easy_market._get_ids() print("Available ids:", curr_inds) @@ -171,7 +172,7 @@ def test_stat_search(): if __name__ == "__main__": - learnware_num = 10 + learnware_num = 5 prepare_learnware(learnware_num) test_market() test_stat_search() diff --git a/examples/workflow_by_code/main.py b/examples/workflow_by_code/main.py new file mode 100644 index 0000000..9d579d0 --- /dev/null +++ b/examples/workflow_by_code/main.py @@ -0,0 +1,58 @@ +import os +import fire + + +class LearnwareMarketWorkflow: + curr_root = os.path.dirname(os.path.abspath(__file__)) + + semantic_specs = [ + { + "Data": {"Values": ["Tabular"], "Type": "Class"}, + "Task": { + "Values": ["Classification"], + "Type": "Class", + }, + "Device": {"Values": ["GPU"], "Type": "Tag"}, + "Scenario": {"Values": ["Nature"], "Type": "Tag"}, + "Description": {"Values": "", "Type": "Description"}, + "Name": {"Values": "learnware_1", "Type": "Name"}, + }, + { + "Data": {"Values": ["Tabular"], "Type": "Class"}, + "Task": { + "Values": ["Classification"], + "Type": "Class", + }, + "Device": {"Values": ["GPU"], "Type": "Tag"}, + "Scenario": {"Values": ["Business", "Nature"], "Type": "Tag"}, + "Description": {"Values": "", "Type": "Description"}, + "Name": {"Values": "learnware_2", "Type": "Name"}, + }, + { + "Data": {"Values": ["Tabular"], "Type": "Class"}, + "Task": { + "Values": ["Classification"], + "Type": "Class", + }, + "Device": {"Values": ["GPU"], "Type": "Tag"}, + "Scenario": {"Values": ["Business"], "Type": "Tag"}, + "Description": {"Values": "", "Type": "Description"}, + "Name": {"Values": "learnware_3", "Type": "Name"}, + }, + ] + + user_senmantic = { + "Data": {"Values": ["Tabular"], "Type": "Class"}, + "Task": { + "Values": ["Classification"], + "Type": "Class", + }, + "Device": {"Values": ["GPU"], "Type": "Tag"}, + "Scenario": {"Values": ["Business"], "Type": "Tag"}, + "Description": {"Values": "", "Type": "Description"}, + "Name": {"Values": "", "Type": "Name"}, + } + + +if __name__ == "__main__": + fire.Fire(LearnwareMarketWorkflow) diff --git a/learnware/__init__.py b/learnware/__init__.py index b2f78f9..b27d0eb 100644 --- a/learnware/__init__.py +++ b/learnware/__init__.py @@ -1,12 +1,22 @@ __version__ = "0.0.1.99" +import os from .logger import get_module_logger def init(**kwargs): from .config import C + C.reset() C.update(**kwargs) logger = get_module_logger("Initialization") + + ## make dirs + os.makedirs(C.root_path, exist_ok=True) + os.makedirs(C.database_path, exist_ok=True) + os.makedirs(C.learnware_pool_path, exist_ok=True) + os.makedirs(C.learnware_zip_pool_path, exist_ok=True) + os.makedirs(C.learnware_folder_pool_path, exist_ok=True) + logger.info(f"init learnware market with {kwargs}") diff --git a/learnware/config.py b/learnware/config.py index f2c650b..2e10988 100644 --- a/learnware/config.py +++ b/learnware/config.py @@ -48,13 +48,22 @@ class Config: self.__dict__["_config"].update(*args, **kwargs) -ROOT_DIRPATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +ROOT_DIRPATH = os.path.join(os.path.expanduser("~"), ".learnware") SPEC_DIRPATH = None + LEARNWARE_POOL_PATH = os.path.join(ROOT_DIRPATH, "learnware_pool") LEARNWARE_ZIP_POOL_PATH = os.path.join(LEARNWARE_POOL_PATH, "zips") LEARNWARE_FOLDER_POOL_PATH = os.path.join(LEARNWARE_POOL_PATH, "learnwares") + +DATABASE_PATH = os.path.join(ROOT_DIRPATH, "database") + + +# TODO: Delete them later +os.makedirs(ROOT_DIRPATH, exist_ok=True) +os.makedirs(LEARNWARE_POOL_PATH, exist_ok=True) os.makedirs(LEARNWARE_ZIP_POOL_PATH, exist_ok=True) os.makedirs(LEARNWARE_FOLDER_POOL_PATH, exist_ok=True) +os.makedirs(DATABASE_PATH, exist_ok=True) semantic_config = { "Data": { @@ -109,11 +118,16 @@ semantic_config = { _DEFAULT_CONFIG = { "root_path": ROOT_DIRPATH, "logging_level": logging.INFO, - "specification_path": SPEC_DIRPATH, + "logging_outfile": None, "semantic_specs": semantic_config, "learnware_pool_path": LEARNWARE_POOL_PATH, "learnware_zip_pool_path": LEARNWARE_ZIP_POOL_PATH, "learnware_folder_pool_path": LEARNWARE_FOLDER_POOL_PATH, + "learnware_folder_config": { + "yaml_file": "learnware.yaml", + "module_file": "__init__.py", + }, + "database_path": DATABASE_PATH, } C = Config(_DEFAULT_CONFIG) diff --git a/learnware/learnware/__init__.py b/learnware/learnware/__init__.py index d1d3b8e..aa24867 100644 --- a/learnware/learnware/__init__.py +++ b/learnware/learnware/__init__.py @@ -6,6 +6,7 @@ from .utils import get_stat_spec_from_config, get_model_from_config from ..specification import Specification from ..utils import read_yaml_to_dict from ..logger import get_module_logger +from ..config import C logger = get_module_logger("learnware.learnware") @@ -44,7 +45,7 @@ def get_learnware_from_dirpath(id: str, semantic_spec: dict, learnware_dirpath: if learnware_dirpath is not None: try: - yaml_config = read_yaml_to_dict(os.path.join(learnware_dirpath, "learnware.yaml")) + yaml_config = read_yaml_to_dict(os.path.join(learnware_dirpath, C.learnware_folder_config["yaml_file"])) except FileNotFoundError: yaml_config = {} @@ -56,7 +57,9 @@ def get_learnware_from_dirpath(id: str, semantic_spec: dict, learnware_dirpath: learnware_config["stat_specifications"] = yaml_config["stat_specifications"].copy() if "module_path" not in learnware_config["model"]: - learnware_config["model"]["module_path"] = os.path.join(learnware_dirpath, "__init__.py") + learnware_config["model"]["module_path"] = os.path.join( + learnware_dirpath, C.learnware_folder_config["module_file"] + ) try: learnware_spec = Specification() diff --git a/learnware/logger.py b/learnware/logger.py index e9aae86..9604929 100644 --- a/learnware/logger.py +++ b/learnware/logger.py @@ -12,6 +12,8 @@ def get_module_logger(module_name: str, level: int = None, outfile: str = None) Logic module name. level : int, optional Logging level, by default None + outfile : str, optional + The output filepath, by default None Returns ------- @@ -20,6 +22,8 @@ def get_module_logger(module_name: str, level: int = None, outfile: str = None) """ if level is None: level = C.logging_level + if outfile is None: + outfile = C.logging_outfile # Get logger. console_handler = logging.StreamHandler() diff --git a/learnware/market/database_ops.py b/learnware/market/database_ops.py index 3fbf548..67c8304 100644 --- a/learnware/market/database_ops.py +++ b/learnware/market/database_ops.py @@ -5,22 +5,20 @@ from copy import deepcopy from ..logger import get_module_logger from ..learnware import get_learnware_from_dirpath +from ..config import C - -ROOT_PATH = os.path.dirname(os.path.abspath(__file__)) -DB_PATH = os.path.join(ROOT_PATH, "market.db") -LOGGER = get_module_logger("db") +logger = get_module_logger("database_ops") def init_empty_db(func): def wrapper(*args, **kwargs): - conn = sqlite3.connect(DB_PATH) + conn = sqlite3.connect(os.path.join(C.database_path, "market.db")) cur = conn.cursor() listOfTables = cur.execute( """SELECT name FROM sqlite_master WHERE type='table' AND name='LEARNWARE'; """ ).fetchall() - if listOfTables == []: - LOGGER.info("Initializing Database in %s..." % (DB_PATH)) + if len(listOfTables) == 0: + logger.info("Initializing Database in %s..." % (os.path.join(C.database_path, "market.db"))) cur.execute( """CREATE TABLE LEARNWARE (ID CHAR(10) PRIMARY KEY NOT NULL, @@ -28,7 +26,7 @@ def init_empty_db(func): ZIP_PATH TEXT NOT NULL, FOLDER_PATH TEXT NOT NULL);""" ) - LOGGER.info("Database Built!") + logger.info("Database Built!") kwargs["cur"] = cur item = func(*args, **kwargs) conn.commit() @@ -46,7 +44,7 @@ def init_empty_db(func): # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @init_empty_db def clear_learnware_table(cur): - LOGGER.warning("!!! Drop Learnware Table !!!") + logger.warning("!!! Drop Learnware Table !!!") cur.execute("DROP TABLE LEARNWARE") @@ -67,7 +65,7 @@ def delete_learnware_from_db(id: str, cur): @init_empty_db def load_market_from_db(cur): - LOGGER.info("Reload from Database") + logger.info("Reload from Database") cursor = cur.execute("SELECT id, semantic_spec, zip_path, FOLDER_PATH from LEARNWARE") learnware_list = {} @@ -86,5 +84,5 @@ def load_market_from_db(cur): folder_list[id] = folder_path max_count = max(max_count, int(id)) - LOGGER.info("Market Reloaded from DB.") + logger.info("Market Reloaded from DB.") return learnware_list, zip_list, folder_list, max_count + 1 diff --git a/learnware/market/easy.py b/learnware/market/easy.py index 1748bce..8b7aeaf 100644 --- a/learnware/market/easy.py +++ b/learnware/market/easy.py @@ -82,7 +82,8 @@ class EasyMarket(BaseMarket): Returns ------- Tuple[str, bool] - str indicating model_id, bool indicating whether the learnware is added successfully. + - str indicating model_id + - bool indicating whether the learnware is added successfully. """ if not os.path.exists(zip_path): @@ -104,6 +105,7 @@ class EasyMarket(BaseMarket): ) except: new_learnware = None + if new_learnware is None: try: os.remove(target_zip_dir) diff --git a/learnware/utils.py b/learnware/utils.py index 18f7285..7f20da5 100644 --- a/learnware/utils.py +++ b/learnware/utils.py @@ -12,13 +12,6 @@ from .logger import get_module_logger logger = get_module_logger("utils") -def make_dir_by_path(dirpath): - if not os.path.exists(dirpath): - os.makedirs(dirpath) - else: - logger.warning(f"Directorty {dirpath} has been exited, ignore mkdir") - - def get_module_by_module_path(module_path: Union[str, ModuleType]): if module_path is None: raise ModuleNotFoundError("None is passed in as parameters as module_path") diff --git a/setup.py b/setup.py index f160a43..8ef3ace 100644 --- a/setup.py +++ b/setup.py @@ -44,6 +44,7 @@ REQUIRED = [ "scikit-learn>=1.2.2", "joblib>=1.2.0", "pyyaml>=6.0", + "fire>=0.5.0", ] here = os.path.abspath(os.path.dirname(__file__))