From 0d1e0e94a1bd7bebd48d7a5922c8496530da75a6 Mon Sep 17 00:00:00 2001 From: Gene Date: Wed, 8 Nov 2023 20:53:40 +0800 Subject: [PATCH 01/15] [MNT] change geatpy to local import --- learnware/reuse/ensemble_pruning.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/learnware/reuse/ensemble_pruning.py b/learnware/reuse/ensemble_pruning.py index d20664c..ab86c66 100644 --- a/learnware/reuse/ensemble_pruning.py +++ b/learnware/reuse/ensemble_pruning.py @@ -1,8 +1,6 @@ import torch import random import numpy as np -import geatpy as ea - from typing import List from learnware.learnware import Learnware @@ -55,6 +53,8 @@ class EnsemblePruningReuser(BaseReuser): np.ndarray Binary one-dimensional vector, 1 indicates that the corresponding model is selected. """ + import geatpy as ea + model_num = v_predict.shape[1] @ea.Problem.single @@ -139,6 +139,8 @@ class EnsemblePruningReuser(BaseReuser): np.ndarray Binary one-dimensional vector, 1 indicates that the corresponding model is selected. """ + import geatpy as ea + model_num = v_predict.shape[1] def find_top_two_freq(row): @@ -253,6 +255,8 @@ class EnsemblePruningReuser(BaseReuser): np.ndarray Binary one-dimensional vector, 1 indicates that the corresponding model is selected. """ + import geatpy as ea + model_num = v_predict.shape[1] v_predict[v_predict == 0.0] = -1 v_true[v_true == 0.0] = -1 From 35188f9184e45e887e3aeeb40d256764026d0a7b Mon Sep 17 00:00:00 2001 From: Gene Date: Wed, 8 Nov 2023 20:54:47 +0800 Subject: [PATCH 02/15] [MNT] remove geatpy and matplotlib in dependencies --- setup.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/setup.py b/setup.py index 67f7254..5d48bfb 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,6 @@ REQUIRED = [ "numpy>=1.20.0", "pandas>=0.25.1", "scipy>=1.0.0", - "matplotlib>=3.1.3", "torch>=1.11.0", "cvxopt>=1.3.0", "tqdm>=4.65.0", @@ -67,7 +66,6 @@ REQUIRED = [ "torchvision>=0.15.1", "sqlalchemy>=2.0.21", "shortuuid>=1.0.11", - "geatpy>=2.7.0", "docker>=6.1.3", "rapidfuzz>=3.4.0", "torchtext>=0.16.0", From 07933633a1c3b3ac9271d49ba3aafe8fc3f22a04 Mon Sep 17 00:00:00 2001 From: bxdd Date: Thu, 9 Nov 2023 14:12:33 +0800 Subject: [PATCH 03/15] [MNT] update workflow name --- tests/test_workflow/test_workflow.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_workflow/test_workflow.py b/tests/test_workflow/test_workflow.py index 492c8a9..fac8348 100644 --- a/tests/test_workflow/test_workflow.py +++ b/tests/test_workflow/test_workflow.py @@ -30,7 +30,7 @@ user_semantic = { } -class TestMarket(unittest.TestCase): +class TestWorkflow(unittest.TestCase): @classmethod def setUpClass(cls) -> None: np.random.seed(2023) @@ -226,11 +226,11 @@ class TestMarket(unittest.TestCase): def suite(): _suite = unittest.TestSuite() - _suite.addTest(TestMarket("test_prepare_learnware_randomly")) - _suite.addTest(TestMarket("test_upload_delete_learnware")) - _suite.addTest(TestMarket("test_search_semantics")) - _suite.addTest(TestMarket("test_stat_search")) - _suite.addTest(TestMarket("test_learnware_reuse")) + _suite.addTest(TestWorkflow("test_prepare_learnware_randomly")) + _suite.addTest(TestWorkflow("test_upload_delete_learnware")) + _suite.addTest(TestWorkflow("test_search_semantics")) + _suite.addTest(TestWorkflow("test_stat_search")) + _suite.addTest(TestWorkflow("test_learnware_reuse")) return _suite From baf86091cda30f98e25cc4e11a93a32f4726bc4d Mon Sep 17 00:00:00 2001 From: bxdd Date: Thu, 9 Nov 2023 14:42:43 +0800 Subject: [PATCH 04/15] [MNT] update support python verison --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 67f7254..59470c6 100644 --- a/setup.py +++ b/setup.py @@ -108,8 +108,10 @@ if __name__ == "__main__": "Operating System :: POSIX :: Linux", "Operating System :: Microsoft :: Windows", "Operating System :: MacOS", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ], ) From 525a7b01bf8e0f9849592a5fe86707c602f326b2 Mon Sep 17 00:00:00 2001 From: bxdd Date: Thu, 9 Nov 2023 15:16:51 +0800 Subject: [PATCH 05/15] [ENH] add extras_require --- setup.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 59470c6..0280320 100644 --- a/setup.py +++ b/setup.py @@ -64,13 +64,13 @@ REQUIRED = [ "fire>=0.3.1", "lightgbm>=3.3.0", "psutil>=5.9.4", - "torchvision>=0.15.1", + "torchvision>=0.14.1", "sqlalchemy>=2.0.21", "shortuuid>=1.0.11", "geatpy>=2.7.0", "docker>=6.1.3", "rapidfuzz>=3.4.0", - "torchtext>=0.16.0", + "torchtext>=0.14.1", "sentence_transformers>=2.2.2", "torch-optimizer>=0.3.0", "langdetect>=1.0.9", @@ -99,6 +99,23 @@ if __name__ == "__main__": long_description_content_type="text/markdown", python_requires=REQUIRES_PYTHON, install_requires=REQUIRED, + extras_require={ + "dev": [ + # For documentation + "sphinx", + "sphinx_rtd_theme", + "pre-commit", + # CI dependencies + "pytest>=3", + "wheel", + "setuptools", + "pylint", + # For static analysis + "mypy<0.981", + "flake8", + "black==23.1.0" + ], + }, classifiers=[ "Intended Audience :: Science/Research", "Intended Audience :: Developers", From 4fa55b96fa52de8904b68af69a465087d4482808 Mon Sep 17 00:00:00 2001 From: bxdd Date: Thu, 9 Nov 2023 15:20:09 +0800 Subject: [PATCH 06/15] [ENH] add pre-commit --- .pre-commit-config.yaml | 12 ++++++++++++ learnware/market/base.py | 2 +- learnware/market/easy/database_ops.py | 2 +- learnware/market/easy/organizer.py | 3 ++- setup.py | 2 +- 5 files changed, 17 insertions(+), 4 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..fe7c6a2 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,12 @@ +repos: +- repo: https://github.com/psf/black + rev: 23.1.0 + hooks: + - id: black + args: [".", "-l 120"] + +- repo: https://github.com/PyCQA/flake8 + rev: 4.0.1 + hooks: + - id: flake8 + args: [".", "--count", "--select=E9,F63,F7,F82", "--show-source", "--statistics"] \ No newline at end of file diff --git a/learnware/market/base.py b/learnware/market/base.py index a422479..d061d7f 100644 --- a/learnware/market/base.py +++ b/learnware/market/base.py @@ -227,7 +227,7 @@ class LearnwareMarket: def reload_learnware(self, learnware_id: str): self.learnware_organizer.reload_learnware(learnware_id) - + def get_learnware_zip_path_by_ids(self, ids: Union[str, List[str]], **kwargs) -> Union[Learnware, List[Learnware]]: return self.learnware_organizer.get_learnware_zip_path_by_ids(ids, **kwargs) diff --git a/learnware/market/easy/database_ops.py b/learnware/market/easy/database_ops.py index 077a04e..c9fb3de 100644 --- a/learnware/market/easy/database_ops.py +++ b/learnware/market/easy/database_ops.py @@ -166,7 +166,7 @@ class DatabaseOperations(object): return int(row[0]) pass pass - + def load_market(self): with self.engine.connect() as conn: cursor = conn.execute(text("SELECT id, semantic_spec, zip_path, folder_path, use_flag FROM tb_learnware;")) diff --git a/learnware/market/easy/organizer.py b/learnware/market/easy/organizer.py index 3d6d6b6..9337841 100644 --- a/learnware/market/easy/organizer.py +++ b/learnware/market/easy/organizer.py @@ -387,7 +387,8 @@ class EasyOrganizer(BaseOrganizer): self.learnware_folder_list[learnware_id] = target_folder_dir semantic_spec = self.dbops.get_learnware_semantic_specification(learnware_id) self.learnware_list[learnware_id] = get_learnware_from_dirpath( - id=learnware_id, semantic_spec=semantic_spec, learnware_dirpath=target_folder_dir) + id=learnware_id, semantic_spec=semantic_spec, learnware_dirpath=target_folder_dir + ) self.use_flags[learnware_id] = self.dbops.get_learnware_use_flag(learnware_id) pass diff --git a/setup.py b/setup.py index 0280320..3b2a8b4 100644 --- a/setup.py +++ b/setup.py @@ -113,7 +113,7 @@ if __name__ == "__main__": # For static analysis "mypy<0.981", "flake8", - "black==23.1.0" + "black==23.1.0", ], }, classifiers=[ From 1db352a308085d624362dee2eddceaf40b5b3c7f Mon Sep 17 00:00:00 2001 From: bxdd Date: Thu, 9 Nov 2023 15:30:20 +0800 Subject: [PATCH 07/15] [MNT] update set up --- .pre-commit-config.yaml | 4 ++-- setup.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fe7c6a2..f8443a1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,10 +3,10 @@ repos: rev: 23.1.0 hooks: - id: black - args: [".", "-l 120"] + args: ["-l 120"] - repo: https://github.com/PyCQA/flake8 rev: 4.0.1 hooks: - id: flake8 - args: [".", "--count", "--select=E9,F63,F7,F82", "--show-source", "--statistics"] \ No newline at end of file + args: ["--count", "--select=E9,F63,F7,F82", "--show-source", "--statistics"] \ No newline at end of file diff --git a/setup.py b/setup.py index 3b2a8b4..fadbcf0 100644 --- a/setup.py +++ b/setup.py @@ -101,10 +101,9 @@ if __name__ == "__main__": install_requires=REQUIRED, extras_require={ "dev": [ - # For documentation + # For documentations "sphinx", "sphinx_rtd_theme", - "pre-commit", # CI dependencies "pytest>=3", "wheel", @@ -114,6 +113,7 @@ if __name__ == "__main__": "mypy<0.981", "flake8", "black==23.1.0", + "pre-commit", ], }, classifiers=[ From c7c57e5823dc5374cdeeb045a138a05962eec6ba Mon Sep 17 00:00:00 2001 From: bxdd Date: Thu, 9 Nov 2023 19:22:03 +0800 Subject: [PATCH 08/15] [MNT] rename test to tests folder --- learnware/client/learnware_client.py | 2 +- learnware/{test => tests}/__init__.py | 0 learnware/{test => tests}/data.py | 0 learnware/{test => tests}/module.py | 0 4 files changed, 1 insertion(+), 1 deletion(-) rename learnware/{test => tests}/__init__.py (100%) rename learnware/{test => tests}/data.py (100%) rename learnware/{test => tests}/module.py (100%) diff --git a/learnware/client/learnware_client.py b/learnware/client/learnware_client.py index aa95e7d..2876863 100644 --- a/learnware/client/learnware_client.py +++ b/learnware/client/learnware_client.py @@ -18,7 +18,7 @@ from ..market import BaseChecker, EasySemanticChecker, EasyStatChecker from ..logger import get_module_logger from ..specification import Specification from ..learnware import get_learnware_from_dirpath -from ..test import get_semantic_specification +from ..tests import get_semantic_specification CHUNK_SIZE = 1024 * 1024 logger = get_module_logger(module_name="LearnwareClient") diff --git a/learnware/test/__init__.py b/learnware/tests/__init__.py similarity index 100% rename from learnware/test/__init__.py rename to learnware/tests/__init__.py diff --git a/learnware/test/data.py b/learnware/tests/data.py similarity index 100% rename from learnware/test/data.py rename to learnware/tests/data.py diff --git a/learnware/test/module.py b/learnware/tests/module.py similarity index 100% rename from learnware/test/module.py rename to learnware/tests/module.py From cad6b817cbfb0214081f41645425ddbd349aa47d Mon Sep 17 00:00:00 2001 From: bxdd Date: Thu, 9 Nov 2023 19:46:44 +0800 Subject: [PATCH 09/15] [MNT] refactor utils to folder, decouple base model with torch --- learnware/model/base.py | 7 +++---- learnware/{utils.py => utils/__init__.py} | 2 +- learnware/utils/import_utils.py | 0 3 files changed, 4 insertions(+), 5 deletions(-) rename learnware/{utils.py => utils/__init__.py} (97%) create mode 100644 learnware/utils/import_utils.py diff --git a/learnware/model/base.py b/learnware/model/base.py index 26ec380..e54d858 100644 --- a/learnware/model/base.py +++ b/learnware/model/base.py @@ -1,5 +1,4 @@ import numpy as np -import torch from typing import Union @@ -19,7 +18,7 @@ class BaseModel: self.input_shape = input_shape self.output_shape = output_shape - def predict(self, X: Union[np.ndarray, torch.tensor]) -> Union[np.ndarray, torch.tensor]: + def predict(self, X: np.ndarray) -> np.ndarray: """The prediction method for model in learnware, which will be checked when learnware is submitted into the market. Parameters @@ -33,10 +32,10 @@ class BaseModel: """ pass - def fit(self, X: Union[np.ndarray, torch.tensor], y: Union[np.ndarray, torch.tensor]): + def fit(self, X: np.ndarray, y: np.ndarray): pass - def finetune(self, X: Union[np.ndarray, torch.tensor], y: np.ndarray): + def finetune(self, X: np.ndarray, y: np.ndarray): """The finetune method for continuing train the model searched by market Parameters diff --git a/learnware/utils.py b/learnware/utils/__init__.py similarity index 97% rename from learnware/utils.py rename to learnware/utils/__init__.py index 9bea664..9708703 100644 --- a/learnware/utils.py +++ b/learnware/utils/__init__.py @@ -8,7 +8,7 @@ import importlib.util from typing import Union from types import ModuleType import zipfile -from .logger import get_module_logger +from ..logger import get_module_logger logger = get_module_logger("utils") diff --git a/learnware/utils/import_utils.py b/learnware/utils/import_utils.py new file mode 100644 index 0000000..e69de29 From c047307b4603e99f85bcfe10f88441a8fcdacc49 Mon Sep 17 00:00:00 2001 From: bxdd Date: Thu, 9 Nov 2023 20:12:19 +0800 Subject: [PATCH 10/15] [MNT] refactor utils to folder, decouple base model with torch --- examples/dataset_text_workflow/get_data.py | 1 - .../dataset_text_workflow/requirements.txt | 1 + learnware/utils/import_utils.py | 32 +++++++++++++++++++ setup.py | 1 - 4 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 examples/dataset_text_workflow/requirements.txt diff --git a/examples/dataset_text_workflow/get_data.py b/examples/dataset_text_workflow/get_data.py index 0d1412d..770fd80 100644 --- a/examples/dataset_text_workflow/get_data.py +++ b/examples/dataset_text_workflow/get_data.py @@ -1,4 +1,3 @@ -import torch from torchtext.datasets import SST2 diff --git a/examples/dataset_text_workflow/requirements.txt b/examples/dataset_text_workflow/requirements.txt new file mode 100644 index 0000000..66500d8 --- /dev/null +++ b/examples/dataset_text_workflow/requirements.txt @@ -0,0 +1 @@ +torchtext>=0.14.1 diff --git a/learnware/utils/import_utils.py b/learnware/utils/import_utils.py index e69de29..fdd31ab 100644 --- a/learnware/utils/import_utils.py +++ b/learnware/utils/import_utils.py @@ -0,0 +1,32 @@ +from ..logger import get_module_logger + +logger = get_module_logger("import_utils") + + +def is_torch_avaliable(): + try: + import torch + except ModuleNotFoundError as err: + logger.warning("ModuleNotFoundError: torch is not installed, please install pytorch!") + return False + return True + + +def is_lightgbm_avaliable(): + try: + import lightgbm + except ModuleNotFoundError as err: + logger.warning("ModuleNotFoundError: lightgbm is not installed, please install lightgbm!") + return False + return True + + +def is_geatpy_avaliable(): + try: + import geatpy + except ModuleNotFoundError as err: + logger.warning( + "ModuleNotFoundError: geatpy is not installed, please install geatpy (only support python version<3.11)!" + ) + return False + return True diff --git a/setup.py b/setup.py index fadbcf0..05c9681 100644 --- a/setup.py +++ b/setup.py @@ -70,7 +70,6 @@ REQUIRED = [ "geatpy>=2.7.0", "docker>=6.1.3", "rapidfuzz>=3.4.0", - "torchtext>=0.14.1", "sentence_transformers>=2.2.2", "torch-optimizer>=0.3.0", "langdetect>=1.0.9", From 41c9124cfef57417cb10c4ae3f553a4f85be648c Mon Sep 17 00:00:00 2001 From: bxdd Date: Thu, 9 Nov 2023 20:23:03 +0800 Subject: [PATCH 11/15] [MNT] refactor utils, and decouple geatpy --- learnware/reuse/__init__.py | 9 +++++- learnware/reuse/ensemble_pruning.py | 7 +---- learnware/utils/__init__.py | 48 ++--------------------------- learnware/utils/file.py | 14 +++++++++ learnware/utils/module.py | 24 +++++++++++++++ 5 files changed, 50 insertions(+), 52 deletions(-) create mode 100644 learnware/utils/file.py create mode 100644 learnware/utils/module.py diff --git a/learnware/reuse/__init__.py b/learnware/reuse/__init__.py index 0ea82c5..575fc33 100644 --- a/learnware/reuse/__init__.py +++ b/learnware/reuse/__init__.py @@ -1,3 +1,10 @@ -from .ensemble_pruning import EnsemblePruningReuser from .averaging import AveragingReuser from .job_selector import JobSelectorReuser + + +from ..utils.import_utils import is_geatpy_avaliable + +if is_geatpy_avaliable(verbose=True): + from .ensemble_pruning import EnsemblePruningReuser +else: + EnsemblePruningReuser = None diff --git a/learnware/reuse/ensemble_pruning.py b/learnware/reuse/ensemble_pruning.py index ab86c66..6001880 100644 --- a/learnware/reuse/ensemble_pruning.py +++ b/learnware/reuse/ensemble_pruning.py @@ -1,6 +1,7 @@ import torch import random import numpy as np +import geatpy as ea from typing import List from learnware.learnware import Learnware @@ -53,8 +54,6 @@ class EnsemblePruningReuser(BaseReuser): np.ndarray Binary one-dimensional vector, 1 indicates that the corresponding model is selected. """ - import geatpy as ea - model_num = v_predict.shape[1] @ea.Problem.single @@ -139,8 +138,6 @@ class EnsemblePruningReuser(BaseReuser): np.ndarray Binary one-dimensional vector, 1 indicates that the corresponding model is selected. """ - import geatpy as ea - model_num = v_predict.shape[1] def find_top_two_freq(row): @@ -255,8 +252,6 @@ class EnsemblePruningReuser(BaseReuser): np.ndarray Binary one-dimensional vector, 1 indicates that the corresponding model is selected. """ - import geatpy as ea - model_num = v_predict.shape[1] v_predict[v_predict == 0.0] = -1 v_true[v_true == 0.0] = -1 diff --git a/learnware/utils/__init__.py b/learnware/utils/__init__.py index 9708703..c73afae 100644 --- a/learnware/utils/__init__.py +++ b/learnware/utils/__init__.py @@ -1,47 +1,9 @@ import os -import sys - -import re -import yaml -import importlib -import importlib.util -from typing import Union -from types import ModuleType import zipfile -from ..logger import get_module_logger - -logger = get_module_logger("utils") - - -def get_module_by_module_path(module_path: Union[str, ModuleType]): - if module_path is None: - raise ModuleNotFoundError("None is passed in as parameters as module_path") - - if isinstance(module_path, ModuleType): - module = module_path - else: - if module_path.endswith(".py"): - module_name = re.sub("^[^a-zA-Z_]+", "", re.sub("[^0-9a-zA-Z_]", "", module_path[:-3].replace("/", "_"))) - module_spec = importlib.util.spec_from_file_location(module_name, module_path) - module = importlib.util.module_from_spec(module_spec) - sys.modules[module_name] = module - module_spec.loader.exec_module(module) - else: - module = importlib.import_module(module_path) - return module - - -def save_dict_to_yaml(dict_value: dict, save_path: str): - """save dict object into yaml file""" - with open(save_path, "w") as file: - file.write(yaml.dump(dict_value, allow_unicode=True)) - -def read_yaml_to_dict(yaml_path: str): - """load yaml file into dict object""" - with open(yaml_path, "r") as file: - dict_value = yaml.load(file.read(), Loader=yaml.FullLoader) - return dict_value +from .import_utils import is_torch_avaliable, is_lightgbm_avaliable, is_geatpy_avaliable +from .module import get_module_by_module_path +from .file import read_yaml_to_dict, save_dict_to_yaml def zip_learnware_folder(path: str, output_name: str): @@ -52,7 +14,3 @@ def zip_learnware_folder(path: str, output_name: str): if file.endswith(".pyc") or os.path.islink(full_path): continue zip_ref.write(full_path, arcname=os.path.relpath(full_path, path)) - pass - pass - pass - pass diff --git a/learnware/utils/file.py b/learnware/utils/file.py new file mode 100644 index 0000000..27ba5f5 --- /dev/null +++ b/learnware/utils/file.py @@ -0,0 +1,14 @@ +import yaml + + +def save_dict_to_yaml(dict_value: dict, save_path: str): + """save dict object into yaml file""" + with open(save_path, "w") as file: + file.write(yaml.dump(dict_value, allow_unicode=True)) + + +def read_yaml_to_dict(yaml_path: str): + """load yaml file into dict object""" + with open(yaml_path, "r") as file: + dict_value = yaml.load(file.read(), Loader=yaml.FullLoader) + return dict_value diff --git a/learnware/utils/module.py b/learnware/utils/module.py new file mode 100644 index 0000000..6f1b414 --- /dev/null +++ b/learnware/utils/module.py @@ -0,0 +1,24 @@ +import sys +import re +import importlib +import importlib.util +from typing import Union +from types import ModuleType + + +def get_module_by_module_path(module_path: Union[str, ModuleType]): + if module_path is None: + raise ModuleNotFoundError("None is passed in as parameters as module_path") + + if isinstance(module_path, ModuleType): + module = module_path + else: + if module_path.endswith(".py"): + module_name = re.sub("^[^a-zA-Z_]+", "", re.sub("[^0-9a-zA-Z_]", "", module_path[:-3].replace("/", "_"))) + module_spec = importlib.util.spec_from_file_location(module_name, module_path) + module = importlib.util.module_from_spec(module_spec) + sys.modules[module_name] = module + module_spec.loader.exec_module(module) + else: + module = importlib.import_module(module_path) + return module From 5037e5a3e0d413fc800c10da7b83c6e3903c1671 Mon Sep 17 00:00:00 2001 From: bxdd Date: Thu, 9 Nov 2023 21:07:55 +0800 Subject: [PATCH 12/15] [MNT] del all dependency --- learnware/__init__.py | 9 +++++-- learnware/market/easy/__init__.py | 16 ++++++++++-- learnware/reuse/__init__.py | 25 ++++++++++++++----- learnware/reuse/utils.py | 23 +++++++++++++++++ learnware/specification/__init__.py | 10 +++++++- learnware/specification/regular/__init__.py | 4 ++- .../specification/regular/image/__init__.py | 13 +++++++++- .../specification/regular/image/utils.py | 12 +++++++++ .../specification/regular/table/__init__.py | 12 ++++++++- .../specification/regular/text/__init__.py | 13 +++++++++- learnware/specification/regular/text/utils.py | 12 +++++++++ learnware/utils/__init__.py | 2 +- learnware/utils/import_utils.py | 20 --------------- setup.py | 3 --- 14 files changed, 135 insertions(+), 39 deletions(-) create mode 100644 learnware/reuse/utils.py create mode 100644 learnware/specification/regular/image/utils.py create mode 100644 learnware/specification/regular/text/utils.py diff --git a/learnware/__init__.py b/learnware/__init__.py index a57329c..b95ec35 100644 --- a/learnware/__init__.py +++ b/learnware/__init__.py @@ -2,6 +2,9 @@ __version__ = "0.1.1.99" import os from .logger import get_module_logger +from .utils import is_torch_avaliable + +logger = get_module_logger("Initialization") def init(make_dir: bool = False, tf_loglevel: str = "2", **kwargs): @@ -10,9 +13,7 @@ def init(make_dir: bool = False, tf_loglevel: str = "2", **kwargs): C.reset() C.update(**kwargs) - logger = get_module_logger("Initialization") logger.info(f"init learnware market with {kwargs}") - ## make dirs if make_dir: os.makedirs(C.root_path, exist_ok=True) @@ -25,3 +26,7 @@ def init(make_dir: bool = False, tf_loglevel: str = "2", **kwargs): ## ignore tensorflow warning # os.environ["TF_CPP_MIN_LOG_LEVEL"] = tf_loglevel # logger.info(f"The tensorflow log level is setted to {tf_loglevel}") + + +if not is_torch_avaliable(verbose=True): + logger.warning("The functionality of learnware is limited due to 'torch' is not installed!") diff --git a/learnware/market/easy/__init__.py b/learnware/market/easy/__init__.py index 2835871..b5514bd 100644 --- a/learnware/market/easy/__init__.py +++ b/learnware/market/easy/__init__.py @@ -1,3 +1,15 @@ from .organizer import EasyOrganizer -from .searcher import EasySearcher -from .checker import EasySemanticChecker, EasyStatChecker + +from ...utils import is_torch_avaliable +from ...logger import get_module_logger + +logger = get_module_logger("market_easy") + +if not is_torch_avaliable(verbose=False): + from .searcher import EasySearcher + from .checker import EasySemanticChecker, EasyStatChecker +else: + EasySearcher = None + EasySemanticChecker = None + EasyStatChecker = None + logger.warning("EasySeacher and EasyChecker are skipped because 'torch' is not installed!") diff --git a/learnware/reuse/__init__.py b/learnware/reuse/__init__.py index 575fc33..017403c 100644 --- a/learnware/reuse/__init__.py +++ b/learnware/reuse/__init__.py @@ -1,10 +1,23 @@ -from .averaging import AveragingReuser -from .job_selector import JobSelectorReuser +from ..logger import get_module_logger +from ..utils import is_torch_avaliable +from .utils import is_geatpy_avaliable, is_lightgbm_avaliable +logger = get_module_logger("reuse") -from ..utils.import_utils import is_geatpy_avaliable - -if is_geatpy_avaliable(verbose=True): +if not is_geatpy_avaliable(verbose=True): + EnsemblePruningReuser = None + logger.warning("EnsemblePruningReuser is skipped due to 'geatpy' is not installed!") +else: from .ensemble_pruning import EnsemblePruningReuser + +if not is_torch_avaliable(verbose=False): + AveragingReuser = None + logger.warning("AveragingReuser is skipped due to 'torch' is not installed!") else: - EnsemblePruningReuser = None + from .averaging import AveragingReuser + +if not is_lightgbm_avaliable(verbose=True) or not is_torch_avaliable(verbose=False): + JobSelectorReuser = None + logger.warning("JobSelectorReuser is skipped due to 'torch' is not installed!") +else: + from .job_selector import JobSelectorReuser diff --git a/learnware/reuse/utils.py b/learnware/reuse/utils.py new file mode 100644 index 0000000..f7f6440 --- /dev/null +++ b/learnware/reuse/utils.py @@ -0,0 +1,23 @@ +from ..logger import get_module_logger + +logger = get_module_logger("reuse_utils") + + +def is_geatpy_avaliable(): + try: + import geatpy + except ModuleNotFoundError as err: + logger.warning( + "ModuleNotFoundError: geatpy is not installed, please install geatpy (only support python version<3.11)!" + ) + return False + return True + + +def is_lightgbm_avaliable(): + try: + import lightgbm + except ModuleNotFoundError as err: + logger.warning("ModuleNotFoundError: lightgbm is not installed, please install lightgbm!") + return False + return True diff --git a/learnware/specification/__init__.py b/learnware/specification/__init__.py index 4ecedc3..2f599a2 100644 --- a/learnware/specification/__init__.py +++ b/learnware/specification/__init__.py @@ -1,4 +1,3 @@ -from .module import generate_stat_spec, generate_rkme_spec, generate_rkme_image_spec, generate_rkme_text_spec from .base import Specification, BaseStatSpecification from .regular import ( RegularStatsSpecification, @@ -7,3 +6,12 @@ from .regular import ( RKMEImageSpecification, RKMETextSpecification, ) +from ..utils import is_torch_avaliable + +if not is_torch_avaliable(verbose=False): + generate_stat_spec = None + generate_rkme_spec = None + generate_rkme_image_spec = None + generate_rkme_text_spec = None +else: + from .module import generate_stat_spec, generate_rkme_spec, generate_rkme_image_spec, generate_rkme_text_spec diff --git a/learnware/specification/regular/__init__.py b/learnware/specification/regular/__init__.py index 9007e4d..9d46114 100644 --- a/learnware/specification/regular/__init__.py +++ b/learnware/specification/regular/__init__.py @@ -1,4 +1,6 @@ +from .base import RegularStatsSpecification +from ...utils import is_torch_avaliable + from .text import RKMETextSpecification from .table import RKMETableSpecification, RKMEStatSpecification from .image import RKMEImageSpecification -from .base import RegularStatsSpecification diff --git a/learnware/specification/regular/image/__init__.py b/learnware/specification/regular/image/__init__.py index 0a18ded..3dd8022 100644 --- a/learnware/specification/regular/image/__init__.py +++ b/learnware/specification/regular/image/__init__.py @@ -1 +1,12 @@ -from .rkme import RKMEImageSpecification +from .utils import is_torch_optimizer_avaliable +from ....utils import is_torch_avaliable +from ....logger import get_module_logger + + +logger = get_module_logger("regular_image_spec") + +if not is_torch_optimizer_avaliable(verbose=True) or not is_torch_avaliable(verbose=False): + RKMEImageSpecification = None + logger.warning("RKMEImageSpecification is skipped because torch or torch-optimizer is not installed!") +else: + from .rkme import RKMEImageSpecification diff --git a/learnware/specification/regular/image/utils.py b/learnware/specification/regular/image/utils.py new file mode 100644 index 0000000..8c08db8 --- /dev/null +++ b/learnware/specification/regular/image/utils.py @@ -0,0 +1,12 @@ +from ....logger import get_module_logger + +logger = get_module_logger("regular_image_spec_utils") + + +def is_torch_optimizer_avaliable(): + try: + import torch_optimizer + except ModuleNotFoundError as err: + logger.warning("ModuleNotFoundError: torch_optimizer is not installed, please install pytorch!") + return False + return True diff --git a/learnware/specification/regular/table/__init__.py b/learnware/specification/regular/table/__init__.py index 19fa956..e8ec903 100644 --- a/learnware/specification/regular/table/__init__.py +++ b/learnware/specification/regular/table/__init__.py @@ -1 +1,11 @@ -from .rkme import RKMETableSpecification, RKMEStatSpecification +from ....utils import is_torch_avaliable +from ....logger import get_module_logger + +logger = get_module_logger("regular_table_spec") + +if not is_torch_avaliable(verbose=False): + RKMETableSpecification = None + RKMEStatSpecification = None + logger.warning("RKMETableSpecification is skipped because torch is not installed!") +else: + from .rkme import RKMETableSpecification, RKMEStatSpecification diff --git a/learnware/specification/regular/text/__init__.py b/learnware/specification/regular/text/__init__.py index 35b8b0a..8e996ea 100644 --- a/learnware/specification/regular/text/__init__.py +++ b/learnware/specification/regular/text/__init__.py @@ -1 +1,12 @@ -from .rkme import RKMETextSpecification +from .utils import is_sentence_transformers_avaliable + +from ....utils import is_torch_avaliable +from ....logger import get_module_logger + +logger = get_module_logger("regular_table_spec") + +if not is_sentence_transformers_avaliable(verbose=True) or not is_torch_avaliable(verbose=False): + RKMETextSpecification = None + logger.warning("RKMETextSpecification is skipped because torch is not installed!") +else: + from .rkme import RKMETextSpecification diff --git a/learnware/specification/regular/text/utils.py b/learnware/specification/regular/text/utils.py new file mode 100644 index 0000000..b8401ce --- /dev/null +++ b/learnware/specification/regular/text/utils.py @@ -0,0 +1,12 @@ +from ....logger import get_module_logger + +logger = get_module_logger("regular_text_spec_utils") + + +def is_sentence_transformers_avaliable(): + try: + import sentence_transformers + except ModuleNotFoundError as err: + logger.warning("ModuleNotFoundError: sentence_transformers is not installed, please install pytorch!") + return False + return True diff --git a/learnware/utils/__init__.py b/learnware/utils/__init__.py index c73afae..60f2b46 100644 --- a/learnware/utils/__init__.py +++ b/learnware/utils/__init__.py @@ -1,7 +1,7 @@ import os import zipfile -from .import_utils import is_torch_avaliable, is_lightgbm_avaliable, is_geatpy_avaliable +from .import_utils import is_torch_avaliable from .module import get_module_by_module_path from .file import read_yaml_to_dict, save_dict_to_yaml diff --git a/learnware/utils/import_utils.py b/learnware/utils/import_utils.py index fdd31ab..3a10ea8 100644 --- a/learnware/utils/import_utils.py +++ b/learnware/utils/import_utils.py @@ -10,23 +10,3 @@ def is_torch_avaliable(): logger.warning("ModuleNotFoundError: torch is not installed, please install pytorch!") return False return True - - -def is_lightgbm_avaliable(): - try: - import lightgbm - except ModuleNotFoundError as err: - logger.warning("ModuleNotFoundError: lightgbm is not installed, please install lightgbm!") - return False - return True - - -def is_geatpy_avaliable(): - try: - import geatpy - except ModuleNotFoundError as err: - logger.warning( - "ModuleNotFoundError: geatpy is not installed, please install geatpy (only support python version<3.11)!" - ) - return False - return True diff --git a/setup.py b/setup.py index 7144375..e996025 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,6 @@ REQUIRED = [ "numpy>=1.20.0", "pandas>=0.25.1", "scipy>=1.0.0", - "torch>=1.11.0", "cvxopt>=1.3.0", "tqdm>=4.65.0", "scikit-learn>=0.22", @@ -68,8 +67,6 @@ REQUIRED = [ "shortuuid>=1.0.11", "docker>=6.1.3", "rapidfuzz>=3.4.0", - "sentence_transformers>=2.2.2", - "torch-optimizer>=0.3.0", "langdetect>=1.0.9", "huggingface-hub<0.18", "portalocker>=2.0.0", From 5a4f4a51e42519a4de12260f6dd8247d3dc7b6c8 Mon Sep 17 00:00:00 2001 From: bxdd Date: Thu, 9 Nov 2023 22:29:42 +0800 Subject: [PATCH 13/15] [MNT] now it pass the test --- learnware/__init__.py | 2 +- learnware/market/__init__.py | 2 +- learnware/market/anchor/__init__.py | 13 +++++- learnware/market/anchor/searcher.py | 40 +----------------- learnware/market/anchor/user_info.py | 41 +++++++++++++++++++ learnware/market/easy/__init__.py | 6 +-- learnware/reuse/__init__.py | 17 ++++++-- learnware/reuse/utils.py | 14 ++++--- .../specification/regular/image/__init__.py | 23 +++++++++-- .../specification/regular/image/utils.py | 15 ++++++- .../specification/regular/text/__init__.py | 17 ++++++-- learnware/specification/regular/text/utils.py | 7 +++- learnware/utils/import_utils.py | 5 ++- setup.py | 2 - 14 files changed, 136 insertions(+), 68 deletions(-) create mode 100644 learnware/market/anchor/user_info.py diff --git a/learnware/__init__.py b/learnware/__init__.py index b95ec35..bbbadd2 100644 --- a/learnware/__init__.py +++ b/learnware/__init__.py @@ -28,5 +28,5 @@ def init(make_dir: bool = False, tf_loglevel: str = "2", **kwargs): # logger.info(f"The tensorflow log level is setted to {tf_loglevel}") -if not is_torch_avaliable(verbose=True): +if not is_torch_avaliable(verbose=False): logger.warning("The functionality of learnware is limited due to 'torch' is not installed!") diff --git a/learnware/market/__init__.py b/learnware/market/__init__.py index 8a3645d..a5fdfff 100644 --- a/learnware/market/__init__.py +++ b/learnware/market/__init__.py @@ -1,4 +1,4 @@ -from .anchor import AnchoredUserInfo, AnchoredOrganizer +from .anchor import AnchoredUserInfo, AnchoredSearcher, AnchoredOrganizer from .base import BaseUserInfo, LearnwareMarket, BaseChecker, BaseOrganizer, BaseSearcher from .evolve_anchor import EvolvedAnchoredOrganizer from .evolve import EvolvedOrganizer diff --git a/learnware/market/anchor/__init__.py b/learnware/market/anchor/__init__.py index f2b453c..c005c0c 100644 --- a/learnware/market/anchor/__init__.py +++ b/learnware/market/anchor/__init__.py @@ -1,2 +1,13 @@ from .organizer import AnchoredOrganizer -from .searcher import AnchoredUserInfo +from .user_info import AnchoredUserInfo + +from ...utils import is_torch_avaliable +from ...logger import get_module_logger + +logger = get_module_logger("market_anchor") + +if not is_torch_avaliable(verbose=False): + AnchoredSearcher = None + logger.warning("AnchoredSearcher is skipped because 'torch' is not installed!") +else: + from .searcher import AnchoredSearcher diff --git a/learnware/market/anchor/searcher.py b/learnware/market/anchor/searcher.py index b0d826f..34d326d 100644 --- a/learnware/market/anchor/searcher.py +++ b/learnware/market/anchor/searcher.py @@ -1,5 +1,6 @@ from typing import List, Dict, Tuple, Any, Union +from .user_info import AnchoredUserInfo from ..base import BaseUserInfo from ..easy.searcher import EasySearcher from ...logger import get_module_logger @@ -8,45 +9,6 @@ from ...learnware import Learnware logger = get_module_logger("anchor_searcher") -class AnchoredUserInfo(BaseUserInfo): - """ - User Information for searching learnware (add the anchor design) - - - UserInfo contains the anchor id list acquired from the market - - UserInfo can update stat_info based on anchors - """ - - def __init__( - self, id: str, semantic_spec: dict = None, stat_info: dict = None, anchor_learnware_ids: List[str] = None - ): - super(AnchoredUserInfo, self).__init__(id, semantic_spec, stat_info) - self.anchor_learnware_ids = [] if anchor_learnware_ids is None else anchor_learnware_ids - - def add_anchor_learnware_ids(self, learnware_ids: Union[str, List[str]]): - """Add the anchor learnware ids acquired from the market - - Parameters - ---------- - learnware_ids : Union[str, List[str]] - Anchor learnware ids - """ - if isinstance(learnware_ids, str): - learnware_ids = [learnware_ids] - self.anchor_learnware_ids += learnware_ids - - def update_stat_info(self, name: str, item: Any): - """Update stat_info based on anchor learnwares - - Parameters - ---------- - name : str - Name of stat_info - item : Any - Statistical information calculated on anchor learnwares - """ - self.stat_info[name] = item - - class AnchoredSearcher(EasySearcher): def search_anchor_learnware(self, user_info: AnchoredUserInfo) -> Tuple[Any, List[Learnware]]: """Search anchor Learnwares from anchor_learnware_list based on user_info diff --git a/learnware/market/anchor/user_info.py b/learnware/market/anchor/user_info.py new file mode 100644 index 0000000..7ae4737 --- /dev/null +++ b/learnware/market/anchor/user_info.py @@ -0,0 +1,41 @@ +from typing import List, Any, Union +from ..base import BaseUserInfo + + +class AnchoredUserInfo(BaseUserInfo): + """ + User Information for searching learnware (add the anchor design) + + - UserInfo contains the anchor id list acquired from the market + - UserInfo can update stat_info based on anchors + """ + + def __init__( + self, id: str, semantic_spec: dict = None, stat_info: dict = None, anchor_learnware_ids: List[str] = None + ): + super(AnchoredUserInfo, self).__init__(id, semantic_spec, stat_info) + self.anchor_learnware_ids = [] if anchor_learnware_ids is None else anchor_learnware_ids + + def add_anchor_learnware_ids(self, learnware_ids: Union[str, List[str]]): + """Add the anchor learnware ids acquired from the market + + Parameters + ---------- + learnware_ids : Union[str, List[str]] + Anchor learnware ids + """ + if isinstance(learnware_ids, str): + learnware_ids = [learnware_ids] + self.anchor_learnware_ids += learnware_ids + + def update_stat_info(self, name: str, item: Any): + """Update stat_info based on anchor learnwares + + Parameters + ---------- + name : str + Name of stat_info + item : Any + Statistical information calculated on anchor learnwares + """ + self.stat_info[name] = item diff --git a/learnware/market/easy/__init__.py b/learnware/market/easy/__init__.py index b5514bd..988a5c6 100644 --- a/learnware/market/easy/__init__.py +++ b/learnware/market/easy/__init__.py @@ -6,10 +6,10 @@ from ...logger import get_module_logger logger = get_module_logger("market_easy") if not is_torch_avaliable(verbose=False): - from .searcher import EasySearcher - from .checker import EasySemanticChecker, EasyStatChecker -else: EasySearcher = None EasySemanticChecker = None EasyStatChecker = None logger.warning("EasySeacher and EasyChecker are skipped because 'torch' is not installed!") +else: + from .searcher import EasySearcher + from .checker import EasySemanticChecker, EasyStatChecker diff --git a/learnware/reuse/__init__.py b/learnware/reuse/__init__.py index 017403c..ccea654 100644 --- a/learnware/reuse/__init__.py +++ b/learnware/reuse/__init__.py @@ -4,7 +4,7 @@ from .utils import is_geatpy_avaliable, is_lightgbm_avaliable logger = get_module_logger("reuse") -if not is_geatpy_avaliable(verbose=True): +if not is_geatpy_avaliable(verbose=False): EnsemblePruningReuser = None logger.warning("EnsemblePruningReuser is skipped due to 'geatpy' is not installed!") else: @@ -16,8 +16,19 @@ if not is_torch_avaliable(verbose=False): else: from .averaging import AveragingReuser -if not is_lightgbm_avaliable(verbose=True) or not is_torch_avaliable(verbose=False): +if not is_lightgbm_avaliable(verbose=False) or not is_torch_avaliable(verbose=False): JobSelectorReuser = None - logger.warning("JobSelectorReuser is skipped due to 'torch' is not installed!") + uninstall_packages = [ + value + for flag, value in zip( + [ + is_lightgbm_avaliable(verbose=False), + is_torch_avaliable(verbose=False), + ], + ["lightgbm", "torch"], + ) + if flag is False + ] + logger.warning(f"JobSelectorReuser is skipped due to {uninstall_packages} is not installed!") else: from .job_selector import JobSelectorReuser diff --git a/learnware/reuse/utils.py b/learnware/reuse/utils.py index f7f6440..d0ab3f8 100644 --- a/learnware/reuse/utils.py +++ b/learnware/reuse/utils.py @@ -3,21 +3,23 @@ from ..logger import get_module_logger logger = get_module_logger("reuse_utils") -def is_geatpy_avaliable(): +def is_geatpy_avaliable(verbose=False): try: import geatpy except ModuleNotFoundError as err: - logger.warning( - "ModuleNotFoundError: geatpy is not installed, please install geatpy (only support python version<3.11)!" - ) + if verbose is True: + logger.warning( + "ModuleNotFoundError: geatpy is not installed, please install geatpy (only support python version<3.11)!" + ) return False return True -def is_lightgbm_avaliable(): +def is_lightgbm_avaliable(verbose=False): try: import lightgbm except ModuleNotFoundError as err: - logger.warning("ModuleNotFoundError: lightgbm is not installed, please install lightgbm!") + if verbose is True: + logger.warning("ModuleNotFoundError: lightgbm is not installed, please install lightgbm!") return False return True diff --git a/learnware/specification/regular/image/__init__.py b/learnware/specification/regular/image/__init__.py index 3dd8022..e883f99 100644 --- a/learnware/specification/regular/image/__init__.py +++ b/learnware/specification/regular/image/__init__.py @@ -1,12 +1,29 @@ -from .utils import is_torch_optimizer_avaliable +from .utils import is_torch_optimizer_avaliable, is_torch_vision_avaliable from ....utils import is_torch_avaliable from ....logger import get_module_logger logger = get_module_logger("regular_image_spec") -if not is_torch_optimizer_avaliable(verbose=True) or not is_torch_avaliable(verbose=False): +if ( + not is_torch_vision_avaliable(verbose=False) + or not is_torch_optimizer_avaliable(verbose=False) + or not is_torch_avaliable(verbose=False) +): RKMEImageSpecification = None - logger.warning("RKMEImageSpecification is skipped because torch or torch-optimizer is not installed!") + uninstall_packages = [ + value + for flag, value in zip( + [ + is_torch_vision_avaliable(verbose=False), + is_torch_optimizer_avaliable(verbose=False), + is_torch_avaliable(verbose=False), + ], + ["torchvision", "torch-optimizer", "torch"], + ) + if flag is False + ] + + logger.warning(f"RKMEImageSpecification is skipped because {uninstall_packages} is not installed!") else: from .rkme import RKMEImageSpecification diff --git a/learnware/specification/regular/image/utils.py b/learnware/specification/regular/image/utils.py index 8c08db8..80c97f2 100644 --- a/learnware/specification/regular/image/utils.py +++ b/learnware/specification/regular/image/utils.py @@ -3,10 +3,21 @@ from ....logger import get_module_logger logger = get_module_logger("regular_image_spec_utils") -def is_torch_optimizer_avaliable(): +def is_torch_optimizer_avaliable(verbose=False): try: import torch_optimizer except ModuleNotFoundError as err: - logger.warning("ModuleNotFoundError: torch_optimizer is not installed, please install pytorch!") + if verbose is True: + logger.warning("ModuleNotFoundError: torch_optimizer is not installed, please install torch_optimizer!") + return False + return True + + +def is_torch_vision_avaliable(verbose=False): + try: + import torchvision + except ModuleNotFoundError as err: + if verbose is True: + logger.warning("ModuleNotFoundError: torchvision is not installed, please install torchvision!") return False return True diff --git a/learnware/specification/regular/text/__init__.py b/learnware/specification/regular/text/__init__.py index 8e996ea..3d8c830 100644 --- a/learnware/specification/regular/text/__init__.py +++ b/learnware/specification/regular/text/__init__.py @@ -3,10 +3,21 @@ from .utils import is_sentence_transformers_avaliable from ....utils import is_torch_avaliable from ....logger import get_module_logger -logger = get_module_logger("regular_table_spec") +logger = get_module_logger("regular_text_spec") -if not is_sentence_transformers_avaliable(verbose=True) or not is_torch_avaliable(verbose=False): +if not is_sentence_transformers_avaliable(verbose=False) or not is_torch_avaliable(verbose=False): RKMETextSpecification = None - logger.warning("RKMETextSpecification is skipped because torch is not installed!") + uninstall_packages = [ + value + for flag, value in zip( + [ + is_sentence_transformers_avaliable(verbose=False), + is_torch_avaliable(verbose=False), + ], + ["sentence_transformers", "torch"], + ) + if flag is False + ] + logger.warning(f"RKMETextSpecification is skipped because {uninstall_packages} is not installed!") else: from .rkme import RKMETextSpecification diff --git a/learnware/specification/regular/text/utils.py b/learnware/specification/regular/text/utils.py index b8401ce..2052c02 100644 --- a/learnware/specification/regular/text/utils.py +++ b/learnware/specification/regular/text/utils.py @@ -3,10 +3,13 @@ from ....logger import get_module_logger logger = get_module_logger("regular_text_spec_utils") -def is_sentence_transformers_avaliable(): +def is_sentence_transformers_avaliable(verbose=False): try: import sentence_transformers except ModuleNotFoundError as err: - logger.warning("ModuleNotFoundError: sentence_transformers is not installed, please install pytorch!") + if verbose is True: + logger.warning( + "ModuleNotFoundError: sentence_transformers is not installed, please install sentence_transformers!" + ) return False return True diff --git a/learnware/utils/import_utils.py b/learnware/utils/import_utils.py index 3a10ea8..9f4406c 100644 --- a/learnware/utils/import_utils.py +++ b/learnware/utils/import_utils.py @@ -3,10 +3,11 @@ from ..logger import get_module_logger logger = get_module_logger("import_utils") -def is_torch_avaliable(): +def is_torch_avaliable(verbose=False): try: import torch except ModuleNotFoundError as err: - logger.warning("ModuleNotFoundError: torch is not installed, please install pytorch!") + if verbose is True: + logger.warning("ModuleNotFoundError: torch is not installed, please install pytorch!") return False return True diff --git a/setup.py b/setup.py index e996025..8f2f812 100644 --- a/setup.py +++ b/setup.py @@ -60,9 +60,7 @@ REQUIRED = [ "joblib>=1.2.0", "pyyaml>=6.0", "fire>=0.3.1", - "lightgbm>=3.3.0", "psutil>=5.9.4", - "torchvision>=0.14.1", "sqlalchemy>=2.0.21", "shortuuid>=1.0.11", "docker>=6.1.3", From 48803599f6647f6b4950f6de0c2541533350eb0a Mon Sep 17 00:00:00 2001 From: Gene Date: Thu, 9 Nov 2023 23:07:10 +0800 Subject: [PATCH 14/15] [FIX] fix bugs in market_module --- learnware/market/module.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/learnware/market/module.py b/learnware/market/module.py index f27d910..866b423 100644 --- a/learnware/market/module.py +++ b/learnware/market/module.py @@ -1,20 +1,24 @@ from .base import LearnwareMarket from .easy import EasyOrganizer, EasySearcher, EasySemanticChecker, EasyStatChecker -MARKET_CONFIG = { - "easy": { - "organizer": EasyOrganizer(), - "searcher": EasySearcher(), - "checker_list": [EasySemanticChecker(), EasyStatChecker()], + +def get_market_config(): + market_config = { + "easy": { + "organizer": EasyOrganizer(), + "searcher": EasySearcher(), + "checker_list": [EasySemanticChecker(), EasyStatChecker()], + } } -} + return market_config def instantiate_learnware_market(market_id="default", name="easy", **kwargs): + market_config = get_market_config() return LearnwareMarket( market_id=market_id, - organizer=MARKET_CONFIG[name]["organizer"], - searcher=MARKET_CONFIG[name]["searcher"], - checker_list=MARKET_CONFIG[name]["checker_list"], + organizer=market_config[name]["organizer"], + searcher=market_config[name]["searcher"], + checker_list=market_config[name]["checker_list"], **kwargs ) From b4b961d0dc262b3ecc7df860c8e3bdda4280d385 Mon Sep 17 00:00:00 2001 From: bxdd Date: Thu, 9 Nov 2023 23:12:58 +0800 Subject: [PATCH 15/15] [MNT] update version --- learnware/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/learnware/__init__.py b/learnware/__init__.py index bbbadd2..b140a86 100644 --- a/learnware/__init__.py +++ b/learnware/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.1.1.99" +__version__ = "0.1.2.99" import os from .logger import get_module_logger