beimingwu
/
learnware
Not watched
1
0
Fork 0
Code
Releases 1 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

Merge branch 'feature/hetero' of https://github.com/Learnware-LAMDA/Learnware into feature/hetero

tags/v0.3.2
liuht 2 years ago
parent
664463ca57 969b2e9dfd
commit
dcf7516a08
34 changed files with 750 additions and 344 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +2
    -2
      docs/workflow/submit.rst
  2. +3
    -4
      examples/dataset_m5_workflow/main.py
  3. +3
    -5
      examples/dataset_pfs_workflow/main.py
  4. +4
    -9
      examples/dataset_text_workflow/main.py
  5. +4
    -5
      examples/workflow_by_code/main.py
  6. +12
    -11
      learnware/client/container.py
  7. +4
    -8
      learnware/client/learnware_client.py
  8. +3
    -2
      learnware/client/package_utils.py
  9. +15
    -3
      learnware/client/utils.py
  10. +3
    -1
      learnware/learnware/__init__.py
  11. +30
    -7
      learnware/market/base.py
  12. +9
    -7
      learnware/market/classes.py
  13. +1
    -1
      learnware/market/easy.py
  14. +13
    -11
      learnware/market/easy2/checker.py
  15. +1
    -2
      learnware/market/easy2/database_ops.py
  16. +41
    -6
      learnware/market/easy2/organizer.py
  17. +2
    -2
      learnware/market/easy2/searcher.py
  18. +1
    -2
      learnware/market/utils.py
  19. +3
    -3
      learnware/reuse/job_selector.py
  20. +1
    -1
      learnware/specification/__init__.py
  21. +223
    -0
      learnware/specification/module.py
  22. +1
    -212
      learnware/specification/utils.py
  23. +1
    -1
      tests/test_learnware_client/test_all_learnware.py
  24. +60
    -26
      tests/test_market/test_easy.py
  25. +1
    -0
      tests/test_market/test_hetero_market/example_learnwares/config.py
  26. +22
    -0
      tests/test_market/test_hetero_market/example_learnwares/example_learnware_0/__init__.py
  27. +8
    -0
      tests/test_market/test_hetero_market/example_learnwares/example_learnware_0/learnware.yaml
  28. +1
    -0
      tests/test_market/test_hetero_market/example_learnwares/example_learnware_0/requirements.txt
  29. +22
    -0
      tests/test_market/test_hetero_market/example_learnwares/example_learnware_1/__init__.py
  30. +8
    -0
      tests/test_market/test_hetero_market/example_learnwares/example_learnware_1/learnware.yaml
  31. +1
    -0
      tests/test_market/test_hetero_market/example_learnwares/example_learnware_1/requirements.txt
  32. +236
    -0
      tests/test_market/test_hetero_market/test_hetero.py
  33. +2
    -3
      tests/test_specification/test_rkme.py
  34. +9
    -10
      tests/test_workflow/test_workflow.py

+ 2
- 2
docs/workflow/submit.rst View File

@@ -80,10 +80,10 @@ the following code snippet offers guidance on how to construct and store the RKM

.. code-block:: python
import learnware.specification as specification
from learnware.specification import generate_rkme_spec
# generate rkme specification for digits dataset
spec = specification.utils.generate_rkme_spec(X=data_X)
spec = generate_rkme_spec(X=data_X)
spec.save("stat.json")

Significantly, the RKME generation process is entirely conducted on your local machine, without any involvement of cloud services,


+ 3
- 4
examples/dataset_m5_workflow/main.py View File

@@ -9,9 +9,8 @@ from shutil import copyfile, rmtree
import learnware
from learnware.market import EasyMarket, BaseUserInfo
from learnware.market import database_ops
from learnware.learnware import Learnware
from learnware.reuse import JobSelectorReuser, AveragingReuser
import learnware.specification as specification
from learnware.specification import generate_rkme_spec
from m5 import DataLoader
from learnware.logger import get_module_logger

@@ -88,7 +87,7 @@ class M5DatasetWorkflow:
for idx in tqdm(idx_list):
train_x, train_y, test_x, test_y = m5.get_idx_data(idx)
st = time.time()
spec = specification.utils.generate_rkme_spec(X=train_x, gamma=0.1, cuda_idx=0)
spec = generate_rkme_spec(X=train_x, gamma=0.1, cuda_idx=0)
ed = time.time()
logger.info("Stat spec generated in %.3f s" % (ed - st))

@@ -140,7 +139,7 @@ class M5DatasetWorkflow:

for idx in idx_list:
train_x, train_y, test_x, test_y = m5.get_idx_data(idx)
user_spec = specification.utils.generate_rkme_spec(X=test_x, gamma=0.1, cuda_idx=0)
user_spec = generate_rkme_spec(X=test_x, gamma=0.1, cuda_idx=0)
user_spec_path = f"./user_spec/user_{idx}.json"
user_spec.save(user_spec_path)



+ 3
- 5
examples/dataset_pfs_workflow/main.py View File

@@ -8,10 +8,8 @@ from shutil import copyfile, rmtree

import learnware
from learnware.market import EasyMarket, BaseUserInfo
from learnware.market import database_ops
from learnware.learnware import Learnware
from learnware.reuse import JobSelectorReuser, AveragingReuser
import learnware.specification as specification
from learnware.specification import generate_rkme_spec
from pfs import Dataloader
from learnware.logger import get_module_logger

@@ -86,7 +84,7 @@ class PFSDatasetWorkflow:
for idx in tqdm(idx_list):
train_x, train_y, test_x, test_y = pfs.get_idx_data(idx)
st = time.time()
spec = specification.utils.generate_rkme_spec(X=train_x, gamma=0.1, cuda_idx=0)
spec = generate_rkme_spec(X=train_x, gamma=0.1, cuda_idx=0)
ed = time.time()
logger.info("Stat spec generated in %.3f s" % (ed - st))

@@ -138,7 +136,7 @@ class PFSDatasetWorkflow:

for idx in idx_list:
train_x, train_y, test_x, test_y = pfs.get_idx_data(idx)
user_spec = specification.utils.generate_rkme_spec(X=test_x, gamma=0.1, cuda_idx=0)
user_spec = generate_rkme_spec(X=test_x, gamma=0.1, cuda_idx=0)
user_spec_path = f"./user_spec/user_{idx}.json"
user_spec.save(user_spec_path)



+ 4
- 9
examples/dataset_text_workflow/main.py View File

@@ -10,9 +10,7 @@ import time
import pickle

from learnware.market import instantiate_learnware_market, BaseUserInfo
from learnware.market import database_ops
from learnware.learnware import Learnware
import learnware.specification as specification
from learnware.specification import RKMETextSpecification
from learnware.logger import get_module_logger

from shutil import copyfile, rmtree
@@ -99,8 +97,7 @@ def prepare_learnware(data_path, model_path, init_file_path, yaml_path, save_roo
semantic_spec = semantic_specs[0]

st = time.time()
# user_spec = specification.utils.generate_rkme_spec(X=X, gamma=0.1, cuda_idx=0)
user_spec = specification.RKMETextSpecification()
user_spec = RKMETextSpecification()
user_spec.generate_stat_spec_from_data(X=X)
ed = time.time()
logger.info("Stat spec generated in %.3f s" % (ed - st))
@@ -163,10 +160,8 @@ def test_search(gamma=0.1, load_market=True):
user_data = pickle.load(f)
with open(user_label_path, "rb") as f:
user_label = pickle.load(f)
# user_data = np.load(user_data_path)
# user_label = np.load(user_label_path)
# user_stat_spec = specification.utils.generate_rkme_spec(X=user_data, gamma=gamma, cuda_idx=0)
user_stat_spec = specification.RKMETextSpecification()

user_stat_spec = RKMETextSpecification()
user_stat_spec.generate_stat_spec_from_data(X=user_data)
user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={"RKMETextSpecification": user_stat_spec})
logger.info("Searching Market for user: %d" % (i))


+ 4
- 5
examples/workflow_by_code/main.py View File

@@ -12,8 +12,7 @@ from shutil import copyfile, rmtree
import learnware
from learnware.market import EasyMarket, BaseUserInfo
from learnware.reuse import JobSelectorReuser, AveragingReuser
import learnware.specification as specification
from learnware.utils import get_module_by_module_path
from learnware.specification import generate_rkme_spec, RKMETableSpecification

curr_root = os.path.dirname(os.path.abspath(__file__))

@@ -54,7 +53,7 @@ class LearnwareMarketWorkflow:

joblib.dump(clf, os.path.join(dir_path, "svm.pkl"))

spec = specification.utils.generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0)
spec = generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0)
spec.save(os.path.join(dir_path, "svm.json"))

init_file = os.path.join(dir_path, "__init__.py")
@@ -148,7 +147,7 @@ class LearnwareMarketWorkflow:
with zipfile.ZipFile(zip_path, "r") as zip_obj:
zip_obj.extractall(path=unzip_dir)

user_spec = specification.RKMETableSpecification()
user_spec = RKMETableSpecification()
user_spec.load(os.path.join(unzip_dir, "svm.json"))
user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={"RKMETableSpecification": user_spec})
(
@@ -174,7 +173,7 @@ class LearnwareMarketWorkflow:
X, y = load_digits(return_X_y=True)
_, data_X, _, data_y = train_test_split(X, y, test_size=0.3, shuffle=True)

stat_spec = specification.utils.generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0)
stat_spec = generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0)
user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={"RKMETableSpecification": stat_spec})

_, _, _, mixture_learnware_list = easy_market.search_learnware(user_info)


+ 12
- 11
learnware/client/container.py View File

@@ -500,6 +500,7 @@ class LearnwaresContainer:
learnwares: Union[List[Learnware], Learnware],
cleanup=True,
mode="conda",
ignore_error=True,
):
"""The initializaiton method for base reuser

@@ -519,6 +520,7 @@ class LearnwaresContainer:
assert self.mode in {"conda", "docker"}, f"mode must be in ['conda', 'docker'], should not be {self.mode}"
self.learnware_list = learnwares
self.cleanup = cleanup
self.ignore_error = ignore_error

def __enter__(self):
if self.mode == "conda":
@@ -548,7 +550,7 @@ class LearnwaresContainer:

model_list = [_learnware.get_model() for _learnware in self.learnware_containers]
with ThreadPoolExecutor(max_workers=max(os.cpu_count() // 2, 1)) as executor:
results = executor.map(self._initialize_model_container, model_list)
results = executor.map(self._initialize_model_container, model_list, [self.ignore_error] * len(model_list))
self.results = list(results)

if sum(self.results) < len(self.learnware_list):
@@ -556,11 +558,6 @@ class LearnwaresContainer:
f"{len(self.learnware_list) - sum(results)} of {len(self.learnware_list)} learnwares init failed! This learnware will be ignored"
)

# if not self.cleanup and self.mode == "docker":
# _model_docker_container = self.learnware_containers[0].get_model()
# _model_docker_container.cleanup_flag = True
# atexit.register(_model_docker_container.remove_env)

return self

def __exit__(self, exc_type, exc_val, exc_tb):
@@ -572,7 +569,7 @@ class LearnwaresContainer:

model_list = [_learnware.get_model() for _learnware in self.learnware_containers]
with ThreadPoolExecutor(max_workers=max(os.cpu_count() // 2, 1)) as executor:
executor.map(self._destroy_model_container, model_list)
executor.map(self._destroy_model_container, model_list, [self.ignore_error] * len(model_list))

self.learnware_containers = None
self.results = None
@@ -581,20 +578,24 @@ class LearnwaresContainer:
ModelDockerContainer._destroy_docker_container(self._docker_container)

@staticmethod
def _initialize_model_container(model: ModelCondaContainer):
def _initialize_model_container(model: ModelCondaContainer, ignore_error=True):
try:
model.init_and_setup_env()
except Exception as err:
logger.error(f"build env {model.conda_env} failed due to {err}")
if not ignore_error:
raise err
logger.warning(f"build env {model.conda_env} failed due to {err}")
return False
return True

@staticmethod
def _destroy_model_container(model: ModelCondaContainer):
def _destroy_model_container(model: ModelCondaContainer, ignore_error=True):
try:
model.remove_env()
except Exception as err:
logger.error(f"remove env {model.conda_env} failed due to {err}")
if not ignore_error:
raise err
logger.warning(f"remove env {model.conda_env} failed due to {err}")
return False
return True



+ 4
- 8
learnware/client/learnware_client.py View File

@@ -312,7 +312,7 @@ class LearnwareClient:
tempdir = self.tempdir_list[-1].name
zip_path = os.path.join(tempdir, f"{str(uuid.uuid4())}.zip")
self.download_learnware(_learnware_id, zip_path)
return zip_path, _get_learnware_by_path(zip_path, tempdir=tempdir)
return _get_learnware_by_path(zip_path, tempdir=tempdir)

def _get_learnware_by_path(_learnware_zippath, tempdir=None):
if tempdir is None:
@@ -342,16 +342,13 @@ class LearnwareClient:
return learnware.get_learnware_from_dirpath(learnware_id, semantic_specification, tempdir)

learnware_list = []
zip_paths = []
if learnware_path is not None:
if isinstance(learnware_path, str):
zip_paths = [learnware_path]
elif isinstance(learnware_path, list):
zip_paths = learnware_path
zip_paths = [learnware_path] if isinstance(learnware_path, str) else learnware_path

for zip_path in zip_paths:
learnware_obj = _get_learnware_by_path(zip_path)
learnware_list.append(learnware_obj)

elif learnware_id is not None:
if isinstance(learnware_id, str):
id_list = [learnware_id]
@@ -359,8 +356,7 @@ class LearnwareClient:
id_list = learnware_id

for idx in id_list:
zip_path, learnware_obj = _get_learnware_by_id(idx)
zip_paths.append(zip_path)
learnware_obj = _get_learnware_by_id(idx)
learnware_list.append(learnware_obj)

if runnable_option is not None:


+ 3
- 2
learnware/client/package_utils.py View File

@@ -4,6 +4,7 @@ import yaml
import tempfile
import subprocess
from typing import List, Tuple
from . import utils


from ..logger import get_module_logger
@@ -15,7 +16,7 @@ def try_to_run(args, timeout=5, retry=5):
sucess = False
for i in range(retry):
try:
subprocess.check_call(args=args, timeout=timeout, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
utils.system_execute(args=args, timeout=timeout, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
sucess = True
break
except subprocess.TimeoutExpired as e:
@@ -120,7 +121,7 @@ def filter_nonexist_conda_packages(packages: list) -> Tuple[List[str], List[str]

if not any(package.startswith("python=") for package in exist_packages):
exist_packages = ["python=3.8"] + exist_packages
return exist_packages, nonexist_packages
else:
return packages, []


+ 15
- 3
learnware/client/utils.py View File

@@ -9,17 +9,29 @@ from .package_utils import filter_nonexist_conda_packages_file, filter_nonexist_
logger = get_module_logger(module_name="client_utils")


def system_execute(args, timeout=None):
com_process = subprocess.run(args, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, timeout=timeout)
def system_execute(args, timeout=None, env=None, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE):
if env is None:
env = os.environ.copy()
pass

if isinstance(args, str):
pass
else:
args = " ".join(args)
pass

com_process = subprocess.run(args, stdout=stdout, stderr=stderr, timeout=timeout, env=env, shell=True)

try:
com_process.check_returncode()
except subprocess.CalledProcessError as err:
logger.error(f"System Execute Error: {com_process.stderr.decode()}")
logger.warning(f"System Execute Error: {com_process.stderr.decode()}")
raise err


def remove_enviroment(conda_env):
system_execute(args=["conda", "env", "remove", "-n", f"{conda_env}"])
logger.info(f"The learnware conda env [{conda_env}] is removed.")


def install_environment(learnware_dirpath, conda_env):


+ 3
- 1
learnware/learnware/__init__.py View File

@@ -12,7 +12,7 @@ from ..config import C
logger = get_module_logger("learnware.learnware")


def get_learnware_from_dirpath(id: str, semantic_spec: dict, learnware_dirpath) -> Learnware:
def get_learnware_from_dirpath(id: str, semantic_spec: dict, learnware_dirpath, ignore_error=True) -> Learnware:
"""Get the learnware object from dirpath, and provide the manage interface tor Learnware class

Parameters
@@ -67,6 +67,8 @@ def get_learnware_from_dirpath(id: str, semantic_spec: dict, learnware_dirpath)
learnware_spec.update_semantic_spec(copy.deepcopy(semantic_spec))

except Exception as e:
if not ignore_error:
raise e
logger.warning(f"Load Learnware {id} failed! Due to {repr(e)}")
return None



+ 30
- 7
learnware/market/base.py View File

@@ -1,5 +1,6 @@
from __future__ import annotations

import traceback
import zipfile
import tempfile
from typing import Tuple, Any, List, Union
@@ -81,8 +82,6 @@ class LearnwareMarket:
pending_learnware = get_learnware_from_dirpath(
id="pending", semantic_spec=semantic_spec, learnware_dirpath=tempdir
)
checker_names = list(self.learnware_checker.keys()) if checker_names is None else checker_names

for name in checker_names:
checker = self.learnware_checker[name]
check_status = checker(pending_learnware)
@@ -92,6 +91,7 @@ class LearnwareMarket:
return BaseChecker.INVALID_LEARNWARE
return final_status
except Exception as err:
traceback.print_exc()
logger.warning(f"Check learnware failed! Due to {err}.")
return BaseChecker.INVALID_LEARNWARE

@@ -115,6 +115,7 @@ class LearnwareMarket:
- str indicating model_id
- int indicating the final learnware check_status
"""
checker_names = list(self.learnware_checker.keys()) if checker_names is None else checker_names
check_status = self.check_learnware(zip_path, semantic_spec, checker_names)
return self.learnware_organizer.add_learnware(
zip_path=zip_path, semantic_spec=semantic_spec, check_status=check_status, **kwargs
@@ -172,12 +173,13 @@ class LearnwareMarket:
int
The final learnware check_status.
"""
zip_path = self.get_learnware_path_by_ids(id) if zip_path is None else zip_path
zip_path = self.get_learnware_zip_path_by_ids(id) if zip_path is None else zip_path
semantic_spec = (
self.get_learnware_by_ids(id).get_specification().get_semantic_spec()
if semantic_spec is None
else semantic_spec
)
checker_names = list(self.learnware_checker.keys()) if checker_names is None else checker_names
update_status = self.check_learnware(zip_path, semantic_spec, checker_names)
check_status = (
update_status if check_status is None or update_status == BaseChecker.INVALID_LEARNWARE else check_status
@@ -223,8 +225,11 @@ class LearnwareMarket:
"""
return self.learnware_organizer.get_learnwares(top, check_status, **kwargs)

def get_learnware_path_by_ids(self, ids: Union[str, List[str]], **kwargs) -> Union[Learnware, List[Learnware]]:
return self.learnware_organizer.get_learnware_path_by_ids(ids, **kwargs)
def get_learnware_zip_path_by_ids(self, ids: Union[str, List[str]], **kwargs) -> Union[Learnware, List[Learnware]]:
return self.learnware_organizer.get_learnware_zip_path_by_ids(ids, **kwargs)

def get_learnware_dir_path_by_ids(self, ids: Union[str, List[str]], **kwargs) -> Union[Learnware, List[Learnware]]:
return self.learnware_organizer.get_learnware_dir_path_by_ids(ids, **kwargs)

def get_learnware_by_ids(self, id: Union[str, List[str]], **kwargs) -> Union[Learnware, List[Learnware]]:
return self.learnware_organizer.get_learnware_by_ids(id, **kwargs)
@@ -331,7 +336,7 @@ class BaseOrganizer:
"""
raise NotImplementedError("get_learnware_by_ids is not implemented in BaseOrganizer")

def get_learnware_path_by_ids(self, ids: Union[str, List[str]]) -> Union[Learnware, List[Learnware]]:
def get_learnware_zip_path_by_ids(self, ids: Union[str, List[str]]) -> Union[Learnware, List[Learnware]]:
"""Get Zipped Learnware file by id

Parameters
@@ -347,7 +352,25 @@ class BaseOrganizer:
Return the path for target learnware or list of path.
None for Learnware NOT Found.
"""
raise NotImplementedError("get_learnware_path_by_ids is not implemented in BaseOrganizer")
raise NotImplementedError("get_learnware_zip_path_by_ids is not implemented in BaseOrganizer")

def get_learnware_dir_path_by_ids(self, ids: Union[str, List[str]]) -> Union[Learnware, List[Learnware]]:
"""Get Learnware dir path by id

Parameters
----------
ids : Union[str, List[str]]
Give a id or a list of ids
str: id of targer learware
List[str]: A list of ids of target learnwares

Returns
-------
Union[Learnware, List[Learnware]]
Return the dir path for target learnware or list of path.
None for Learnware NOT Found.
"""
raise NotImplementedError("get_learnware_dir_path_by_ids is not implemented in BaseOrganizer")

def get_learnware_ids(self, top: int = None, check_status: int = None) -> List[str]:
"""get the list of learnware ids


+ 9
- 7
learnware/market/classes.py View File

@@ -1,3 +1,4 @@
import traceback
from .base import BaseChecker
from ..learnware import Learnware
from ..client.container import LearnwaresContainer
@@ -12,11 +13,12 @@ class CondaChecker(BaseChecker):
super(CondaChecker, self).__init__(**kwargs)

def __call__(self, learnware: Learnware) -> int:
with LearnwaresContainer(learnware) as env_container:
if not all(env_container.get_learnware_flags()):
logger.warning(f"Conda Checker failed due to installed learnware failed")
return BaseChecker.INVALID_LEARNWARE
learnwares = env_container.get_learnwares_with_container()
check_status = self.inner_checker(learnwares[0])

try:
with LearnwaresContainer(learnware, ignore_error=False) as env_container:
learnwares = env_container.get_learnwares_with_container()
check_status = self.inner_checker(learnwares[0])
except Exception as e:
traceback.print_exc()
logger.warning(f"Conda Checker failed due to installed learnware failed and {e}")
return BaseChecker.INVALID_LEARNWARE
return check_status

+ 1
- 1
learnware/market/easy.py View File

@@ -926,7 +926,7 @@ class EasyMarket(LearnwareMarket):
logger.warning("Learnware ID '%s' NOT Found!" % (ids))
return None

def get_learnware_path_by_ids(self, ids: Union[str, List[str]]) -> Union[Learnware, List[Learnware]]:
def get_learnware_zip_path_by_ids(self, ids: Union[str, List[str]]) -> Union[Learnware, List[Learnware]]:
"""Get Zipped Learnware file by id

Parameters


+ 13
- 11
learnware/market/easy2/checker.py View File

@@ -95,14 +95,13 @@ class EasyStatChecker(BaseChecker):
# Check input shape
input_shape = learnware_model.input_shape

## WHY: why write this?
if semantic_spec["Data"]["Values"][0] == "Table" and input_shape != (
int(semantic_spec["Input"]["Dimension"]),
):
logger.warning("input shapes of model and semantic specifications are different")
return self.INVALID_LEARNWARE

spec_type = parse_specification_type(learnware.get_specification())
spec_type = parse_specification_type(learnware.get_specification().stat_spec)
if spec_type is None:
logger.warning(f"No valid specification is found in stat spec {spec_type}")
return self.INVALID_LEARNWARE
@@ -119,13 +118,12 @@ class EasyStatChecker(BaseChecker):
inputs = np.random.randint(0, 255, size=(10, *input_shape))
else:
raise ValueError(f"not supported spec type for spec_type = {spec_type}")
outputs = learnware.predict(inputs)
# Check output
if outputs.ndim == 1:
outputs = outputs.reshape(-1, 1)

if outputs.shape[1:] != learnware_model.output_shape:
logger.warning(f"The learnware [{learnware.id}] output dimention mismatch!")
# Check output
try:
outputs = learnware.predict(inputs)
except Exception:
logger.warning(f"learnware {learnware} prediction method is not valid!")
return self.INVALID_LEARNWARE

if semantic_spec["Task"]["Values"][0] in ("Classification", "Regression", "Feature Extraction"):
@@ -136,11 +134,15 @@ class EasyStatChecker(BaseChecker):
logger.warning(f"The learnware [{learnware.id}] output must be np.ndarray or torch.Tensor!")
return self.INVALID_LEARNWARE

if outputs.ndim == 1:
outputs = outputs.reshape(-1, 1)
# Check output shape
if outputs[0].shape != learnware_model.output_shape or learnware_model.output_shape != int(
semantic_spec["Output"]["Dimension"]
if outputs[0].shape != learnware_model.output_shape or learnware_model.output_shape != (
int(semantic_spec["Output"]["Dimension"]),
):
logger.warning(f"The learnware [{learnware.id}] output dimention mismatch!")
logger.warning(
f"The learnware [{learnware.id}] output dimension mismatch!, where pred_shape={outputs[0].shape}, model_shape={learnware_model.output_shape}, semantic_shape={(int(semantic_spec['Output']['Dimension']), )}"
)
return self.INVALID_LEARNWARE

except Exception as e:


+ 1
- 2
learnware/market/easy2/database_ops.py View File

@@ -166,10 +166,9 @@ class DatabaseOperations(object):
# assert new_learnware is not None
zip_list[id] = zip_path
folder_list[id] = folder_path
use_flags[id] = use_flag
use_flags[id] = int(use_flag)
max_count = max(max_count, int(id))
pass

return learnware_list, zip_list, folder_list, use_flags, max_count + 1
pass



+ 41
- 6
learnware/market/easy2/organizer.py View File

@@ -256,7 +256,7 @@ class EasyOrganizer(BaseOrganizer):
logger.warning("Learnware ID '%s' NOT Found!" % (ids))
return None

def get_learnware_path_by_ids(self, ids: Union[str, List[str]]) -> Union[Learnware, List[Learnware]]:
def get_learnware_zip_path_by_ids(self, ids: Union[str, List[str]]) -> Union[Learnware, List[Learnware]]:
"""Get Zipped Learnware file by id

Parameters
@@ -288,6 +288,38 @@ class EasyOrganizer(BaseOrganizer):
logger.warning("Learnware ID '%s' NOT Found!" % (ids))
return None

def get_learnware_dir_path_by_ids(self, ids: Union[str, List[str]]) -> Union[Learnware, List[Learnware]]:
"""Get Learnware dir path by id

Parameters
----------
ids : Union[str, List[str]]
Give a id or a list of ids
str: id of targer learware
List[str]: A list of ids of target learnwares

Returns
-------
Union[Learnware, List[Learnware]]
Return the dir path for target learnware or list of path.
None for Learnware NOT Found.
"""
if isinstance(ids, list):
ret = []
for id in ids:
if id in self.learnware_folder_list:
ret.append(self.learnware_folder_list[id])
else:
logger.warning("Learnware ID '%s' NOT Found!" % (id))
ret.append(None)
return ret
else:
try:
return self.learnware_folder_list[ids]
except:
logger.warning("Learnware ID '%s' NOT Found!" % (ids))
return None

def get_learnware_ids(self, top: int = None, check_status: int = None) -> List[str]:
"""Get learnware ids

@@ -305,11 +337,14 @@ class EasyOrganizer(BaseOrganizer):
Learnware ids
"""
if check_status is None:
filtered_ids = self.use_flags.keys()
elif check_status is True:
filtered_ids = [key for key, value in self.use_flags.items() if value == BaseChecker.USABLE_LEARWARE]
elif check_status is False:
filtered_ids = [key for key, value in self.use_flags.items() if value == BaseChecker.NONUSABLE_LEARNWARE]
filtered_ids = list(self.use_flags.keys())
elif check_status in [BaseChecker.NONUSABLE_LEARNWARE, BaseChecker.USABLE_LEARWARE]:
filtered_ids = [key for key, value in self.use_flags.items() if value == check_status]
else:
logger.warning(
f"check_status must be in [{BaseChecker.NONUSABLE_LEARNWARE}, {BaseChecker.USABLE_LEARWARE}]!"
)
return None

if top is None:
return filtered_ids


+ 2
- 2
learnware/market/easy2/searcher.py View File

@@ -565,7 +565,7 @@ class EasyStatSearcher(BaseSearcher):
max_search_num: int = 5,
search_method: str = "greedy",
) -> Tuple[List[float], List[Learnware], float, List[Learnware]]:
self.stat_spec_type = parse_specification_type(stat_spec=user_info.stat_info)
self.stat_spec_type = parse_specification_type(stat_specs=user_info.stat_info)
if self.stat_spec_type is None:
raise KeyError("No supported stat specification is given in the user info")

@@ -646,7 +646,7 @@ class EasySearcher(BaseSearcher):
if len(learnware_list) == 0:
return [], [], 0.0, []

if parse_specification_type(stat_spec=user_info.stat_info) is not None:
if parse_specification_type(stat_specs=user_info.stat_info) is not None:
return self.stat_searcher(learnware_list, user_info, max_search_num, search_method)
else:
return None, learnware_list, 0.0, None

+ 1
- 2
learnware/market/utils.py View File

@@ -2,9 +2,8 @@ from ..specification import Specification


def parse_specification_type(
stat_spec: Specification, spec_list=["RKMETableSpecification", "RKMETextSpecification", "RKMEImageSpecification"]
stat_specs: dict, spec_list=["RKMETableSpecification", "RKMETextSpecification", "RKMEImageSpecification"]
):
stat_specs = stat_spec.stat_spec
for spec in spec_list:
if spec in stat_specs:
return spec


+ 3
- 3
learnware/reuse/job_selector.py View File

@@ -11,7 +11,7 @@ from .base import BaseReuser
from ..market.utils import parse_specification_type
from ..learnware import Learnware
from ..specification import RKMETableSpecification, RKMETextSpecification
from ..specification.utils import generate_rkme_spec
from ..specification import generate_rkme_spec
from ..logger import get_module_logger

logger = get_module_logger("job_selector_reuse")
@@ -49,7 +49,7 @@ class JobSelectorReuser(BaseReuser):
"""
raw_user_data = user_data
if isinstance(user_data[0], str):
stat_spec_type = parse_specification_type(self.learnware_list[0].get_specification())
stat_spec_type = parse_specification_type(self.learnware_list[0].get_specification().stat_spec)
assert (
stat_spec_type == "RKMETextSpecification"
), "stat_spec_type must be 'RKMETextSpecification' when user data is the List of string."
@@ -97,7 +97,7 @@ class JobSelectorReuser(BaseReuser):
user_data_num = len(user_data)
return np.array([0] * user_data_num)
else:
stat_spec_type = parse_specification_type(self.learnware_list[0].get_specification())
stat_spec_type = parse_specification_type(self.learnware_list[0].get_specification().stat_spec)
learnware_rkme_spec_list = [
learnware.specification.get_stat_spec_by_name(stat_spec_type) for learnware in self.learnware_list
]


+ 1
- 1
learnware/specification/__init__.py View File

@@ -1,4 +1,4 @@
from .utils import generate_stat_spec, generate_rkme_spec, generate_rkme_image_spec
from .module import generate_stat_spec, generate_rkme_spec, generate_rkme_image_spec, generate_rkme_text_spec
from .base import Specification, BaseStatSpecification
from .regular import (
RegularStatsSpecification,


+ 223
- 0
learnware/specification/module.py View File

@@ -0,0 +1,223 @@
import torch
import numpy as np
import pandas as pd
from typing import Union, List

from .utils import convert_to_numpy
from .base import BaseStatSpecification
from .regular import RKMETableSpecification, RKMEImageSpecification, RKMETextSpecification
from ..config import C


def generate_rkme_spec(
X: Union[np.ndarray, pd.DataFrame, torch.Tensor],
gamma: float = 0.1,
reduced_set_size: int = 100,
step_size: float = 0.1,
steps: int = 3,
nonnegative_beta: bool = True,
reduce: bool = True,
cuda_idx: int = None,
) -> RKMETableSpecification:
"""
Interface for users to generate Reduced Kernel Mean Embedding (RKME) specification.
Return a RKMETableSpecification object, use .save() method to save as json file.

Parameters
----------
X : np.ndarray, pd.DataFrame, or torch.Tensor
Raw data in np.ndarray, pd.DataFrame, or torch.Tensor format.
The shape of X:
First dimension represents the number of samples (data points).
The remaining dimensions represent the dimensions (features) of each sample.
For example, if X has shape (100, 3), it means there are 100 samples, and each sample has 3 features.
gamma : float
Bandwidth in gaussian kernel, by default 0.1.
reduced_set_size : int
Size of the construced reduced set.
step_size : float
Step size for gradient descent in the iterative optimization.
steps : int
Total rounds in the iterative optimization.
nonnegative_beta : bool, optional
True if weights for the reduced set are intended to be kept non-negative, by default False.
reduce : bool, optional
Whether shrink original data to a smaller set, by default True
cuda_idx : int
A flag indicating whether use CUDA during RKME computation. -1 indicates CUDA not used.
None indicates that CUDA is automatically selected.

Returns
-------
RKMETableSpecification
A RKMETableSpecification object
"""
# Convert data type
X = convert_to_numpy(X)
X = np.ascontiguousarray(X).astype(np.float32)

# Check reduced_set_size
max_reduced_set_size = C.max_reduced_set_size
if reduced_set_size * X[0].size > max_reduced_set_size:
reduced_set_size = max(20, max_reduced_set_size // X[0].size)

# Check cuda_idx
if not torch.cuda.is_available() or cuda_idx == -1:
cuda_idx = -1
else:
num_cuda_devices = torch.cuda.device_count()
if cuda_idx is None or not (cuda_idx >= 0 and cuda_idx < num_cuda_devices):
cuda_idx = 0

# Generate rkme spec
rkme_spec = RKMETableSpecification(gamma=gamma, cuda_idx=cuda_idx)
rkme_spec.generate_stat_spec_from_data(X, reduced_set_size, step_size, steps, nonnegative_beta, reduce)
return rkme_spec


def generate_rkme_image_spec(
X: Union[np.ndarray, torch.Tensor],
reduced_set_size: int = 50,
step_size: float = 0.01,
steps: int = 100,
resize: bool = True,
nonnegative_beta: bool = True,
reduce: bool = True,
verbose: bool = True,
cuda_idx: int = None,
) -> RKMEImageSpecification:
"""
Interface for users to generate Reduced Kernel Mean Embedding (RKME) specification for Image.
Return a RKMEImageSpecification object, use .save() method to save as json file.

Parameters
----------
X : np.ndarray, or torch.Tensor
Raw data in np.ndarray, or torch.Tensor format.
The shape of X: [N, C, H, W]
N: Number of images.
C: Number of channels.
H: Height of images.
W: Width of images.s
For example, if X has shape (100, 3, 32, 32), it means there are 100 samples, and each sample is a 3-channel (RGB) image of size 32x32.
reduced_set_size : int
Size of the construced reduced set.
step_size : float
Step size for gradient descent in the iterative optimization.
steps : int
Total rounds in the iterative optimization.
resize : bool
Whether to scale the image to the requested size, by default True.
nonnegative_beta : bool, optional
True if weights for the reduced set are intended to be kept non-negative, by default False.
reduce : bool, optional
Whether shrink original data to a smaller set, by default True
cuda_idx : int
A flag indicating whether use CUDA during RKME computation. -1 indicates CUDA not used.
None indicates that CUDA is automatically selected.
verbose : bool, optional
Whether to print training progress, by default True

Returns
-------
RKMEImageSpecification
A RKMEImageSpecification object
"""

# Check cuda_idx
if not torch.cuda.is_available() or cuda_idx == -1:
cuda_idx = -1
else:
num_cuda_devices = torch.cuda.device_count()
if cuda_idx is None or not (0 <= cuda_idx < num_cuda_devices):
cuda_idx = 0

# Generate rkme spec
rkme_image_spec = RKMEImageSpecification(cuda_idx=cuda_idx)
rkme_image_spec.generate_stat_spec_from_data(
X, reduced_set_size, step_size, steps, resize, nonnegative_beta, reduce, verbose
)
return rkme_image_spec


def generate_rkme_text_spec(
X: List[str],
gamma: float = 0.1,
reduced_set_size: int = 100,
step_size: float = 0.1,
steps: int = 3,
nonnegative_beta: bool = True,
reduce: bool = True,
cuda_idx: int = None,
) -> RKMETextSpecification:
"""
Interface for users to generate Reduced Kernel Mean Embedding (RKME) specification for Text.
Return a RKMETextSpecification object, use .save() method to save as json file.

Parameters
----------
X : List[str]
Raw data of text.
gamma : float
Bandwidth in gaussian kernel, by default 0.1.
reduced_set_size : int
Size of the construced reduced set.
step_size : float
Step size for gradient descent in the iterative optimization.
steps : int
Total rounds in the iterative optimization.
nonnegative_beta : bool, optional
True if weights for the reduced set are intended to be kept non-negative, by default False.
reduce : bool, optional
Whether shrink original data to a smaller set, by default True
cuda_idx : int
A flag indicating whether use CUDA during RKME computation. -1 indicates CUDA not used.
None indicates that CUDA is automatically selected.

Returns
-------
RKMETextSpecification
A RKMETextSpecification object
"""
# Check input type
if not isinstance(X, list) or not all(isinstance(item, str) for item in X):
raise TypeError("Input data must be a list of strings.")

# Check cuda_idx
if not torch.cuda.is_available() or cuda_idx == -1:
cuda_idx = -1
else:
num_cuda_devices = torch.cuda.device_count()
if cuda_idx is None or not (cuda_idx >= 0 and cuda_idx < num_cuda_devices):
cuda_idx = 0

# Generate rkme text spec
rkme_text_spec = RKMETextSpecification(gamma=gamma, cuda_idx=cuda_idx)
rkme_text_spec.generate_stat_spec_from_data(X, reduced_set_size, step_size, steps, nonnegative_beta, reduce)
return rkme_text_spec


def generate_stat_spec(type="table", *args, **kwargs) -> BaseStatSpecification:
"""
Interface for users to generate statistical specification.
Return a StatSpecification object, use .save() method to save as npy file.

Parameters
----------
X : np.ndarray
Raw data in np.ndarray format.
Size of array: (n*d)

Returns
-------
StatSpecification
A StatSpecification object
"""
if type == "table":
return generate_rkme_spec(*args, **kwargs)
elif type == "text":
return generate_rkme_text_spec(*args, **kwargs)
elif type == "image":
return generate_rkme_image_spec(*args, **kwargs)
else:
raise TypeError(f"type {type} is not supported!")

+ 1
- 212
learnware/specification/utils.py View File

@@ -1,11 +1,7 @@
import torch
import numpy as np
import pandas as pd
from typing import Union, List

from .base import BaseStatSpecification
from .regular import RKMETableSpecification, RKMEImageSpecification, RKMETextSpecification
from ..config import C
from typing import Union


def convert_to_numpy(data: Union[np.ndarray, pd.DataFrame, torch.Tensor]):
@@ -31,210 +27,3 @@ def convert_to_numpy(data: Union[np.ndarray, pd.DataFrame, torch.Tensor]):
raise TypeError(
"Unsupported data format. Please provide a NumPy array, a Pandas DataFrame, or a PyTorch Tensor."
)


def generate_rkme_spec(
X: Union[np.ndarray, pd.DataFrame, torch.Tensor],
gamma: float = 0.1,
reduced_set_size: int = 100,
step_size: float = 0.1,
steps: int = 3,
nonnegative_beta: bool = True,
reduce: bool = True,
cuda_idx: int = None,
) -> RKMETableSpecification:
"""
Interface for users to generate Reduced Kernel Mean Embedding (RKME) specification.
Return a RKMETableSpecification object, use .save() method to save as json file.

Parameters
----------
X : np.ndarray, pd.DataFrame, or torch.Tensor
Raw data in np.ndarray, pd.DataFrame, or torch.Tensor format.
The shape of X:
First dimension represents the number of samples (data points).
The remaining dimensions represent the dimensions (features) of each sample.
For example, if X has shape (100, 3), it means there are 100 samples, and each sample has 3 features.
gamma : float
Bandwidth in gaussian kernel, by default 0.1.
reduced_set_size : int
Size of the construced reduced set.
step_size : float
Step size for gradient descent in the iterative optimization.
steps : int
Total rounds in the iterative optimization.
nonnegative_beta : bool, optional
True if weights for the reduced set are intended to be kept non-negative, by default False.
reduce : bool, optional
Whether shrink original data to a smaller set, by default True
cuda_idx : int
A flag indicating whether use CUDA during RKME computation. -1 indicates CUDA not used.
None indicates that CUDA is automatically selected.

Returns
-------
RKMETableSpecification
A RKMETableSpecification object
"""
# Convert data type
X = convert_to_numpy(X)
X = np.ascontiguousarray(X).astype(np.float32)

# Check reduced_set_size
max_reduced_set_size = C.max_reduced_set_size
if reduced_set_size * X[0].size > max_reduced_set_size:
reduced_set_size = max(20, max_reduced_set_size // X[0].size)

# Check cuda_idx
if not torch.cuda.is_available() or cuda_idx == -1:
cuda_idx = -1
else:
num_cuda_devices = torch.cuda.device_count()
if cuda_idx is None or not (cuda_idx >= 0 and cuda_idx < num_cuda_devices):
cuda_idx = 0

# Generate rkme spec
rkme_spec = RKMETableSpecification(gamma=gamma, cuda_idx=cuda_idx)
rkme_spec.generate_stat_spec_from_data(X, reduced_set_size, step_size, steps, nonnegative_beta, reduce)
return rkme_spec


def generate_rkme_image_spec(
X: Union[np.ndarray, torch.Tensor],
reduced_set_size: int = 50,
step_size: float = 0.01,
steps: int = 100,
resize: bool = True,
nonnegative_beta: bool = True,
reduce: bool = True,
verbose: bool = True,
cuda_idx: int = None,
) -> RKMEImageSpecification:
"""
Interface for users to generate Reduced Kernel Mean Embedding (RKME) specification for Image.
Return a RKMEImageSpecification object, use .save() method to save as json file.

Parameters
----------
X : np.ndarray, or torch.Tensor
Raw data in np.ndarray, or torch.Tensor format.
The shape of X: [N, C, H, W]
N: Number of images.
C: Number of channels.
H: Height of images.
W: Width of images.s
For example, if X has shape (100, 3, 32, 32), it means there are 100 samples, and each sample is a 3-channel (RGB) image of size 32x32.
reduced_set_size : int
Size of the construced reduced set.
step_size : float
Step size for gradient descent in the iterative optimization.
steps : int
Total rounds in the iterative optimization.
resize : bool
Whether to scale the image to the requested size, by default True.
nonnegative_beta : bool, optional
True if weights for the reduced set are intended to be kept non-negative, by default False.
reduce : bool, optional
Whether shrink original data to a smaller set, by default True
cuda_idx : int
A flag indicating whether use CUDA during RKME computation. -1 indicates CUDA not used.
None indicates that CUDA is automatically selected.
verbose : bool, optional
Whether to print training progress, by default True

Returns
-------
RKMEImageSpecification
A RKMEImageSpecification object
"""

# Check cuda_idx
if not torch.cuda.is_available() or cuda_idx == -1:
cuda_idx = -1
else:
num_cuda_devices = torch.cuda.device_count()
if cuda_idx is None or not (0 <= cuda_idx < num_cuda_devices):
cuda_idx = 0

# Generate rkme spec
rkme_image_spec = RKMEImageSpecification(cuda_idx=cuda_idx)
rkme_image_spec.generate_stat_spec_from_data(
X, reduced_set_size, step_size, steps, resize, nonnegative_beta, reduce, verbose
)
return rkme_image_spec


def generate_rkme_text_spec(
X: List[str],
gamma: float = 0.1,
reduced_set_size: int = 100,
step_size: float = 0.1,
steps: int = 3,
nonnegative_beta: bool = True,
reduce: bool = True,
cuda_idx: int = None,
) -> RKMETextSpecification:
"""
Interface for users to generate Reduced Kernel Mean Embedding (RKME) specification for Text.
Return a RKMETextSpecification object, use .save() method to save as json file.

Parameters
----------
X : List[str]
Raw data of text.
gamma : float
Bandwidth in gaussian kernel, by default 0.1.
reduced_set_size : int
Size of the construced reduced set.
step_size : float
Step size for gradient descent in the iterative optimization.
steps : int
Total rounds in the iterative optimization.
nonnegative_beta : bool, optional
True if weights for the reduced set are intended to be kept non-negative, by default False.
reduce : bool, optional
Whether shrink original data to a smaller set, by default True
cuda_idx : int
A flag indicating whether use CUDA during RKME computation. -1 indicates CUDA not used.
None indicates that CUDA is automatically selected.

Returns
-------
RKMETextSpecification
A RKMETextSpecification object
"""
# Check input type
if not isinstance(X, list) or not all(isinstance(item, str) for item in X):
raise TypeError("Input data must be a list of strings.")

# Check cuda_idx
if not torch.cuda.is_available() or cuda_idx == -1:
cuda_idx = -1
else:
num_cuda_devices = torch.cuda.device_count()
if cuda_idx is None or not (cuda_idx >= 0 and cuda_idx < num_cuda_devices):
cuda_idx = 0

# Generate rkme text spec
rkme_text_spec = RKMETextSpecification(gamma=gamma, cuda_idx=cuda_idx)
rkme_text_spec.generate_stat_spec_from_data(X, reduced_set_size, step_size, steps, nonnegative_beta, reduce)
return rkme_text_spec


def generate_stat_spec(X: np.ndarray) -> BaseStatSpecification:
"""
Interface for users to generate statistical specification.
Return a StatSpecification object, use .save() method to save as npy file.

Parameters
----------
X : np.ndarray
Raw data in np.ndarray format.
Size of array: (n*d)

Returns
-------
StatSpecification
A StatSpecification object
"""
return None

+ 1
- 1
tests/test_learnware_client/test_all_learnware.py View File

@@ -43,7 +43,7 @@ class TestAllLearnware(unittest.TestCase):
failed_ids.append(idx)
print(f"check learnware {idx} failed!!!")

print(f"failed learnware ids: {failed_ids}")
print(f"The currently failed learnware ids: {failed_ids}")


if __name__ == "__main__":


+ 60
- 26
tests/test_market/test_easy.py View File

@@ -12,12 +12,13 @@ from shutil import copyfile, rmtree

import learnware
from learnware.market import instantiate_learnware_market, BaseUserInfo
import learnware.specification as specification
from learnware.specification import RKMETableSpecification, generate_rkme_spec
from learnware.reuse import JobSelectorReuser, AveragingReuser, EnsemblePruningReuser

curr_root = os.path.dirname(os.path.abspath(__file__))

user_semantic = {
"Data": {"Values": ["Image"], "Type": "Class"},
"Data": {"Values": ["Table"], "Type": "Class"},
"Task": {
"Values": ["Classification"],
"Type": "Class",
@@ -26,12 +27,6 @@ user_semantic = {
"Scenario": {"Values": ["Education"], "Type": "Tag"},
"Description": {"Values": "", "Type": "String"},
"Name": {"Values": "", "Type": "String"},
"Output": {
"Dimension": 10,
"Description": {
"0": "the probability of the label is zero",
},
},
}


@@ -43,7 +38,7 @@ class TestMarket(unittest.TestCase):

def _init_learnware_market(self):
"""initialize learnware market"""
easy_market = instantiate_learnware_market(market_id="sklearn_digits", name="easy", rebuild=True)
easy_market = instantiate_learnware_market(market_id="sklearn_digits_easy", name="easy", rebuild=True)
return easy_market

def test_prepare_learnware_randomly(self, learnware_num=5):
@@ -62,7 +57,7 @@ class TestMarket(unittest.TestCase):

joblib.dump(clf, os.path.join(dir_path, "svm.pkl"))

spec = specification.utils.generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0)
spec = generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0)
spec.save(os.path.join(dir_path, "svm.json"))

init_file = os.path.join(dir_path, "__init__.py")
@@ -103,11 +98,21 @@ class TestMarket(unittest.TestCase):
semantic_spec = copy.deepcopy(user_semantic)
semantic_spec["Name"]["Values"] = "learnware_%d" % (idx)
semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx)
semantic_spec["Input"] = {
"Dimension": 64,
"Description": {
f"{i}": f"The value in the grid {i // 8}{i % 8} of the image of hand-written digit."
for i in range(64)
},
}
semantic_spec["Output"] = {
"Dimension": 10,
"Description": {f"{i}": "The probability for each digit for 0 to 9." for i in range(10)},
}
easy_market.add_learnware(zip_path, semantic_spec)

print("Total Item:", len(easy_market))
assert len(easy_market) == self.learnware_num, f"The number of learnwares must be {self.learnware_num}!"

curr_inds = easy_market.get_learnware_ids()
print("Available ids After Uploading Learnwares:", curr_inds)
assert len(curr_inds) == self.learnware_num, f"The number of learnwares must be {self.learnware_num}!"
@@ -128,31 +133,29 @@ class TestMarket(unittest.TestCase):
easy_market = self.test_upload_delete_learnware(learnware_num, delete=False)
print("Total Item:", len(easy_market))
assert len(easy_market) == self.learnware_num, f"The number of learnwares must be {self.learnware_num}!"
test_folder = os.path.join(curr_root, "test_semantics")

# unzip -o -q zip_path -d unzip_dir
if os.path.exists(test_folder):
rmtree(test_folder)
os.makedirs(test_folder, exist_ok=True)

with zipfile.ZipFile(self.zip_path_list[0], "r") as zip_obj:
zip_obj.extractall(path=test_folder)

semantic_spec = copy.deepcopy(user_semantic)
semantic_spec["Name"]["Values"] = f"learnware_{learnware_num - 1}"
semantic_spec["Description"]["Values"] = f"test_learnware_number_{learnware_num - 1}"

user_info = BaseUserInfo(semantic_spec=semantic_spec)
_, single_learnware_list, _, _ = easy_market.search_learnware(user_info)

print("User info:", user_info.get_semantic_spec())
print(f"Search result:")
assert len(single_learnware_list) == 1, f"Exact semantic search failed!"
for learnware in single_learnware_list:
semantic_spec1 = learnware.get_specification().get_semantic_spec()
print("Choose learnware:", learnware.id, semantic_spec1)
assert semantic_spec1["Name"]["Values"] == semantic_spec["Name"]["Values"], f"Exact semantic search failed!"
print("Choose learnware:", learnware.id, learnware.get_specification().get_semantic_spec())

semantic_spec["Name"]["Values"] = "laernwaer"
user_info = BaseUserInfo(semantic_spec=semantic_spec)
_, single_learnware_list, _, _ = easy_market.search_learnware(user_info)

print("User info:", user_info.get_semantic_spec())
print(f"Search result:")
assert len(single_learnware_list) == self.learnware_num, f"Fuzzy semantic search failed!"
for learnware in single_learnware_list:
semantic_spec1 = learnware.get_specification().get_semantic_spec()
print("Choose learnware:", learnware.id, semantic_spec1)
rmtree(test_folder) # rm -r test_folder

def test_stat_search(self, learnware_num=5):
easy_market = self.test_upload_delete_learnware(learnware_num, delete=False)
@@ -170,7 +173,7 @@ class TestMarket(unittest.TestCase):
with zipfile.ZipFile(zip_path, "r") as zip_obj:
zip_obj.extractall(path=unzip_dir)

user_spec = specification.rkme.RKMETableSpecification()
user_spec = RKMETableSpecification()
user_spec.load(os.path.join(unzip_dir, "svm.json"))
user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={"RKMETableSpecification": user_spec})
(
@@ -190,6 +193,36 @@ class TestMarket(unittest.TestCase):

rmtree(test_folder) # rm -r test_folder

def test_learnware_reuse(self, learnware_num=5):
easy_market = self.test_upload_delete_learnware(learnware_num, delete=False)
print("Total Item:", len(easy_market))

X, y = load_digits(return_X_y=True)
train_X, data_X, train_y, data_y = train_test_split(X, y, test_size=0.3, shuffle=True)

stat_spec = generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0)
user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={"RKMETableSpecification": stat_spec})

_, _, _, mixture_learnware_list = easy_market.search_learnware(user_info)

# Based on user information, the learnware market returns a list of learnwares (learnware_list)
# Use jobselector reuser to reuse the searched learnwares to make prediction
reuse_job_selector = JobSelectorReuser(learnware_list=mixture_learnware_list)
job_selector_predict_y = reuse_job_selector.predict(user_data=data_X)

# Use averaging ensemble reuser to reuse the searched learnwares to make prediction
reuse_ensemble = AveragingReuser(learnware_list=mixture_learnware_list, mode="vote_by_prob")
ensemble_predict_y = reuse_ensemble.predict(user_data=data_X)

# Use ensemble pruning reuser to reuse the searched learnwares to make prediction
reuse_ensemble = EnsemblePruningReuser(learnware_list=mixture_learnware_list, mode="classification")
reuse_ensemble.fit(train_X[-200:], train_y[-200:])
ensemble_pruning_predict_y = reuse_ensemble.predict(user_data=data_X)

print("Job Selector Acc:", np.sum(np.argmax(job_selector_predict_y, axis=1) == data_y) / len(data_y))
print("Averaging Reuser Acc:", np.sum(np.argmax(ensemble_predict_y, axis=1) == data_y) / len(data_y))
print("Ensemble Pruning Reuser Acc:", np.sum(ensemble_pruning_predict_y == data_y) / len(data_y))


def suite():
_suite = unittest.TestSuite()
@@ -197,6 +230,7 @@ def suite():
_suite.addTest(TestMarket("test_upload_delete_learnware"))
_suite.addTest(TestMarket("test_search_semantics"))
_suite.addTest(TestMarket("test_stat_search"))
_suite.addTest(TestMarket("test_learnware_reuse"))
return _suite




+ 1
- 0
tests/test_market/test_hetero_market/example_learnwares/config.py View File

@@ -0,0 +1 @@
input_shape_list=[20, 30] # 20-input shape of example learnware 0, 30-input shape of example learnware 1

+ 22
- 0
tests/test_market/test_hetero_market/example_learnwares/example_learnware_0/__init__.py View File

@@ -0,0 +1,22 @@
from learnware.model import BaseModel
import numpy as np
import joblib
import os


class MyModel(BaseModel):
def __init__(self):
super(MyModel, self).__init__(input_shape=(20,), output_shape=(1,))
dir_path = os.path.dirname(os.path.abspath(__file__))
model_path=os.path.join(dir_path, "ridge.pkl")
model = joblib.load(model_path)
self.model=model

def fit(self, X: np.ndarray, y: np.ndarray):
pass

def predict(self, X: np.ndarray) -> np.ndarray:
return self.model.predict(X)
def finetune(self, X: np.ndarray, y: np.ndarray):
pass

+ 8
- 0
tests/test_market/test_hetero_market/example_learnwares/example_learnware_0/learnware.yaml View File

@@ -0,0 +1,8 @@
model:
class_name: MyModel
kwargs: {}
stat_specifications:
- module_path: learnware.specification
class_name: RKMETableSpecification
file_name: stat.json
kwargs: {}

+ 1
- 0
tests/test_market/test_hetero_market/example_learnwares/example_learnware_0/requirements.txt View File

@@ -0,0 +1 @@
learnware == 0.1.0.999

+ 22
- 0
tests/test_market/test_hetero_market/example_learnwares/example_learnware_1/__init__.py View File

@@ -0,0 +1,22 @@
from learnware.model import BaseModel
import numpy as np
import joblib
import os


class MyModel(BaseModel):
def __init__(self):
super(MyModel, self).__init__(input_shape=(30,), output_shape=(1,))
dir_path = os.path.dirname(os.path.abspath(__file__))
model_path=os.path.join(dir_path, "ridge.pkl")
model = joblib.load(model_path)
self.model=model

def fit(self, X: np.ndarray, y: np.ndarray):
pass

def predict(self, X: np.ndarray) -> np.ndarray:
return self.model.predict(X)
def finetune(self, X: np.ndarray, y: np.ndarray):
pass

+ 8
- 0
tests/test_market/test_hetero_market/example_learnwares/example_learnware_1/learnware.yaml View File

@@ -0,0 +1,8 @@
model:
class_name: MyModel
kwargs: {}
stat_specifications:
- module_path: learnware.specification
class_name: RKMETableSpecification
file_name: stat.json
kwargs: {}

+ 1
- 0
tests/test_market/test_hetero_market/example_learnwares/example_learnware_1/requirements.txt View File

@@ -0,0 +1 @@
learnware == 0.1.0.999

+ 236
- 0
tests/test_market/test_hetero_market/test_hetero.py View File

@@ -0,0 +1,236 @@
import sys
import unittest
import os
import copy
import joblib
import zipfile
import numpy as np
from sklearn.linear_model import Ridge
from sklearn.datasets import make_regression
from sklearn.datasets import load_digits
from shutil import copyfile, rmtree
from multiprocessing import Pool
from learnware.client import LearnwareClient

import learnware
from learnware.market import instantiate_learnware_market, BaseUserInfo
import learnware.specification as specification
from example_learnwares.config import input_shape_list

curr_root = os.path.dirname(os.path.abspath(__file__))

user_semantic = {
"Data": {"Values": ["Image"], "Type": "Class"},
"Task": {
"Values": ["Classification"],
"Type": "Class",
},
"Library": {"Values": ["Scikit-learn"], "Type": "Class"},
"Scenario": {"Values": ["Education"], "Type": "Tag"},
"Description": {"Values": "", "Type": "String"},
"Name": {"Values": "", "Type": "String"},
"Output": {
"Dimension": 10,
"Description": {
"0": "the probability of the label is zero",
},
},
}


def check_learnware(learnware_name, dir_path=os.path.join(curr_root, "learnware_pool")):
print(f"Checking Learnware: {learnware_name}")
zip_file_path = os.path.join(dir_path, learnware_name)
client = LearnwareClient()
# if check_learnware doesn't raise an exception, return True, otherwise, return false
try:
client.check_learnware(zip_file_path)
return True
except Exception as e:
print(f"Learnware {learnware_name} failed the check: {e}")
return False


class TestMarket(unittest.TestCase):
@classmethod
def setUpClass(cls) -> None:
np.random.seed(2023)
learnware.init()

def _init_learnware_market(self):
"""initialize learnware market"""
hetero_market = instantiate_learnware_market(market_id="hetero_toy", name="hetero", rebuild=True)
return hetero_market

def test_prepare_learnware_randomly(self, learnware_num=5):
self.zip_path_list = []
X, y = load_digits(return_X_y=True)

for i in range(learnware_num):
dir_path = os.path.join(curr_root, "learnware_pool", "ridge_%d" % (i))
os.makedirs(dir_path, exist_ok=True)

print("Preparing Learnware: %d" % (i))

example_learnware_idx=i%2
input_dim=input_shape_list[example_learnware_idx]
example_learnware_name="example_learnwares/example_learnware_%d" % (example_learnware_idx)

X, y = make_regression(n_samples=5000, n_features=input_dim, noise=0.1, random_state=42)

clf=Ridge(alpha=1.0)
clf.fit(X, y)

joblib.dump(clf, os.path.join(dir_path, "ridge.pkl"))

spec = specification.utils.generate_rkme_spec(X=X, gamma=0.1, cuda_idx=0)
spec.save(os.path.join(dir_path, "stat.json"))

init_file = os.path.join(dir_path, "__init__.py")
copyfile(
os.path.join(curr_root, example_learnware_name, "__init__.py"), init_file
) # cp example_init.py init_file

yaml_file = os.path.join(dir_path, "learnware.yaml")
copyfile(os.path.join(curr_root, example_learnware_name, "learnware.yaml"), yaml_file) # cp example.yaml yaml_file

env_file = os.path.join(dir_path, "requirements.txt")
copyfile(os.path.join(curr_root, example_learnware_name, "requirements.txt"), env_file)

zip_file = dir_path + ".zip"
# zip -q -r -j zip_file dir_path
with zipfile.ZipFile(zip_file, "w") as zip_obj:
for foldername, subfolders, filenames in os.walk(dir_path):
for filename in filenames:
file_path = os.path.join(foldername, filename)
zip_info = zipfile.ZipInfo(filename)
zip_info.compress_type = zipfile.ZIP_STORED
with open(file_path, "rb") as file:
zip_obj.writestr(zip_info, file.read())

rmtree(dir_path) # rm -r dir_path

def test_generated_learnwares(self):
curr_root = os.path.dirname(os.path.abspath(__file__))
dir_path = os.path.join(curr_root, "learnware_pool")

# Execute multi-process checking using Pool
with Pool() as pool:
results = pool.starmap(check_learnware, [(name, dir_path) for name in os.listdir(dir_path)])

# Use an assert statement to ensure that all checks return True
self.assertTrue(all(results), "Not all learnwares passed the check")

def test_upload_delete_learnware(self, learnware_num=5, delete=True):
hetero_market = self._init_learnware_market()
self.test_prepare_learnware_randomly(learnware_num)
self.learnware_num = learnware_num

print("Total Item:", len(hetero_market))
assert len(hetero_market) == 0, f"The market should be empty!"

for idx, zip_path in enumerate(self.zip_path_list):
semantic_spec = copy.deepcopy(user_semantic)
semantic_spec["Name"]["Values"] = "learnware_%d" % (idx)
semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx)
hetero_market.add_learnware(zip_path, semantic_spec)

print("Total Item:", len(hetero_market))
assert len(hetero_market) == self.learnware_num, f"The number of learnwares must be {self.learnware_num}!"

curr_ids = hetero_market.get_learnware_ids()
print("Available ids After Uploading Learnwares:", curr_ids)
assert len(curr_ids) == self.learnware_num, f"The number of learnwares must be {self.learnware_num}!"

if delete:
for learnware_id in curr_ids:
hetero_market.delete_learnware(learnware_id)
self.learnware_num -= 1
assert len(hetero_market) == self.learnware_num, f"The number of learnwares must be {self.learnware_num}!"

curr_ids = hetero_market.get_learnware_ids()
print("Available ids After Deleting Learnwares:", curr_ids)
assert len(curr_ids) == 0, f"The market should be empty!"

return hetero_market

# def test_search_semantics(self, learnware_num=5):
# easy_market = self.test_upload_delete_learnware(learnware_num, delete=False)
# print("Total Item:", len(easy_market))
# assert len(easy_market) == self.learnware_num, f"The number of learnwares must be {self.learnware_num}!"

# semantic_spec = copy.deepcopy(user_semantic)
# semantic_spec["Name"]["Values"] = f"learnware_{learnware_num - 1}"

# user_info = BaseUserInfo(semantic_spec=semantic_spec)
# _, single_learnware_list, _, _ = easy_market.search_learnware(user_info)

# print("User info:", user_info.get_semantic_spec())
# print(f"Search result:")
# assert len(single_learnware_list) == 1, f"Exact semantic search failed!"
# for learnware in single_learnware_list:
# semantic_spec1 = learnware.get_specification().get_semantic_spec()
# print("Choose learnware:", learnware.id, semantic_spec1)
# assert semantic_spec1["Name"]["Values"] == semantic_spec["Name"]["Values"], f"Exact semantic search failed!"

# semantic_spec["Name"]["Values"] = "laernwaer"
# user_info = BaseUserInfo(semantic_spec=semantic_spec)
# _, single_learnware_list, _, _ = easy_market.search_learnware(user_info)

# print("User info:", user_info.get_semantic_spec())
# print(f"Search result:")
# assert len(single_learnware_list) == self.learnware_num, f"Fuzzy semantic search failed!"
# for learnware in single_learnware_list:
# semantic_spec1 = learnware.get_specification().get_semantic_spec()
# print("Choose learnware:", learnware.id, semantic_spec1)

# def test_stat_search(self, learnware_num=5):
# easy_market = self.test_upload_delete_learnware(learnware_num, delete=False)
# print("Total Item:", len(easy_market))

# test_folder = os.path.join(curr_root, "test_stat")

# for idx, zip_path in enumerate(self.zip_path_list):
# unzip_dir = os.path.join(test_folder, f"{idx}")

# # unzip -o -q zip_path -d unzip_dir
# if os.path.exists(unzip_dir):
# rmtree(unzip_dir)
# os.makedirs(unzip_dir, exist_ok=True)
# with zipfile.ZipFile(zip_path, "r") as zip_obj:
# zip_obj.extractall(path=unzip_dir)

# user_spec = specification.rkme.RKMETableSpecification()
# user_spec.load(os.path.join(unzip_dir, "svm.json"))
# user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={"RKMETableSpecification": user_spec})
# (
# sorted_score_list,
# single_learnware_list,
# mixture_score,
# mixture_learnware_list,
# ) = easy_market.search_learnware(user_info)

# assert len(single_learnware_list) == self.learnware_num, f"Statistical search failed!"
# print(f"search result of user{idx}:")
# for score, learnware in zip(sorted_score_list, single_learnware_list):
# print(f"score: {score}, learnware_id: {learnware.id}")
# print(f"mixture_score: {mixture_score}\n")
# mixture_id = " ".join([learnware.id for learnware in mixture_learnware_list])
# print(f"mixture_learnware: {mixture_id}\n")

# rmtree(test_folder) # rm -r test_folder


def suite():
_suite = unittest.TestSuite()
_suite.addTest(TestMarket("test_prepare_learnware_randomly"))
_suite.addTest(TestMarket("test_generated_learnwares"))
# _suite.addTest(TestMarket("test_upload_delete_learnware"))
# _suite.addTest(TestMarket("test_search_semantics"))
# _suite.addTest(TestMarket("test_stat_search"))
return _suite


if __name__ == "__main__":
runner = unittest.TextTestRunner()
runner.run(suite())

+ 2
- 3
tests/test_specification/test_rkme.py View File

@@ -7,9 +7,8 @@ import unittest
import tempfile
import numpy as np

import learnware.specification as specification
from learnware.specification import RKMETableSpecification, RKMEImageSpecification, RKMETextSpecification
from learnware.specification import generate_rkme_image_spec, generate_rkme_spec
from learnware.specification import generate_rkme_image_spec, generate_rkme_spec, generate_rkme_text_spec


class TestRKME(unittest.TestCase):
@@ -71,7 +70,7 @@ class TestRKME(unittest.TestCase):
return text_list

def _test_text_rkme(X):
rkme = specification.utils.generate_rkme_text_spec(X)
rkme = generate_rkme_text_spec(X)

with tempfile.TemporaryDirectory(prefix="learnware_") as tempdir:
rkme_path = os.path.join(tempdir, "rkme.json")


+ 9
- 10
tests/test_workflow/test_workflow.py View File

@@ -13,12 +13,12 @@ from shutil import copyfile, rmtree
import learnware
from learnware.market import EasyMarket, BaseUserInfo
from learnware.reuse import JobSelectorReuser, AveragingReuser, EnsemblePruningReuser
import learnware.specification as specification
from learnware.specification import RKMETableSpecification, generate_rkme_spec

curr_root = os.path.dirname(os.path.abspath(__file__))

user_semantic = {
"Data": {"Values": ["Table"], "Type": "Class"},
"Data": {"Values": ["Image"], "Type": "Class"},
"Task": {
"Values": ["Classification"],
"Type": "Class",
@@ -57,7 +57,7 @@ class TestAllWorkflow(unittest.TestCase):

joblib.dump(clf, os.path.join(dir_path, "svm.pkl"))

spec = specification.utils.generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0)
spec = generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0)
spec.save(os.path.join(dir_path, "svm.json"))

init_file = os.path.join(dir_path, "__init__.py")
@@ -96,11 +96,10 @@ class TestAllWorkflow(unittest.TestCase):
semantic_spec = copy.deepcopy(user_semantic)
semantic_spec["Name"]["Values"] = "learnware_%d" % (idx)
semantic_spec["Description"]["Values"] = "test_learnware_number_%d" % (idx)
semantic_spec["Input"] = {"Dimension": 64}
semantic_spec["Input"].update(
{f"{i}": f"The value in the digit image with row is {i // 8} and col is {i % 8}." for i in range(64)}
)
semantic_spec["Output"] = {"Dimension": 1, "Description": {"0": "The label of the hand-written digit."}}
semantic_spec["Output"] = {
"Dimension": 10,
"Description": {f"{i}": "The probability for each digit for 0 to 9." for i in range(10)},
}
easy_market.add_learnware(zip_path, semantic_spec)

print("Total Item:", len(easy_market))
@@ -159,7 +158,7 @@ class TestAllWorkflow(unittest.TestCase):
with zipfile.ZipFile(zip_path, "r") as zip_obj:
zip_obj.extractall(path=unzip_dir)

user_spec = specification.RKMETableSpecification()
user_spec = RKMETableSpecification()
user_spec.load(os.path.join(unzip_dir, "svm.json"))
user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={"RKMETableSpecification": user_spec})
(
@@ -185,7 +184,7 @@ class TestAllWorkflow(unittest.TestCase):
X, y = load_digits(return_X_y=True)
train_X, data_X, train_y, data_y = train_test_split(X, y, test_size=0.3, shuffle=True)

stat_spec = specification.utils.generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0)
stat_spec = generate_rkme_spec(X=data_X, gamma=0.1, cuda_idx=0)
user_info = BaseUserInfo(semantic_spec=user_semantic, stat_info={"RKMETableSpecification": stat_spec})

_, _, _, mixture_learnware_list = easy_market.search_learnware(user_info)


Write Preview
Loading…
Cancel
Save