Browse Source

[ENH] add check_learnware in client

tags/v0.3.2
Gene 2 years ago
parent
commit
7e36e63f59
1 changed files with 19 additions and 103 deletions
  1. +19
    -103
      learnware/client/learnware_client.py

+ 19
- 103
learnware/client/learnware_client.py View File

@@ -19,7 +19,8 @@ from .container import LearnwaresContainer
from ..market.easy import EasyMarket
from ..logger import get_module_logger
from ..specification import Specification
from ..learnware import BaseReuser, Learnware
from ..learnware import BaseReuser, Learnware, get_learnware_from_dirpath
from ..test import get_semantic_specification

CHUNK_SIZE = 1024 * 1024
logger = get_module_logger(module_name="LearnwareClient")
@@ -264,29 +265,6 @@ class LearnwareClient:
raise Exception("delete failed: " + json.dumps(result))
pass

def check_learnware(self, path, semantic_specification):
if os.path.isfile(path):
with tempfile.TemporaryDirectory() as tempdir:
with zipfile.ZipFile(path, "r") as z_file:
z_file.extractall(tempdir)
pass
return self.check_learnware_folder(tempdir, semantic_specification)
pass
else:
return self.check_learnware_folder(path, semantic_specification)
pass
pass

def check_learnware_folder(self, folder, semantic_specification):
learnware_obj = learnware.get_learnware_from_dirpath("test_id", semantic_specification, folder)

check_result = EasyMarket.check_learnware(learnware_obj)
if check_result == EasyMarket.USABLE_LEARWARE:
return True
else:
return False
pass

def create_semantic_specification(
self, name, description, data_type, task_type, library_type, senarioes, input_description, output_description
):
@@ -408,91 +386,29 @@ class LearnwareClient:
return learnware_list[0]
else:
return learnware_list

def system(self, command):
retcd = os.system(command)
if retcd != 0:
raise RuntimeError(f"Command {command} failed with return code {retcd}")
pass

def install_environment(self, zip_path, conda_env=None):
"""Install environment of a learnware

Parameters
----------
zip_path : str
Path of the learnware zip file
conda_env : optional
If it is not None, a new conda environment will be created with the given name;
If it is None, use current environment.

Raises
------
Exception
Lack of the environment configuration file.
"""
with tempfile.TemporaryDirectory(prefix="learnware_") as tempdir:
with zipfile.ZipFile(zip_path, "r") as z_file:
logger.info(f"zip_file namelist: {z_file.namelist}")
if "environment.yaml" in z_file.namelist():
z_file.extract("environment.yaml", tempdir)
yaml_path = os.path.join(tempdir, "environment.yaml")
yaml_path_filter = os.path.join(tempdir, "environment_filter.yaml")
package_utils.filter_nonexist_conda_packages_file(yaml_path, yaml_path_filter)
# create environment
if conda_env is not None:
self.system(f"conda env update --name {conda_env} --file {yaml_path_filter}")
pass
else:
self.system(f"conda env update --file {yaml_path_filter}")
pass
pass
elif "requirements.txt" in z_file.namelist():
z_file.extract("requirements.txt", tempdir)
requirements_path = os.path.join(tempdir, "requirements.txt")
requirements_path_filter = os.path.join(tempdir, "requirements_filter.txt")
package_utils.filter_nonexist_pip_packages_file(requirements_path, requirements_path_filter)

if conda_env is not None:
self.system(f"conda create -y --name {conda_env} python=3.8")
self.system(
f"conda run --name {conda_env} --no-capture-output python3 -m pip install -r {requirements_path_filter}"
)
else:
self.system(f"python3 -m pip install -r {requirements_path_filter}")
pass
pass
else:
raise Exception("Environment.yaml or requirements.txt not found in the learnware zip file.")
pass
pass
pass

def test_learnware(self, zip_path, semantic_specification=None):
@staticmethod
def check_learnware(zip_path, semantic_specification=None):
if semantic_specification is None:
semantic_specification = dict()
pass

semantic_specification = get_semantic_specification()
with tempfile.TemporaryDirectory(prefix="learnware_") as tempdir:
with zipfile.ZipFile(zip_path, mode="r") as z_file:
z_file.extractall(tempdir)
pass

learnware_obj = learnware.get_learnware_from_dirpath("test_id", semantic_specification, tempdir)

if learnware_obj is None:
learnware = get_learnware_from_dirpath(
id="test", semantic_spec=semantic_specification, learnware_dirpath=tempdir
)
if learnware is None:
raise Exception("The learnware is not valid.")

learnware_obj.instantiate_model()

if len(semantic_specification) > 0:
if EasyMarket.check_learnware(learnware_obj) != EasyMarket.USABLE_LEARWARE:
with LearnwaresContainer(learnware, zip_path) as env_container:
learnware = env_container.get_learnwares_with_container()[0]
if EasyMarket.check_learnware(learnware) == EasyMarket.USABLE_LEARWARE:
logger.info("The learnware passed the local test.")
else:
raise Exception("The learnware is not usable.")
pass
pass

logger.info("test ok")
pass

def cleanup(self):
for tempdir in self.tempdir_list:


Loading…
Cancel
Save