beimingwu
/
learnware
Not watched
1
0
Fork 0
Code
Releases 1 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

[MNT] black format

tags/v0.3.2
bxdd 2 years ago
parent
2e3f0cbe80
commit
8f43a40199
7 changed files with 42 additions and 33 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +23
    -12
      learnware/client/container.py
  2. +5
    -3
      learnware/client/learnware_client.py
  3. +2
    -6
      learnware/client/utils.py
  4. +4
    -3
      learnware/learnware/__init__.py
  5. +4
    -4
      learnware/learnware/base.py
  6. +1
    -1
      learnware/market/__init__.py
  7. +3
    -4
      learnware/market/classes.py

+ 23
- 12
learnware/client/container.py View File

@@ -94,7 +94,9 @@ class ModelCondaContainer(ModelContainer):
model_path = os.path.join(tempdir, "model.pkl")

model_config = self.model_config.copy()
model_config["module_path"] = Learnware.get_model_module_abspath(self.learnware_dirpath, model_config["module_path"])
model_config["module_path"] = Learnware.get_model_module_abspath(
self.learnware_dirpath, model_config["module_path"]
)
with open(model_path, "wb") as model_fp:
pickle.dump(model_config, model_fp)

@@ -130,10 +132,12 @@ class ModelCondaContainer(ModelContainer):
input_path = os.path.join(tempdir, "input.pkl")
output_path = os.path.join(tempdir, "output.pkl")
model_path = os.path.join(tempdir, "model.pkl")
model_config = self.model_config.copy()
model_config["module_path"] = Learnware.get_model_module_abspath(self.learnware_dirpath, model_config["module_path"])
model_config["module_path"] = Learnware.get_model_module_abspath(
self.learnware_dirpath, model_config["module_path"]
)

with open(model_path, "wb") as model_fp:
pickle.dump(model_config, model_fp)

@@ -299,9 +303,7 @@ class ModelDockerContainer(ModelContainer):
logger.info(f"Create and update conda env [{conda_env}] according to .yaml file")
for i in range(run_cmd_times):
result = self.docker_container.exec_run(
" ".join(
["conda", "env", "update", "--name", f"{conda_env}", "--file", f"{yaml_path_filter}"]
)
" ".join(["conda", "env", "update", "--name", f"{conda_env}", "--file", f"{yaml_path_filter}"])
)
if result.exit_code == 0:
success_flag = True
@@ -386,7 +388,9 @@ class ModelDockerContainer(ModelContainer):
self.docker_model_script_path = os.path.join(tempdir, "run_model.py")

docker_model_config = self.model_config.copy()
docker_model_config["module_path"] = Learnware.get_model_module_abspath(self.learnware_dirpath, docker_model_config["module_path"])
docker_model_config["module_path"] = Learnware.get_model_module_abspath(
self.learnware_dirpath, docker_model_config["module_path"]
)
self._copy_file_to_container(self.learnware_dirpath, self.learnware_dirpath)

with open(model_path, "wb") as model_fp:
@@ -436,7 +440,9 @@ class ModelDockerContainer(ModelContainer):
model_path = os.path.join(tempdir, "model.pkl")

docker_model_config = self.model_config.copy()
docker_model_config["module_path"] = Learnware.get_model_module_abspath(self.learnware_dirpath, docker_model_config["module_path"])
docker_model_config["module_path"] = Learnware.get_model_module_abspath(
self.learnware_dirpath, docker_model_config["module_path"]
)
with open(model_path, "wb") as model_fp:
pickle.dump(docker_model_config, model_fp)

@@ -518,7 +524,10 @@ class LearnwaresContainer:
if self.mode == "conda":
self.learnware_containers = [
Learnware(
_learnware.id, ModelCondaContainer(_learnware.get_model(), _learnware.get_dirpath()), _learnware.get_specification(), _learnware.get_dirpath()
_learnware.id,
ModelCondaContainer(_learnware.get_model(), _learnware.get_dirpath()),
_learnware.get_specification(),
_learnware.get_dirpath(),
)
for _learnware in self.learnware_list
]
@@ -527,9 +536,11 @@ class LearnwaresContainer:
self.learnware_containers = [
Learnware(
_learnware.id,
ModelDockerContainer(_learnware.get_model(), _learnware.get_dirpath(), self._docker_container, build=False),
ModelDockerContainer(
_learnware.get_model(), _learnware.get_dirpath(), self._docker_container, build=False
),
_learnware.get_specification(),
_learnware.get_dirpath()
_learnware.get_dirpath(),
)
for _learnware in self.learnware_list
]


+ 5
- 3
learnware/client/learnware_client.py View File

@@ -388,7 +388,7 @@ class LearnwareClient:
def _check_semantic_specification(semantic_spec):
semantic_checker = EasySemanticChecker()
return semantic_checker(semantic_spec) != BaseChecker.INVALID_LEARNWARE
@staticmethod
def _check_stat_specification(learnware):
stat_checker = CondaChecker(inner_checker=EasyStatChecker())
@@ -399,8 +399,10 @@ class LearnwareClient:
semantic_specification = (
get_semantic_specification() if semantic_specification is None else semantic_specification
)
assert LearnwareClient._check_semantic_specification(semantic_specification), "Semantic specification check failed!"

assert LearnwareClient._check_semantic_specification(
semantic_specification
), "Semantic specification check failed!"

with tempfile.TemporaryDirectory(prefix="learnware_") as tempdir:
with zipfile.ZipFile(learnware_zip_path, mode="r") as z_file:


+ 2
- 6
learnware/client/utils.py View File

@@ -46,17 +46,13 @@ def install_environment(learnware_dirpath, conda_env):
filter_nonexist_conda_packages_file(yaml_file=yaml_path, output_yaml_file=yaml_path_filter)
# create environment
logger.info(f"create conda env [{conda_env}] according to .yaml file")
system_execute(
args=["conda", "env", "create", "--name", f"{conda_env}", "--file", f"{yaml_path_filter}"]
)
system_execute(args=["conda", "env", "create", "--name", f"{conda_env}", "--file", f"{yaml_path_filter}"])

elif "requirements.txt" in os.listdir(learnware_dirpath):
requirements_path: str = os.path.join(learnware_dirpath, "requirements.txt")
requirements_path_filter: str = os.path.join(tempdir, "requirements_filter.txt")
logger.info(f"checking the avaliabe pip packages for {conda_env}")
filter_nonexist_pip_packages_file(
requirements_file=requirements_path, output_file=requirements_path_filter
)
filter_nonexist_pip_packages_file(requirements_file=requirements_path, output_file=requirements_path_filter)
logger.info(f"create empty conda env [{conda_env}]")
system_execute(args=["conda", "create", "-y", "--name", f"{conda_env}", "python=3.8"])
logger.info(f"install pip requirements for conda env [{conda_env}]")


+ 4
- 3
learnware/learnware/__init__.py View File

@@ -46,7 +46,7 @@ def get_learnware_from_dirpath(id: str, semantic_spec: dict, learnware_dirpath)

try:
yaml_config = read_yaml_to_dict(os.path.join(learnware_dirpath, C.learnware_folder_config["yaml_file"]))
if "name" in yaml_config:
learnware_config["name"] = yaml_config["name"]
if "model" in yaml_config:
@@ -57,7 +57,6 @@ def get_learnware_from_dirpath(id: str, semantic_spec: dict, learnware_dirpath)
if "module_path" not in learnware_config["model"]:
learnware_config["model"]["module_path"] = C.learnware_folder_config["module_file"]

learnware_spec = Specification()
for _stat_spec in learnware_config["stat_specifications"]:
stat_spec = _stat_spec.copy()
@@ -71,4 +70,6 @@ def get_learnware_from_dirpath(id: str, semantic_spec: dict, learnware_dirpath)
logger.warning(f"Load Learnware {id} failed! Due to {repr(e)}")
return None

return Learnware(id=id, model=learnware_config["model"], specification=learnware_spec, learnware_dirpath=learnware_dirpath)
return Learnware(
id=id, model=learnware_config["model"], specification=learnware_spec, learnware_dirpath=learnware_dirpath
)

+ 4
- 4
learnware/learnware/base.py View File

@@ -37,7 +37,7 @@ class Learnware:
specification : Specification
The specification including the semantic specification and the statistic specification
dirpath: str
The path of the learnware directory
The path of the learnware directory
"""
self.id = id
self.model = model
@@ -52,7 +52,7 @@ class Learnware:
if isinstance(module_path, str) and module_path.endswith(".py") and not os.path.isabs(module_path):
module_path = os.path.join(learnware_dirpath, module_path)
return module_path
def instantiate_model(self):
if isinstance(self.model, BaseModel):
logger.info("The learnware had been instantiated, thus the instantiation operation is ignored!")
@@ -72,13 +72,13 @@ class Learnware:

def get_model(self) -> Union[dict, BaseModel]:
return self.model
def get_specification(self) -> Specification:
return self.specification

def get_dirpath(self) -> str:
return self.learnware_dirpath
def update_stat_spec(self, name, new_stat_spec: BaseStatSpecification):
self.specification.update_stat_spec(**{name: new_stat_spec})



+ 1
- 1
learnware/market/__init__.py View File

@@ -7,4 +7,4 @@ from .hetergeneous import HeterogeneousOrganizer, MappingFunction

from .easy import EasyMarket
from .module import instatiate_learnware_market
from .classes import CondaChecker
from .classes import CondaChecker

+ 3
- 4
learnware/market/classes.py View File

@@ -7,13 +7,12 @@ class CondaChecker(BaseChecker):
def __init__(self, inner_checker, **kwargs):
self.inner_checker = inner_checker
super(CondaChecker, self).__init__(**kwargs)
def __call__(self, learnware: Learnware) -> int:
with LearnwaresContainer(learnware) as env_container:
learnwares = env_container.get_learnwares_with_container()
if len(learnwares) == 0:
raise AssertionError(f"The env of learnware {learnware} installed failed")
check_status = self.inner_checker(learnware[0])
return check_status
return check_status

Write Preview
Loading…
Cancel
Save