Browse Source

[MNT] make container install env with dirpath rather than zippath

tags/v0.3.2
bxdd 2 years ago
parent
commit
96b2b43028
2 changed files with 49 additions and 52 deletions
  1. +8
    -8
      learnware/client/container.py
  2. +41
    -44
      learnware/client/utils.py

+ 8
- 8
learnware/client/container.py View File

@@ -22,10 +22,10 @@ logger = get_module_logger(module_name="client_container")


class ModelContainer(BaseModel):
def __init__(self, model_config: dict, learnware_zippath: str, build: bool = True):
def __init__(self, model_config: dict, learnware_dirpath: str, build: bool = True):
self.model_script = os.path.join(C.package_path, "client", "scripts", "run_model.py")
self.model_config = model_config
self.learnware_zippath = learnware_zippath
self.learnware_dirpath = learnware_dirpath
self.build = build
self.cleanup_flag = False

@@ -78,12 +78,12 @@ class ModelContainer(BaseModel):


class ModelCondaContainer(ModelContainer):
def __init__(self, model_config: dict, learnware_zippath: str, conda_env: str = None, build: bool = True):
def __init__(self, model_config: dict, learnware_dirpath: str, conda_env: str = None, build: bool = True):
self.conda_env = f"learnware_{shortuuid.uuid()}" if conda_env is None else conda_env
super(ModelCondaContainer, self).__init__(model_config, learnware_zippath, build)
super(ModelCondaContainer, self).__init__(model_config, learnware_dirpath, build)

def _init_env(self):
install_environment(self.learnware_zippath, self.conda_env)
install_environment(self.learnware_dirpath, self.conda_env)

def _remove_env(self):
remove_enviroment(self.conda_env)
@@ -175,7 +175,7 @@ class ModelDockerContainer(ModelContainer):
def __init__(
self,
model_config: dict,
learnware_zippath: str,
learnware_dirpath: str,
docker_container: object = None,
build: bool = True,
):
@@ -192,7 +192,7 @@ class ModelDockerContainer(ModelContainer):
self.docker_model_config = None
self.docker_model_script_path = None
# call init method of parent of parent class
super(ModelDockerContainer, self).__init__(model_config, learnware_zippath, build)
super(ModelDockerContainer, self).__init__(model_config, learnware_dirpath, build)

@staticmethod
def _generate_docker_container():
@@ -377,7 +377,7 @@ class ModelDockerContainer(ModelContainer):

def _setup_env_and_metadata(self):
"""setup env and set the input and output shape by communicating with docker"""
self._install_environment(self.learnware_zippath, self.conda_env)
self._install_environment(self.learnware_dirpath, self.conda_env)
with tempfile.TemporaryDirectory(prefix="learnware_") as tempdir:
output_path = os.path.join(tempdir, "output.pkl")
model_path = os.path.join(tempdir, "model.pkl")


+ 41
- 44
learnware/client/utils.py View File

@@ -22,13 +22,13 @@ def remove_enviroment(conda_env):
system_execute(args=["conda", "env", "remove", "-n", f"{conda_env}"])


def install_environment(zip_path, conda_env):
def install_environment(learnware_dirpath, conda_env):
"""Install environment of a learnware

Parameters
----------
zip_path : str
Path of the learnware zip file
learnware_dirpath : str
Path of the learnware folder
conda_env : str
a new conda environment will be created with the given name;

@@ -38,48 +38,45 @@ def install_environment(zip_path, conda_env):
Lack of the environment configuration file.
"""
with tempfile.TemporaryDirectory(prefix="learnware_") as tempdir:
with zipfile.ZipFile(file=zip_path, mode="r") as z_file:
logger.info(f"zip_file namelist: {z_file.namelist()}")
if "environment.yaml" in z_file.namelist():
z_file.extract(member="environment.yaml", path=tempdir)
yaml_path: str = os.path.join(tempdir, "environment.yaml")
yaml_path_filter: str = os.path.join(tempdir, "environment_filter.yaml")
logger.info(f"checking the avaliabe conda packages for {conda_env}")
filter_nonexist_conda_packages_file(yaml_file=yaml_path, output_yaml_file=yaml_path_filter)
# create environment
logger.info(f"create conda env [{conda_env}] according to .yaml file")
system_execute(
args=["conda", "env", "create", "--name", f"{conda_env}", "--file", f"{yaml_path_filter}"]
)
logger.info(f"zip_file namelist: {os.listdir(learnware_dirpath)}")
if "environment.yaml" in os.listdir(learnware_dirpath):
yaml_path: str = os.path.join(learnware_dirpath, "environment.yaml")
yaml_path_filter: str = os.path.join(tempdir, "environment_filter.yaml")
logger.info(f"checking the avaliabe conda packages for {conda_env}")
filter_nonexist_conda_packages_file(yaml_file=yaml_path, output_yaml_file=yaml_path_filter)
# create environment
logger.info(f"create conda env [{conda_env}] according to .yaml file")
system_execute(
args=["conda", "env", "create", "--name", f"{conda_env}", "--file", f"{yaml_path_filter}"]
)

elif "requirements.txt" in z_file.namelist():
z_file.extract(member="requirements.txt", path=tempdir)
requirements_path: str = os.path.join(tempdir, "requirements.txt")
requirements_path_filter: str = os.path.join(tempdir, "requirements_filter.txt")
logger.info(f"checking the avaliabe pip packages for {conda_env}")
filter_nonexist_pip_packages_file(
requirements_file=requirements_path, output_file=requirements_path_filter
)
logger.info(f"create empty conda env [{conda_env}]")
system_execute(args=["conda", "create", "-y", "--name", f"{conda_env}", "python=3.8"])
logger.info(f"install pip requirements for conda env [{conda_env}]")
system_execute(
args=[
"conda",
"run",
"-n",
f"{conda_env}",
"--no-capture-output",
"python3",
"-m",
"pip",
"install",
"-r",
f"{requirements_path_filter}",
]
)
else:
raise Exception("Environment.yaml or requirements.txt not found in the learnware zip file.")
elif "requirements.txt" in os.listdir(learnware_dirpath):
requirements_path: str = os.path.join(learnware_dirpath, "requirements.txt")
requirements_path_filter: str = os.path.join(tempdir, "requirements_filter.txt")
logger.info(f"checking the avaliabe pip packages for {conda_env}")
filter_nonexist_pip_packages_file(
requirements_file=requirements_path, output_file=requirements_path_filter
)
logger.info(f"create empty conda env [{conda_env}]")
system_execute(args=["conda", "create", "-y", "--name", f"{conda_env}", "python=3.8"])
logger.info(f"install pip requirements for conda env [{conda_env}]")
system_execute(
args=[
"conda",
"run",
"-n",
f"{conda_env}",
"--no-capture-output",
"python3",
"-m",
"pip",
"install",
"-r",
f"{requirements_path_filter}",
]
)
else:
raise Exception("Environment.yaml or requirements.txt not found in the learnware folder.")

logger.info(f"install learnware package for conda env [{conda_env}]")
system_execute(


Loading…
Cancel
Save