Merge pull request #11 from Learnware-LAMDA/bixd/dev

Make Model Container Run With Docker, and Add Docker Container Manager

examples/dataset_pfs_workflow/pfs/pfs_cross_transfer.py

@@ -85,9 +85,7 @@ def get_split_errs(algo):
                 split = train_xs.shape[0] - proportion_list[tmp]
 
                 model.fit(
-                    train_xs[
-                        split:,
-                    ],
+                    train_xs[split:,],
                     train_ys[split:],
                     eval_set=[(val_xs, val_ys)],
                     early_stopping_rounds=50,

learnware/client/container.py

@@ -1,15 +1,19 @@
 import os
+import docker
 import pickle
 import atexit
+import tarfile
+import zipfile
 import tempfile
 import shortuuid
-from concurrent.futures import ProcessPoolExecutor
+from concurrent.futures import ThreadPoolExecutor
 
-from typing import List
+from typing import List, Union
 from .utils import system_execute, install_environment, remove_enviroment
 from ..config import C
 from ..learnware import Learnware
 from ..model.base import BaseModel
+from .package_utils import filter_nonexist_conda_packages_file, filter_nonexist_pip_packages_file
 
 from ..logger import get_module_logger
 
@@ -17,15 +21,74 @@ from ..logger import get_module_logger
 logger = get_module_logger(module_name="client_container")
 
 
-class ModelEnvContainer(BaseModel):
-    def __init__(self, model_config: dict, learnware_zippath: str):
+class ModelContainer(BaseModel):
+    def __init__(self, model_config: dict, learnware_zippath: str, build: bool = True):
         self.model_script = os.path.join(C.package_path, "client", "scripts", "run_model.py")
         self.model_config = model_config
-        self.conda_env = f"learnware_{shortuuid.uuid()}"
         self.learnware_zippath = learnware_zippath
+        self.build = build
+        self.cleanup_flag = False
+
+    def __enter__(self):
+        self.init_and_setup_env()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.remove_env()
+
+    def reset(self, input_shape, output_shape):
+        self.input_shape = input_shape
+        self.output_shape = output_shape
+
+    def init_and_setup_env(self):
+        """We must set `input_shape` and `output_shape`"""
+        if self.build:
+            self.cleanup_flag = True
+            self._init_env()
+            atexit.register(self.remove_env)
+
+        self._setup_env_and_metadata()
+
+    def remove_env(self):
+        if self.cleanup_flag is True:
+            self.cleanup_flag = False
+            try:
+                self._remove_env()
+            except Exception as err:
+                self.cleanup_flag = True
+                raise err
+
+    def _setup_env_and_metadata(self):
+        raise NotImplementedError("_setup_env_and_metadata method is not implemented!")
+
+    def _init_env(self):
+        raise NotImplementedError("_init_env method is not implemented!")
+
+    def _remove_env(self):
+        raise NotImplementedError("_remove_env method is not implemented!")
+
+    def fit(self, X, y):
+        raise NotImplementedError("fit method is not implemented!")
+
+    def predict(self, X):
+        raise NotImplementedError("predict method is not implemented!")
+
+    def finetune(self, X, y) -> None:
+        raise NotImplementedError("finetune method is not implemented!")
+
 
-    def init_env_and_metadata(self):
+class ModelCondaContainer(ModelContainer):
+    def __init__(self, model_config: dict, learnware_zippath: str, conda_env: str = None, build: bool = True):
+        self.conda_env = f"learnware_{shortuuid.uuid()}" if conda_env is None else conda_env
+        super(ModelCondaContainer, self).__init__(model_config, learnware_zippath, build)
+
+    def _init_env(self):
         install_environment(self.learnware_zippath, self.conda_env)
+
+    def _remove_env(self):
+        remove_enviroment(self.conda_env)
+
+    def _setup_env_and_metadata(self):
         with tempfile.TemporaryDirectory(prefix="learnware_") as tempdir:
             output_path = os.path.join(tempdir, "output.pkl")
             model_path = os.path.join(tempdir, "model.pkl")
@@ -42,7 +105,7 @@ class ModelEnvContainer(BaseModel):
                     "--no-capture-output",
                     "python3",
                     f"{self.model_script}",
-                    f"--model-path",
+                    "--model-path",
                     f"{model_path}",
                     "--output-path",
                     f"{output_path}",
@@ -54,14 +117,13 @@ class ModelEnvContainer(BaseModel):
 
         if output_results["status"] != "success":
             raise output_results["error_info"]
+
         input_shape = output_results["metadata"]["input_shape"]
         output_shape = output_results["metadata"]["output_shape"]
-        super(ModelEnvContainer, self).__init__(input_shape, output_shape)
+        logger.info(f"input_shape: {input_shape}, output_shape: {output_shape}")
+        self.reset(input_shape=input_shape, output_shape=output_shape)
 
-    def remove_env(self):
-        remove_enviroment(self.conda_env)
-
-    def run_model_with_script(self, method, **kargs):
+    def _run_model_with_script(self, method, **kargs):
         with tempfile.TemporaryDirectory(prefix="learnware_") as tempdir:
             input_path = os.path.join(tempdir, "input.pkl")
             output_path = os.path.join(tempdir, "output.pkl")
@@ -82,11 +144,11 @@ class ModelEnvContainer(BaseModel):
                     "--no-capture-output",
                     "python3",
                     f"{self.model_script}",
-                    f"--model-path",
+                    "--model-path",
                     f"{model_path}",
-                    f"--input-path",
+                    "--input-path",
                     f"{input_path}",
-                    f"--output-path",
+                    "--output-path",
                     f"{output_path}",
                 ]
             )
@@ -100,17 +162,321 @@ class ModelEnvContainer(BaseModel):
         return output_results[method]
 
     def fit(self, X, y):
-        self.run_model_with_script("fit", X=X, y=y)
+        self._run_model_with_script("fit", X=X, y=y)
+
+    def predict(self, X):
+        return self._run_model_with_script("predict", X=X)
+
+    def finetune(self, X, y) -> None:
+        self._run_model_with_script("finetune", X=X, y=y)
+
+
+class ModelDockerContainer(ModelContainer):
+    def __init__(
+        self,
+        model_config: dict,
+        learnware_zippath: str,
+        docker_container: object = None,
+        build: bool = True,
+    ):
+        """_summary_
+
+        Parameters
+        ----------
+        build : bool, optional
+            Whether to build the docker env, by default True
+        """
+
+        self.docker_container = docker_container
+        self.conda_env = f"learnware_{shortuuid.uuid()}"
+        self.docker_model_config = None
+        self.docker_model_script_path = None
+        # call init method of parent of parent class
+        super(ModelDockerContainer, self).__init__(model_config, learnware_zippath, build)
+
+    @staticmethod
+    def _generate_docker_container():
+        client = docker.from_env()
+        http_proxy = os.environ.get("http_proxy")
+        https_proxy = os.environ.get("https_proxy")
+
+        container_config = {
+            "image": "continuumio/miniconda3",
+            "network_mode": "host",
+            "detach": True,
+            "tty": True,
+            "command": "bash",
+            "environment": {"http_proxy": http_proxy, "https_proxy": https_proxy},
+        }
+        container = client.containers.run(**container_config)
+        environment_cmd = [
+            "pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple",
+            "conda config --add channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/free/",
+            "conda config --add channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/main/",
+            "conda config --add channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/pytorch/",
+            "conda config --set show_channel_urls yes",
+        ]
+        for _cmd in environment_cmd:
+            container.exec_run(_cmd)
+        return container
+
+    @staticmethod
+    def _destroy_docker_container(docker_container):
+        docker_container.stop()
+        docker_container.remove()
+
+    def _copy_file_to_container(self, local_path, container_path):
+        directory_path = os.path.dirname(container_path)
+        container_name = os.path.basename(container_path)
+        self.docker_container.exec_run(f"mkdir -p {directory_path}")
+
+        with tempfile.TemporaryDirectory(prefix="learnware_tar_") as tempdir:
+            # Create a temporary tar file
+            tar_file_path = os.path.join(tempdir, container_name + ".tar")
+            with tarfile.open(tar_file_path, "w") as tar:
+                tar.add(local_path, arcname=container_name)
+
+            # Transfer the tar file to container
+            with open(tar_file_path, "rb") as file_data:
+                self.docker_container.put_archive(directory_path, file_data.read())
+
+    def _copy_file_from_container(self, container_path, local_path):
+        try:
+            data, stat = self.docker_container.get_archive(container_path)
+            tar_local_file = local_path + ".tar"
+            with open(tar_local_file, "wb") as f:
+                for chunk in data:
+                    f.write(chunk)
+            with tarfile.open(tar_local_file, "r") as tar:
+                tar.extractall(os.path.dirname(tar_local_file))
+            os.remove(tar_local_file)
+        except docker.errors.NotFound as err:
+            logger.error(f"Copy file from container failed due to {err}")
+
+    def _install_environment(self, zip_path, conda_env):
+        """Install environment of a learnware in docker container
+
+        Parameters
+        ----------
+        zip_path : str
+            Path of the learnware zip file
+        conda_env : str
+            a new conda environment will be created with the given name
+
+        Raises
+        ------
+        Exception
+            Lack of the environment configuration file.
+        """
+        run_cmd_times = 5
+        with tempfile.TemporaryDirectory(prefix="learnware_") as tempdir:
+            with zipfile.ZipFile(file=zip_path, mode="r") as z_file:
+                logger.info(f"zip_file namelist: {z_file.namelist()}")
+
+                if "environment.yaml" in z_file.namelist():
+                    z_file.extract(member="environment.yaml", path=tempdir)
+                    yaml_path: str = os.path.join(tempdir, "environment.yaml")
+                    yaml_path_filter: str = os.path.join(tempdir, "environment_filter.yaml")
+                    logger.info(f"checking the avaliabe conda packages for {conda_env}")
+                    filter_nonexist_conda_packages_file(yaml_file=yaml_path, output_yaml_file=yaml_path_filter)
+                    self._copy_file_to_container(yaml_path_filter, yaml_path_filter)
+
+                    # create environment
+                    logger.info(f"create and update conda env [{conda_env}] according to .yaml file")
+                    for i in range(run_cmd_times):
+                        result = self.docker_container.exec_run(
+                            " ".join(
+                                ["conda", "env", "update", "--name", f"{conda_env}", "--file", f"{yaml_path_filter}"]
+                            )
+                        )
+                        if result.exit_code == 0:
+                            break
+
+                elif "requirements.txt" in z_file.namelist():
+                    z_file.extract(member="requirements.txt", path=tempdir)
+                    requirements_path: str = os.path.join(tempdir, "requirements.txt")
+                    requirements_path_filter: str = os.path.join(tempdir, "requirements_filter.txt")
+                    logger.info(f"checking the avaliabe pip packages for {conda_env}")
+                    filter_nonexist_pip_packages_file(
+                        requirements_file=requirements_path, output_file=requirements_path_filter
+                    )
+                    logger.info(f"create empty conda env [{conda_env}]")
+                    for i in range(run_cmd_times):
+                        result = self.docker_container.exec_run(
+                            " ".join(["conda", "create", "-y", "--name", f"{conda_env}", "python=3.8"])
+                        )
+                        if result.exit_code == 0:
+                            break
+                    logger.info(f"install pip requirements for conda env [{conda_env}]")
+
+                    self._copy_file_to_container(requirements_path_filter, requirements_path_filter)
+                    for i in range(run_cmd_times):
+                        result = self.docker_container.exec_run(
+                            " ".join(
+                                [
+                                    "conda",
+                                    "run",
+                                    "-n",
+                                    f"{conda_env}",
+                                    "--no-capture-output",
+                                    "python3",
+                                    "-m",
+                                    "pip",
+                                    "install",
+                                    "-r",
+                                    f"{requirements_path_filter}",
+                                ]
+                            )
+                        )
+                        if result.exit_code == 0:
+                            break
+                else:
+                    raise Exception("Environment.yaml or requirements.txt not found in the learnware zip file.")
+
+        logger.info(f"install learnware package for conda env [{conda_env}]")
+        for i in range(run_cmd_times):
+            result = self.docker_container.exec_run(
+                " ".join(
+                    [
+                        "conda",
+                        "run",
+                        "-n",
+                        f"{conda_env}",
+                        "--no-capture-output",
+                        "python3",
+                        "-m",
+                        "pip",
+                        "install",
+                        "learnware",
+                    ]
+                )
+            )
+            if result.exit_code == 0:
+                break
+
+    def _setup_env_and_metadata(self):
+        """setup env and set the input and output shape by communicating with docker"""
+        self._install_environment(self.learnware_zippath, self.conda_env)
+        with tempfile.TemporaryDirectory(prefix="learnware_") as tempdir:
+            output_path = os.path.join(tempdir, "output.pkl")
+            model_path = os.path.join(tempdir, "model.pkl")
+            self.docker_model_script_path = os.path.join(tempdir, "run_model.py")
+
+            docker_model_config = self.model_config.copy()
+            with tempfile.TemporaryDirectory(prefix="learnware_model_config_") as config_tempdir:
+                basename = os.path.basename(self.model_config["module_path"])
+                docker_model_config["module_path"] = os.path.join(config_tempdir, basename)
+                self._copy_file_to_container(
+                    os.path.dirname(self.model_config["module_path"]),
+                    os.path.dirname(docker_model_config["module_path"]),
+                )
+
+            self.docker_model_config = docker_model_config
+            with open(model_path, "wb") as model_fp:
+                pickle.dump(self.docker_model_config, model_fp)
+
+            self._copy_file_to_container(model_path, model_path)
+            self._copy_file_to_container(self.model_script, self.docker_model_script_path)
+            self.docker_container.exec_run(
+                " ".join(
+                    [
+                        "conda",
+                        "run",
+                        "-n",
+                        f"{self.conda_env}",
+                        "--no-capture-output",
+                        "python3",
+                        f"{self.docker_model_script_path}",
+                        "--model-path",
+                        f"{model_path}",
+                        "--output-path",
+                        f"{output_path}",
+                    ]
+                )
+            )
+            self._copy_file_from_container(output_path, output_path)
+
+            with open(output_path, "rb") as output_fp:
+                output_results = pickle.load(output_fp)
+
+        input_shape = output_results["metadata"]["input_shape"]
+        output_shape = output_results["metadata"]["output_shape"]
+        logger.info(f"input_shape: {input_shape}, output_shape: {output_shape}")
+        self.reset(input_shape=input_shape, output_shape=output_shape)
+
+    def _init_env(self):
+        """create docker container according to the str"""
+        self.docker_container = ModelDockerContainer._generate_docker_container()
+
+    def _remove_env(self):
+        """remove the docker container"""
+        ModelDockerContainer._destroy_docker_container(docker_container)
+
+    def _run_model_with_script(self, method, **kargs):
+        with tempfile.TemporaryDirectory(prefix="learnware_") as tempdir:
+            input_path = os.path.join(tempdir, "input.pkl")
+            output_path = os.path.join(tempdir, "output.pkl")
+            model_path = os.path.join(tempdir, "model.pkl")
+
+            with open(model_path, "wb") as model_fp:
+                pickle.dump(self.docker_model_config, model_fp)
+
+            with open(input_path, "wb") as input_fp:
+                pickle.dump({"method": method, "kargs": kargs}, input_fp)
+
+            self._copy_file_to_container(model_path, model_path)
+            self._copy_file_to_container(input_path, input_path)
+
+            self.docker_container.exec_run(
+                " ".join(
+                    [
+                        "conda",
+                        "run",
+                        "-n",
+                        f"{self.conda_env}",
+                        "--no-capture-output",
+                        "python3",
+                        f"{self.docker_model_script_path}",
+                        "--model-path",
+                        f"{model_path}",
+                        "--input-path",
+                        f"{input_path}",
+                        "--output-path",
+                        f"{output_path}",
+                    ]
+                )
+            )
+            self._copy_file_from_container(output_path, output_path)
+
+            with open(output_path, "rb") as output_fp:
+                output_results = pickle.load(output_fp)
+
+        if output_results["status"] != "success":
+            raise output_results["error_info"]
+
+        return output_results[method]
+
+    def fit(self, X, y):
+        """fit model by the communicating with docker"""
+        self._run_model_with_script("fit", X=X, y=y)
 
     def predict(self, X):
-        return self.run_model_with_script("predict", X=X)
+        """predict model by the communicating with docker"""
+        return self._run_model_with_script("predict", X=X)
 
     def finetune(self, X, y) -> None:
-        self.run_model_with_script("finetune", X=X, y=y)
+        """finetune model by the communicating with docker"""
+        self._run_model_with_script("finetune", X=X, y=y)
 
 
 class LearnwaresContainer:
-    def __init__(self, learnware_list: List[Learnware], learnware_zippaths: List[str]):
+    def __init__(
+        self,
+        learnwares: Union[List[Learnware], Learnware],
+        learnware_zippaths: Union[List[str], str],
+        cleanup=True,
+        mode="conda",
+    ):
         """The initializaiton method for base reuser
 
         Parameters
@@ -118,33 +484,88 @@ class LearnwaresContainer:
         learnware_list : List[Learnware]
             The learnware list to reuse and make predictions
         """
+        if isinstance(learnwares, Learnware):
+            learnwares = [learnwares]
+        if isinstance(learnware_zippaths, str):
+            learnware_zippaths = [learnware_zippaths]
+
         assert all(
-            [isinstance(_learnware.get_model(), dict) for _learnware in learnware_list]
+            [isinstance(_learnware.get_model(), dict) for _learnware in learnwares]
         ), "the learnwre model should not be instantiated for reuser with containter"
-        self.learnware_list = [
-            Learnware(
-                _learnware.id, ModelEnvContainer(_learnware.get_model(), _zippath), _learnware.get_specification()
+
+        self.mode = mode
+        assert self.mode in {"conda", "docker"}, f"mode must be in ['conda', 'docker'], should not be {self.mode}"
+        self.learnware_list = learnwares
+        self.learnware_zippaths = learnware_zippaths
+        self.cleanup = cleanup
+
+    def __enter__(self):
+        if self.mode == "conda":
+            self.learnware_containers = [
+                Learnware(
+                    _learnware.id, ModelCondaContainer(_learnware.get_model(), _zippath), _learnware.get_specification()
+                )
+                for _learnware, _zippath in zip(self.learnware_list, self.learnware_zippaths)
+            ]
+        else:
+            self._docker_container = ModelDockerContainer._generate_docker_container()
+            self.learnware_containers = [
+                Learnware(
+                    _learnware.id,
+                    ModelDockerContainer(_learnware.get_model(), _zippath, self._docker_container, build=False),
+                    _learnware.get_specification(),
+                )
+                for _learnware, _zippath in zip(self.learnware_list, self.learnware_zippaths)
+            ]
+
+        model_list = [_learnware.get_model() for _learnware in self.learnware_containers]
+        with ThreadPoolExecutor(max_workers=max(os.cpu_count() // 2, 1)) as executor:
+            results = executor.map(self._initialize_model_container, model_list)
+        self.results = list(results)
+
+        if sum(self.results) < len(self.learnware_list):
+            logger.warning(
+                f"{len(self.learnware_list) - sum(results)} of {len(self.learnware_list)} learnwares init failed! This learnware will be ignored"
             )
-            for _learnware, _zippath in zip(learnware_list, learnware_zippaths)
-        ]
+        return self
 
-        model_list = [_learnware.get_model() for _learnware in self.learnware_list]
-        with ProcessPoolExecutor(max_workers=max(os.cpu_count() // 2, 1)) as executor:
-            executor.map(self._initialize_model_container, model_list)
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if not self.cleanup:
+            logger.warning(f"Notice, the learnware container env is not cleaned up!")
+            self.learnware_containers = None
+            self.results = None
+            return
 
-        atexit.register(self.cleanup)
+        model_list = [_learnware.get_model() for _learnware in self.learnware_containers]
+        with ThreadPoolExecutor(max_workers=max(os.cpu_count() // 2, 1)) as executor:
+            executor.map(self._destroy_model_container, model_list)
 
-    @staticmethod
-    def _initialize_model_container(model: ModelEnvContainer):
-        model.init_env_and_metadata()
+        self.learnware_containers = None
+        self.results = None
+
+        if self.mode == "docker":
+            ModelDockerContainer._destroy_docker_container(self._docker_container)
 
     @staticmethod
-    def _destroy_model_container(model: ModelEnvContainer):
-        model.remove_env()
+    def _initialize_model_container(model: ModelCondaContainer):
+        try:
+            model.init_and_setup_env()
+        except Exception as err:
+            logger.error(f"build env {model.conda_env} failed due to {err}")
+            return False
+        return True
 
-    def get_learnware_list_with_container(self):
-        return self.learnware_list
+    @staticmethod
+    def _destroy_model_container(model: ModelCondaContainer):
+        try:
+            model.remove_env()
+        except Exception as err:
+            logger.error(f"remove env {model.conda_env} failed due to {err}")
+            return False
+        return True
 
-    def cleanup(self):
-        for _learnware in self.learnware_list:
-            self._destroy_model_container(_learnware.get_model())
+    def get_learnwares_with_container(self):
+        learnware_containers = [
+            _learnware for _learnware, _result in zip(self.learnware_containers, self.results) if _result is True
+        ]
+        return learnware_containers

learnware/client/learnware_client.py

@@ -401,8 +401,8 @@ class LearnwareClient:
                 for i in range(len(learnware_list)):
                     learnware_list[i].instantiate_model()
             elif runnable_option == "conda_env":
-                env_container = LearnwaresContainer(learnware_list, zip_paths)
-                learnware_list = env_container.get_learnware_list_with_container()
+                with LearnwaresContainer(learnware_list, zip_paths, cleanup=False) as env_container:
+                    learnware_list = env_container.get_learnwares_with_container()
 
         if len(learnware_list) == 1:
             return learnware_list[0]

learnware/client/utils.py

@@ -14,7 +14,7 @@ def system_execute(args, timeout=None):
     try:
         com_process.check_returncode()
     except subprocess.CalledProcessError as err:
-        print("System Execute Error:", str(com_process.stderr))
+        print("System Execute Error:", com_process.stderr.decode())
         raise err
 
 

learnware/test/__init__.py

@@ -0,0 +1 @@
+from .module import get_semantic_specification

tests/test_client/test_learnware.py → learnware/test/module.py

@@ -1,10 +1,4 @@
-import os
-
-import learnware
-from learnware.client.learnware_client import LearnwareClient
-
-
-if __name__ == "__main__":
+def get_semantic_specification():
     semantic_specification = dict()
     semantic_specification["Data"] = {"Type": "Class", "Values": ["Text"]}
     semantic_specification["Task"] = {"Type": "Class", "Values": ["Ranking"]}
@@ -12,8 +6,4 @@ if __name__ == "__main__":
     semantic_specification["Scenario"] = {"Type": "Tag", "Values": "Financial"}
     semantic_specification["Name"] = {"Type": "String", "Values": "test"}
     semantic_specification["Description"] = {"Type": "String", "Values": "test"}
-
-    zip_path = "test.zip"
-    client = LearnwareClient()
-    client.install_environment(zip_path)
-    client.test_learnware(zip_path, semantic_specification)
+    return semantic_specification

setup.py

@@ -68,6 +68,7 @@ REQUIRED = [
     "sqlalchemy>=2.0.21",
     "shortuuid>=1.0.11",
     "geatpy>=2.7.0",
+    "docker>=6.1.3",
 ]
 
 if get_platform() != MACOS:

tests/test_client/test_reuse.py

@@ -1,42 +0,0 @@
-import zipfile
-import numpy as np
-
-from learnware.learnware import get_learnware_from_dirpath, Learnware
-from learnware.market import EasyMarket
-from learnware.client.container import ModelEnvContainer, LearnwaresContainer
-from learnware.learnware.reuse import AveragingReuser
-
-if __name__ == "__main__":
-    semantic_specification = dict()
-    semantic_specification["Data"] = {"Type": "Class", "Values": ["Text"]}
-    semantic_specification["Task"] = {"Type": "Class", "Values": ["Ranking"]}
-    semantic_specification["Library"] = {"Type": "Class", "Values": ["Scikit-learn"]}
-    semantic_specification["Scenario"] = {"Type": "Tag", "Values": "Financial"}
-    semantic_specification["Name"] = {"Type": "String", "Values": "test"}
-    semantic_specification["Description"] = {"Type": "String", "Values": "test"}
-
-    zip_paths = [
-        "/home/bixd/workspace/learnware/Learnware/tests/test_learnware_client/rf_tic.zip",
-        "/home/bixd/workspace/learnware/Learnware/tests/test_learnware_client/svc_tic.zip",
-    ]
-    dir_paths = [
-        "/home/bixd/workspace/learnware/Learnware/tests/test_learnware_client/rf_tic",
-        "/home/bixd/workspace/learnware/Learnware/tests/test_learnware_client/svc_tic",
-    ]
-
-    learnware_list = []
-    for id, (zip_path, dir_path) in enumerate(zip(zip_paths, dir_paths)):
-        with zipfile.ZipFile(zip_path, "r") as z_file:
-            z_file.extractall(dir_path)
-
-        learnware = get_learnware_from_dirpath(f"test_id{id}", semantic_specification, dir_path)
-        learnware_list.append(learnware)
-
-    with LearnwaresContainer(learnware_list, zip_paths) as env_container:
-
-        learnware_list = env_container.get_learnware_list_with_container()
-        reuser = AveragingReuser(learnware_list, mode="vote_by_label")
-        input_array = np.random.randint(0, 3, size=(20, 9))
-        print(reuser.predict(input_array).argmax(axis=1))
-        for id, ind_learner in enumerate(learnware_list):
-            print(f"learner_{id}", reuser.predict(input_array).argmax(axis=1))

tests/test_learnware_client/test_docker.py

@@ -0,0 +1,33 @@
+import os
+import zipfile
+import numpy as np
+
+import learnware
+from learnware.client import LearnwareClient
+from learnware.client.container import LearnwaresContainer
+from learnware.learnware.reuse import AveragingReuser
+
+
+if __name__ == "__main__":
+    email = "liujd@lamda.nju.edu.cn"
+    token = "f7e647146a314c6e8b4e2e1079c4bca4"
+
+    client = LearnwareClient()
+    client.login(email, token)
+
+    root = os.path.dirname(__file__)
+    learnware_ids = ["00000084", "00000154", "00000155"]
+    zip_paths = [os.path.join(root, x) for x in ["1.zip", "2.zip", "3.zip"]]
+
+    for learnware_id, zip_path in zip(learnware_ids, zip_paths):
+        client.download_learnware(learnware_id, zip_path)
+
+    learnware_list = [client.load_learnware(learnware_path=zippath) for zippath in zip_paths]
+    with LearnwaresContainer(learnware_list, zip_paths, mode="docker") as env_container:
+        learnware_list = env_container.get_learnwares_with_container()
+        reuser = AveragingReuser(learnware_list, mode="vote_by_label")
+        input_array = np.random.random(size=(20, 13))
+        print(reuser.predict(input_array))
+
+        for learnware in learnware_list:
+            print(learnware.id, learnware.predict(input_array))

tests/test_learnware_client/test_learnware.py

@@ -0,0 +1,23 @@
+import os
+import zipfile
+import tempfile
+from learnware.learnware import get_learnware_from_dirpath
+from learnware.test import get_semantic_specification
+from learnware.client.container import LearnwaresContainer
+from learnware.market import EasyMarket
+
+if __name__ == "__main__":
+    semantic_specification = get_semantic_specification()
+
+    zip_path = "rf_tic.zip"
+    with tempfile.TemporaryDirectory(suffix="learnware") as tempdir:
+        learnware_dirpath = os.path.join(tempdir, "test")
+        with zipfile.ZipFile(zip_path, "r") as z_file:
+            z_file.extractall(learnware_dirpath)
+        learnware = get_learnware_from_dirpath(
+            id="test", semantic_spec=semantic_specification, learnware_dirpath=learnware_dirpath
+        )
+
+        with LearnwaresContainer(learnware, zip_path) as env_container:
+            learnware = env_container.get_learnwares_with_container()[0]
+            print(EasyMarket.check_learnware(learnware))

tests/test_client/test_load.py → tests/test_learnware_client/test_load.py

@@ -6,7 +6,7 @@ import numpy as np
 import learnware
 from learnware.learnware import get_learnware_from_dirpath
 from learnware.client import LearnwareClient
-from learnware.client.container import ModelEnvContainer, LearnwaresContainer
+from learnware.client.container import ModelCondaContainer, LearnwaresContainer
 from learnware.learnware.reuse import AveragingReuser
 
 
@@ -24,7 +24,7 @@ class TestLearnwareLoad(unittest.TestCase):
         self.zip_paths = [os.path.join(root, x) for x in ["1.zip", "2.zip", "3.zip"]]
 
     def test_load_single_learnware_by_zippath(self):
-        for (learnware_id, zip_path) in zip(self.learnware_ids, self.zip_paths):
+        for learnware_id, zip_path in zip(self.learnware_ids, self.zip_paths):
             self.client.download_learnware(learnware_id, zip_path)
 
         learnware_list = [
@@ -39,7 +39,7 @@ class TestLearnwareLoad(unittest.TestCase):
             print(learnware.id, learnware.predict(input_array))
 
     def test_load_multi_learnware_by_zippath(self):
-        for (learnware_id, zip_path) in zip(self.learnware_ids, self.zip_paths):
+        for learnware_id, zip_path in zip(self.learnware_ids, self.zip_paths):
             self.client.download_learnware(learnware_id, zip_path)
 
         learnware_list = self.client.load_learnware(learnware_path=self.zip_paths, runnable_option="conda_env")

tests/test_learnware_client/test_reuse.py

@@ -1,20 +1,13 @@
 import zipfile
 import numpy as np
 
-from learnware.learnware import get_learnware_from_dirpath, Learnware
-from learnware.market import EasyMarket
-from learnware.client.container import ModelEnvContainer, LearnwaresContainer
+from learnware.learnware import get_learnware_from_dirpath
+from learnware.client.container import LearnwaresContainer
 from learnware.learnware.reuse import AveragingReuser
+from learnware.test.module import get_semantic_specification
 
 if __name__ == "__main__":
-    semantic_specification = dict()
-    semantic_specification["Data"] = {"Type": "Class", "Values": ["Text"]}
-    semantic_specification["Task"] = {"Type": "Class", "Values": ["Ranking"]}
-    semantic_specification["Library"] = {"Type": "Class", "Values": ["Scikit-learn"]}
-    semantic_specification["Scenario"] = {"Type": "Tag", "Values": "Financial"}
-    semantic_specification["Name"] = {"Type": "String", "Values": "test"}
-    semantic_specification["Description"] = {"Type": "String", "Values": "test"}
-
+    semantic_specification = get_semantic_specification()
     zip_paths = [
         "/home/bixd/workspace/learnware/Learnware/tests/test_learnware_client/rf_tic.zip",
         "/home/bixd/workspace/learnware/Learnware/tests/test_learnware_client/svc_tic.zip",
@@ -33,7 +26,7 @@ if __name__ == "__main__":
         learnware_list.append(learnware)
 
     with LearnwaresContainer(learnware_list, zip_paths) as env_container:
-        learnware_list = env_container.get_learnware_list_with_container()
+        learnware_list = env_container.get_learnwares_with_container()
         reuser = AveragingReuser(learnware_list, mode="vote")
         input_array = np.random.randint(0, 3, size=(20, 9))
         print(reuser.predict(input_array).argmax(axis=1))

tests/test_workflow/test_workflow.py

@@ -68,6 +68,9 @@ class TestAllWorkflow(unittest.TestCase):
             yaml_file = os.path.join(dir_path, "learnware.yaml")
             copyfile(os.path.join(curr_root, "learnware_example/example.yaml"), yaml_file)  # cp example.yaml yaml_file
 
+            env_file = os.path.join(dir_path, "environment.yaml")
+            copyfile(os.path.join(curr_root, "learnware_example/environment.yaml"), env_file)
+
             zip_file = dir_path + ".zip"
             # zip -q -r -j zip_file dir_path
             with zipfile.ZipFile(zip_file, "w") as zip_obj: