From 6b3bfe14c4f37155b7fa32effd52c47301043634 Mon Sep 17 00:00:00 2001 From: Gene Date: Sat, 9 Sep 2023 15:38:34 +0800 Subject: [PATCH] [MNT] Format code --- learnware/client/__init__.py | 1 - learnware/client/learnware_client.py | 270 +++++++++++++++------------ learnware/client/package_utils.py | 97 +++++----- 3 files changed, 205 insertions(+), 163 deletions(-) diff --git a/learnware/client/__init__.py b/learnware/client/__init__.py index 1270854..9849835 100644 --- a/learnware/client/__init__.py +++ b/learnware/client/__init__.py @@ -1,2 +1 @@ - from .learnware_client import LearnwareClient, SemanticSpecificationKey diff --git a/learnware/client/learnware_client.py b/learnware/client/learnware_client.py index 9257fd3..c829a35 100644 --- a/learnware/client/learnware_client.py +++ b/learnware/client/learnware_client.py @@ -1,20 +1,22 @@ -from ..specification import Specification -from ..config import C -from .. import learnware -from ..market.easy import EasyMarket -from . import package_utils -import requests +import os +import yaml import json -from tqdm import tqdm +import zipfile import hashlib -import os +import requests import tempfile -import zipfile -import yaml from enum import Enum +from tqdm import tqdm +from ..config import C +from .. import learnware +from . import package_utils +from ..market.easy import EasyMarket +from ..logger import get_module_logger +from ..specification import Specification CHUNK_SIZE = 1024 * 1024 +logger = get_module_logger("LearnwareClient") def require_login(func): @@ -22,6 +24,7 @@ def require_login(func): if self.headers is None: raise Exception("Please login first.") return func(self, *args, **kwargs) + return wrapper @@ -52,6 +55,7 @@ class SemanticSpecificationKey(Enum): SENARIOES = "Scenario" pass + class LearnwareClient: def __init__(self, host=None): self.headers = None @@ -67,16 +71,15 @@ class LearnwareClient: def login(self, email, token): url = f"{self.host}/auth/login_by_token" - - response = requests.post(url, json={'email': email, 'token': token}) + + response = requests.post(url, json={"email": email, "token": token}) result = response.json() - if result['code'] != 0: - raise Exception('login failed: ' + json.dumps(result)) - - - token = result['data']['token'] - self.headers = {'Authorization': f'Bearer {token}'} + if result["code"] != 0: + raise Exception("login failed: " + json.dumps(result)) + + token = result["data"]["token"] + self.headers = {"Authorization": f"Bearer {token}"} pass @require_login @@ -84,11 +87,11 @@ class LearnwareClient: url = f"{self.host}/auth/logout" response = requests.post(url, headers=self.headers) result = response.json() - if result['code'] != 0: - raise Exception('logout failed: ' + json.dumps(result)) + if result["code"] != 0: + raise Exception("logout failed: " + json.dumps(result)) self.headers = None pass - + @require_login def upload_learnware(self, semantic_specification, learnware_file): file_hash = compute_file_hash(learnware_file) @@ -99,71 +102,83 @@ class LearnwareClient: bar = tqdm(total=num_chunks, desc="Uploading", unit="MB") begin = 0 for chunk in file_chunks(learnware_file): - response = requests.post(url_upload, files={ - "chunk_file": chunk, - }, data={ - "file_hash": file_hash, - "chunk_begin": begin, - }, headers=self.headers) + response = requests.post( + url_upload, + files={ + "chunk_file": chunk, + }, + data={ + "file_hash": file_hash, + "chunk_begin": begin, + }, + headers=self.headers, + ) result = response.json() - if result['code'] != 0: - raise Exception('upload failed: ' + json.dumps(result)) - + if result["code"] != 0: + raise Exception("upload failed: " + json.dumps(result)) + begin += len(chunk) bar.update(1) pass bar.close() - + url_add = f"{self.host}/user/add_learnware_uploaded" - response = requests.post(url_add, json={ - "file_hash": file_hash, - "semantic_specification": json.dumps(semantic_specification), - }, headers=self.headers) + response = requests.post( + url_add, + json={ + "file_hash": file_hash, + "semantic_specification": json.dumps(semantic_specification), + }, + headers=self.headers, + ) result = response.json() - if result['code'] != 0: - raise Exception('upload failed: ' + json.dumps(result)) - - return result['data']['learnware_id'] + if result["code"] != 0: + raise Exception("upload failed: " + json.dumps(result)) + + return result["data"]["learnware_id"] - def download_learnware(self, learnware_id, save_path): url = f"{self.host}/engine/download_learnware" - response = requests.get(url, params={ - "learnware_id": learnware_id, - }, headers=self.headers, stream=True) + response = requests.get( + url, + params={ + "learnware_id": learnware_id, + }, + headers=self.headers, + stream=True, + ) if response.status_code != 200: - raise Exception('download failed: ' + json.dumps(response.json())) - - num_chunks = int(response.headers['Content-Length']) // CHUNK_SIZE + 1 + raise Exception("download failed: " + json.dumps(response.json())) + + num_chunks = int(response.headers["Content-Length"]) // CHUNK_SIZE + 1 bar = tqdm(total=num_chunks, desc="Downloading", unit="MB") - with open(save_path, 'wb') as f: - for chunk in response.iter_content(chunk_size=CHUNK_SIZE): + with open(save_path, "wb") as f: + for chunk in response.iter_content(chunk_size=CHUNK_SIZE): f.write(chunk) bar.update(1) pass pass pass - + @require_login def list_learnware(self): url = f"{self.host}/user/list_learnware" - response = requests.post( - url, json={'page': 0, 'limit': 10000}, headers=self.headers) + response = requests.post(url, json={"page": 0, "limit": 10000}, headers=self.headers) result = response.json() - if result['code'] != 0: - raise Exception('list failed: ' + json.dumps(result)) - - learnware_list = result['data']['learnware_list'] + if result["code"] != 0: + raise Exception("list failed: " + json.dumps(result)) + + learnware_list = result["data"]["learnware_list"] return learnware_list @@ -180,7 +195,7 @@ class LearnwareClient: else: stat_spec = None pass - + returns = [] with tempfile.NamedTemporaryFile(prefix="learnware_stat_", suffix=".json") as ftemp: if stat_spec is not None: @@ -192,7 +207,7 @@ class LearnwareClient: if semantic_specification is None: semantic_specification = {} pass - + semantic_specification.pop("Input", None) semantic_specification.pop("Output", None) @@ -203,21 +218,29 @@ class LearnwareClient: pass response = requests.post( - url, files=files, - data={"semantic_specification": json.dumps(specification.get_semantic_spec()), "limit": page_size, "page": page_index}, - headers=self.headers) - + url, + files=files, + data={ + "semantic_specification": json.dumps(specification.get_semantic_spec()), + "limit": page_size, + "page": page_index, + }, + headers=self.headers, + ) + result = response.json() - if result['code'] != 0: - raise Exception('search failed: ' + json.dumps(result)) - - for learnware in result['data']['learnware_list_single']: - returns.append({ - "learnware_id": learnware['learnware_id'], - "semantic_specification": learnware['semantic_specification'], - "matching": learnware['matching'], - }) + if result["code"] != 0: + raise Exception("search failed: " + json.dumps(result)) + + for learnware in result["data"]["learnware_list_single"]: + returns.append( + { + "learnware_id": learnware["learnware_id"], + "semantic_specification": learnware["semantic_specification"], + "matching": learnware["matching"], + } + ) pass pass pass @@ -227,13 +250,12 @@ class LearnwareClient: @require_login def delete_learnware(self, learnware_id): url = f"{self.host}/user/delete_learnware" - response = requests.post( - url, json={'learnware_id': learnware_id}, headers=self.headers) - + response = requests.post(url, json={"learnware_id": learnware_id}, headers=self.headers) + result = response.json() - if result['code'] != 0: - raise Exception('delete failed: ' + json.dumps(result)) + if result["code"] != 0: + raise Exception("delete failed: " + json.dumps(result)) pass def check_learnware(self, path, semantic_specification): @@ -250,7 +272,7 @@ class LearnwareClient: pass def check_learnware_folder(self, folder, semantic_specification): - learnware_obj = learnware.get_learnware_from_dirpath('test_id', semantic_specification, folder) + learnware_obj = learnware.get_learnware_from_dirpath("test_id", semantic_specification, folder) check_result = EasyMarket.check_learnware(learnware_obj) if check_result == EasyMarket.USABLE_LEARWARE: @@ -260,9 +282,9 @@ class LearnwareClient: pass def create_semantic_specification( - self, name, description, data_type, task_type, library_type, senarioes, input_description, - output_description): - + self, name, description, data_type, task_type, library_type, senarioes, input_description, output_description + ): + semantic_specification = dict() semantic_specification["Input"] = input_description semantic_specification["Output"] = output_description @@ -271,20 +293,20 @@ class LearnwareClient: semantic_specification["Library"] = {"Type": "Class", "Values": [library_type]} semantic_specification["Scenario"] = {"Type": "Tag", "Values": senarioes} semantic_specification["Name"] = {"Type": "String", "Values": name} - semantic_specification["Description"] = {"Type": "String", "Values": description} - + semantic_specification["Description"] = {"Type": "String", "Values": description} + return semantic_specification - + def list_semantic_specification_values(self, key: SemanticSpecificationKey): url = f"{self.host}/engine/semantic_specification" response = requests.get(url, headers=self.headers) result = response.json() - semantic_conf = result['data']['semantic_specification'] + semantic_conf = result["data"]["semantic_specification"] - return semantic_conf[key.value]['Values'] + return semantic_conf[key.value]["Values"] - def load_learnware(self, learnware_file: str, load_model: bool=True): - with tempfile.TemporaryDirectory(prefix='learnware_') as tempdir: + def load_learnware(self, learnware_file: str, load_model: bool = True): + with tempfile.TemporaryDirectory(prefix="learnware_") as tempdir: with zipfile.ZipFile(learnware_file, "r") as z_file: z_file.extractall(tempdir) pass @@ -295,17 +317,17 @@ class LearnwareClient: learnware_info = yaml.safe_load(fin) pass - learnware_id = learnware_info.get('id') + learnware_id = learnware_info.get("id") if learnware_id is None: learnware_id = "test_id" pass - semantic_specification = learnware_info.get('semantic_specification') + semantic_specification = learnware_info.get("semantic_specification") if semantic_specification is None: semantic_specification = {} pass else: - semantic_file = semantic_specification.get('file_name') + semantic_file = semantic_specification.get("file_name") with open(os.path.join(tempdir, semantic_file), "r") as fin: semantic_specification = json.load(fin) @@ -328,45 +350,55 @@ class LearnwareClient: raise RuntimeError(f"Command {command} failed with return code {retcd}") pass - def install_environment(self, zip_path, conda_env=None): - '''install environment of a learnware - - @param: zip_path: path of the learnware zip file - @param: conda_env: if it is not None, a new conda environment will be created with the given name - if it is None, use current environment - ''' - with tempfile.TemporaryDirectory(prefix='learnware_') as tempdir: + """Install environment of a learnware + + Parameters + ---------- + zip_path : str + Path of the learnware zip file + conda_env : optional + If it is not None, a new conda environment will be created with the given name; + If it is None, use current environment. + + Raises + ------ + Exception + Lack of the environment configuration file. + """ + with tempfile.TemporaryDirectory(prefix="learnware_") as tempdir: with zipfile.ZipFile(zip_path, "r") as z_file: - print(z_file.namelist) - if 'environment.yaml' in z_file.namelist(): - z_file.extract('environment.yaml', tempdir) - yaml_path = os.path.join(tempdir, 'environment.yaml') - yaml_path_filter = os.path.join(tempdir, 'environment_filter.yaml') + logger.info(f"zip_file namelist: {z_file.namelist}") + if "environment.yaml" in z_file.namelist(): + z_file.extract("environment.yaml", tempdir) + yaml_path = os.path.join(tempdir, "environment.yaml") + yaml_path_filter = os.path.join(tempdir, "environment_filter.yaml") package_utils.filter_nonexist_conda_packages_file(yaml_path, yaml_path_filter) # create environment if conda_env is not None: - self.system(f'conda env update --name {conda_env} --file {yaml_path_filter}') + self.system(f"conda env update --name {conda_env} --file {yaml_path_filter}") pass else: - self.system(f'conda env update --file {yaml_path_filter}') + self.system(f"conda env update --file {yaml_path_filter}") pass pass - elif 'requirements.txt' in z_file.namelist(): - z_file.extract('requirements.txt', tempdir) - requirements_path = os.path.join(tempdir, 'requirements.txt') - requirements_path_filter = os.path.join(tempdir, 'requirements_filter.txt') + elif "requirements.txt" in z_file.namelist(): + z_file.extract("requirements.txt", tempdir) + requirements_path = os.path.join(tempdir, "requirements.txt") + requirements_path_filter = os.path.join(tempdir, "requirements_filter.txt") package_utils.filter_nonexist_pip_packages_file(requirements_path, requirements_path_filter) if conda_env is not None: - self.system(f'conda create --name {conda_env}') - self.system(f'conda run --no-capture-output python3 -m pip install -r {requirements_path_filter}') + self.system(f"conda create --name {conda_env}") + self.system( + f"conda run --no-capture-output python3 -m pip install -r {requirements_path_filter}" + ) else: - self.system(f'python3 -m pip install -r {requirements_path_filter}') + self.system(f"python3 -m pip install -r {requirements_path_filter}") pass pass else: - raise Exception("environment.yaml or requirements.txt not found in the learnware zip file.") + raise Exception("Environment.yaml or requirements.txt not found in the learnware zip file.") pass pass pass @@ -376,16 +408,16 @@ class LearnwareClient: semantic_specification = dict() pass - with tempfile.TemporaryDirectory(prefix='learnware_') as tempdir: + with tempfile.TemporaryDirectory(prefix="learnware_") as tempdir: with zipfile.ZipFile(zip_path, "r") as z_file: z_file.extractall(tempdir) pass - - learnware_obj = learnware.get_learnware_from_dirpath('test_id', semantic_specification, tempdir) + + learnware_obj = learnware.get_learnware_from_dirpath("test_id", semantic_specification, tempdir) if learnware_obj is None: raise Exception("The learnware is not valid.") - + learnware_obj.instantiate_model() if len(semantic_specification) > 0: @@ -393,6 +425,6 @@ class LearnwareClient: raise Exception("The learnware is not usable.") pass pass - print('test ok') - pass - pass \ No newline at end of file + + logger.info("test ok") + pass \ No newline at end of file diff --git a/learnware/client/package_utils.py b/learnware/client/package_utils.py index 6b5ca84..02b92f9 100644 --- a/learnware/client/package_utils.py +++ b/learnware/client/package_utils.py @@ -1,8 +1,12 @@ -from typing import List, Tuple -import subprocess -import yaml import os +import yaml import time +import subprocess +from typing import List, Tuple + + +from ..logger import get_module_logger +logger = get_module_logger("package_utils") def try_to_run(args, timeout=5, retry=5): @@ -18,21 +22,22 @@ def try_to_run(args, timeout=5, retry=5): if not sucess: raise subprocess.TimeoutExpired(args, timeout) - pass + def parse_pip_requirement(line: str): - '''parse pip requirement line to package name - ''' + """Parse pip requirement line to package name + """ + line = line.strip() if len(line) == 0: return None - if line[0] in ('#', '-'): + if line[0] in ("#", "-"): return None package_str = line - for split_ch in ('=', '>', '<', '!', '~', ' '): + for split_ch in ("=", ">", "<", "!", "~", " "): split_ch_index = package_str.find(split_ch) if split_ch_index != -1: package_str = package_str[:split_ch_index] @@ -41,13 +46,14 @@ def parse_pip_requirement(line: str): return package_str + def read_pip_packages_from_requirements(requirements_file: str) -> List[str]: - '''read requiremnts.txt and parse it to list - ''' + """Read requiremnts.txt and parse it to list + """ packages = [] lines = [] - with open(requirements_file, 'r') as fin: + with open(requirements_file, "r") as fin: for line in fin: package_str = parse_pip_requirement(line) packages.append(package_str) @@ -58,23 +64,25 @@ def read_pip_packages_from_requirements(requirements_file: str) -> List[str]: def filter_nonexist_pip_packages(packages: list) -> Tuple[List[str], List[str]]: - '''filter non-exist pip requirements + """Filter non-exist pip requirements - Returns: + Returns + ------- + Tuple[List[str], List[str]] exist_packages: list of exist packages nonexist_packages: list of non-exist packages - ''' + """ exist_packages = [] nonexist_packages = [] for package in packages: try: # os.system("python3 -m pip index versions {0}".format(package)) - print('check package existence: {0}'.format(package)) + logger.info("check package existence: {0}".format(package)) try_to_run(args=["python3", "-m", "pip", "index", "versions", package], timeout=5) exist_packages.append(package) except Exception as e: - print(e) + logger.error(e) nonexist_packages.append(package) pass pass @@ -83,12 +91,14 @@ def filter_nonexist_pip_packages(packages: list) -> Tuple[List[str], List[str]]: def filter_nonexist_conda_packages(packages: list) -> Tuple[List[str], List[str]]: - '''filter non-exist conda requirements + """Filter non-exist conda requirements - Returns: + Returns + ------- + Tuple[List[str], List[str]] exist_packages: list of exist packages nonexist_packages: list of non-exist packages - ''' + """ exist_packages = [] nonexist_packages = [] @@ -104,17 +114,22 @@ def filter_nonexist_conda_packages(packages: list) -> Tuple[List[str], List[str] return exist_packages, nonexist_packages -def read_conda_packages_from_dict( - env_desc: dict) -> Tuple[List[str], List[str]]: - ''' +def read_conda_packages_from_dict(env_desc: dict) -> Tuple[List[str], List[str]]: + """Read conda packages - :param env_desc: dict of environment description + Parameters + ---------- + env_desc : dict + Dict of environment description - :return conda packages: list of conda packages - :return pip packages: list of pip packages - ''' + Returns + ------- + Tuple[List[str], List[str]] + conda_packages: list of conda packages + pip_packages: list of pip packages + """ - conda_packages = env_desc.get('dependencies') + conda_packages = env_desc.get("dependencies") if conda_packages is None: conda_packages = [] pip_packages = [] @@ -123,8 +138,8 @@ def read_conda_packages_from_dict( pip_packages = [] conda_packages_ = [] for package in conda_packages: - if isinstance(package, dict) and 'pip' in package: - pip_packages = package['pip'] + if isinstance(package, dict) and "pip" in package: + pip_packages = package["pip"] pip_packages = [parse_pip_requirement(line) for line in pip_packages] pass elif isinstance(package, str): @@ -136,11 +151,10 @@ def read_conda_packages_from_dict( pass return conda_packages, pip_packages - pass def filter_nonexist_conda_packages_file(yaml_file: str, output_yaml_file: str): - with open(yaml_file, 'r') as fin: + with open(yaml_file, "r") as fin: env_desc = yaml.safe_load(fin) pass @@ -149,36 +163,33 @@ def filter_nonexist_conda_packages_file(yaml_file: str, output_yaml_file: str): conda_packages, nonexist_conda_packages = filter_nonexist_conda_packages(conda_packages) pip_packages, nonexist_pip_packages = filter_nonexist_pip_packages(pip_packages) - env_desc['dependencies'] = conda_packages + env_desc["dependencies"] = conda_packages if len(pip_packages) > 0: - env_desc['dependencies'].append({'pip': pip_packages}) + env_desc["dependencies"].append({"pip": pip_packages}) pass - with open(output_yaml_file, 'w') as fout: - yaml.safe_dump(env_desc, fout) + with open(output_yaml_file, "w") as fout: + yaml.safe_dump(env_desc, fout) pass return conda_packages, pip_packages, nonexist_conda_packages, nonexist_pip_packages - pass def filter_nonexist_pip_packages_file(requirements_file: str, output_file: str): - + packages, lines = read_pip_packages_from_requirements(requirements_file) exist_packages, nonexist_packages = filter_nonexist_pip_packages(packages) exist_packages = set(exist_packages) - with open(output_file, 'w') as fout: + with open(output_file, "w") as fout: for package, line in zip(packages, lines): if package is not None and package in exist_packages: - fout.write(line + '\n') + fout.write(line + "\n") pass pass pass - pass - print(f"exist packages: {packages}") - return exist_packages, nonexist_packages - pass + logger.info(f"exist packages: {packages}") + return exist_packages, nonexist_packages \ No newline at end of file