beimingwu
/
learnware
Not watched
1
0
Fork 0
Code
Releases 1 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

[MNT] Format code

tags/v0.3.2
Gene 2 years ago
parent
5c8ab650ac
commit
6b3bfe14c4
3 changed files with 205 additions and 163 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +0
    -1
      learnware/client/__init__.py
  2. +151
    -119
      learnware/client/learnware_client.py
  3. +54
    -43
      learnware/client/package_utils.py

+ 0
- 1
learnware/client/__init__.py View File

@@ -1,2 +1 @@

from .learnware_client import LearnwareClient, SemanticSpecificationKey

+ 151
- 119
learnware/client/learnware_client.py View File

@@ -1,20 +1,22 @@
from ..specification import Specification
from ..config import C
from .. import learnware
from ..market.easy import EasyMarket
from . import package_utils
import requests
import os
import yaml
import json
from tqdm import tqdm
import zipfile
import hashlib
import os
import requests
import tempfile
import zipfile
import yaml
from enum import Enum
from tqdm import tqdm

from ..config import C
from .. import learnware
from . import package_utils
from ..market.easy import EasyMarket
from ..logger import get_module_logger
from ..specification import Specification

CHUNK_SIZE = 1024 * 1024
logger = get_module_logger("LearnwareClient")


def require_login(func):
@@ -22,6 +24,7 @@ def require_login(func):
if self.headers is None:
raise Exception("Please login first.")
return func(self, *args, **kwargs)

return wrapper


@@ -52,6 +55,7 @@ class SemanticSpecificationKey(Enum):
SENARIOES = "Scenario"
pass


class LearnwareClient:
def __init__(self, host=None):
self.headers = None
@@ -67,16 +71,15 @@ class LearnwareClient:

def login(self, email, token):
url = f"{self.host}/auth/login_by_token"
response = requests.post(url, json={'email': email, 'token': token})
response = requests.post(url, json={"email": email, "token": token})

result = response.json()
if result['code'] != 0:
raise Exception('login failed: ' + json.dumps(result))
token = result['data']['token']
self.headers = {'Authorization': f'Bearer {token}'}
if result["code"] != 0:
raise Exception("login failed: " + json.dumps(result))

token = result["data"]["token"]
self.headers = {"Authorization": f"Bearer {token}"}
pass

@require_login
@@ -84,11 +87,11 @@ class LearnwareClient:
url = f"{self.host}/auth/logout"
response = requests.post(url, headers=self.headers)
result = response.json()
if result['code'] != 0:
raise Exception('logout failed: ' + json.dumps(result))
if result["code"] != 0:
raise Exception("logout failed: " + json.dumps(result))
self.headers = None
pass
@require_login
def upload_learnware(self, semantic_specification, learnware_file):
file_hash = compute_file_hash(learnware_file)
@@ -99,71 +102,83 @@ class LearnwareClient:
bar = tqdm(total=num_chunks, desc="Uploading", unit="MB")
begin = 0
for chunk in file_chunks(learnware_file):
response = requests.post(url_upload, files={
"chunk_file": chunk,
}, data={
"file_hash": file_hash,
"chunk_begin": begin,
}, headers=self.headers)
response = requests.post(
url_upload,
files={
"chunk_file": chunk,
},
data={
"file_hash": file_hash,
"chunk_begin": begin,
},
headers=self.headers,
)

result = response.json()

if result['code'] != 0:
raise Exception('upload failed: ' + json.dumps(result))
if result["code"] != 0:
raise Exception("upload failed: " + json.dumps(result))
begin += len(chunk)
bar.update(1)
pass
bar.close()
url_add = f"{self.host}/user/add_learnware_uploaded"

response = requests.post(url_add, json={
"file_hash": file_hash,
"semantic_specification": json.dumps(semantic_specification),
}, headers=self.headers)
response = requests.post(
url_add,
json={
"file_hash": file_hash,
"semantic_specification": json.dumps(semantic_specification),
},
headers=self.headers,
)

result = response.json()

if result['code'] != 0:
raise Exception('upload failed: ' + json.dumps(result))
return result['data']['learnware_id']
if result["code"] != 0:
raise Exception("upload failed: " + json.dumps(result))
return result["data"]["learnware_id"]

def download_learnware(self, learnware_id, save_path):
url = f"{self.host}/engine/download_learnware"

response = requests.get(url, params={
"learnware_id": learnware_id,
}, headers=self.headers, stream=True)
response = requests.get(
url,
params={
"learnware_id": learnware_id,
},
headers=self.headers,
stream=True,
)

if response.status_code != 200:
raise Exception('download failed: ' + json.dumps(response.json()))
num_chunks = int(response.headers['Content-Length']) // CHUNK_SIZE + 1
raise Exception("download failed: " + json.dumps(response.json()))
num_chunks = int(response.headers["Content-Length"]) // CHUNK_SIZE + 1
bar = tqdm(total=num_chunks, desc="Downloading", unit="MB")

with open(save_path, 'wb') as f:
for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
with open(save_path, "wb") as f:
for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
f.write(chunk)
bar.update(1)
pass
pass
pass
@require_login
def list_learnware(self):
url = f"{self.host}/user/list_learnware"
response = requests.post(
url, json={'page': 0, 'limit': 10000}, headers=self.headers)
response = requests.post(url, json={"page": 0, "limit": 10000}, headers=self.headers)

result = response.json()

if result['code'] != 0:
raise Exception('list failed: ' + json.dumps(result))
learnware_list = result['data']['learnware_list']
if result["code"] != 0:
raise Exception("list failed: " + json.dumps(result))
learnware_list = result["data"]["learnware_list"]

return learnware_list

@@ -180,7 +195,7 @@ class LearnwareClient:
else:
stat_spec = None
pass
returns = []
with tempfile.NamedTemporaryFile(prefix="learnware_stat_", suffix=".json") as ftemp:
if stat_spec is not None:
@@ -192,7 +207,7 @@ class LearnwareClient:
if semantic_specification is None:
semantic_specification = {}
pass
semantic_specification.pop("Input", None)
semantic_specification.pop("Output", None)

@@ -203,21 +218,29 @@ class LearnwareClient:
pass

response = requests.post(
url, files=files,
data={"semantic_specification": json.dumps(specification.get_semantic_spec()), "limit": page_size, "page": page_index},
headers=self.headers)
url,
files=files,
data={
"semantic_specification": json.dumps(specification.get_semantic_spec()),
"limit": page_size,
"page": page_index,
},
headers=self.headers,
)

result = response.json()

if result['code'] != 0:
raise Exception('search failed: ' + json.dumps(result))
for learnware in result['data']['learnware_list_single']:
returns.append({
"learnware_id": learnware['learnware_id'],
"semantic_specification": learnware['semantic_specification'],
"matching": learnware['matching'],
})
if result["code"] != 0:
raise Exception("search failed: " + json.dumps(result))

for learnware in result["data"]["learnware_list_single"]:
returns.append(
{
"learnware_id": learnware["learnware_id"],
"semantic_specification": learnware["semantic_specification"],
"matching": learnware["matching"],
}
)
pass
pass
pass
@@ -227,13 +250,12 @@ class LearnwareClient:
@require_login
def delete_learnware(self, learnware_id):
url = f"{self.host}/user/delete_learnware"
response = requests.post(
url, json={'learnware_id': learnware_id}, headers=self.headers)
response = requests.post(url, json={"learnware_id": learnware_id}, headers=self.headers)

result = response.json()

if result['code'] != 0:
raise Exception('delete failed: ' + json.dumps(result))
if result["code"] != 0:
raise Exception("delete failed: " + json.dumps(result))
pass

def check_learnware(self, path, semantic_specification):
@@ -250,7 +272,7 @@ class LearnwareClient:
pass

def check_learnware_folder(self, folder, semantic_specification):
learnware_obj = learnware.get_learnware_from_dirpath('test_id', semantic_specification, folder)
learnware_obj = learnware.get_learnware_from_dirpath("test_id", semantic_specification, folder)

check_result = EasyMarket.check_learnware(learnware_obj)
if check_result == EasyMarket.USABLE_LEARWARE:
@@ -260,9 +282,9 @@ class LearnwareClient:
pass

def create_semantic_specification(
self, name, description, data_type, task_type, library_type, senarioes, input_description,
output_description):
self, name, description, data_type, task_type, library_type, senarioes, input_description, output_description
):
semantic_specification = dict()
semantic_specification["Input"] = input_description
semantic_specification["Output"] = output_description
@@ -271,20 +293,20 @@ class LearnwareClient:
semantic_specification["Library"] = {"Type": "Class", "Values": [library_type]}
semantic_specification["Scenario"] = {"Type": "Tag", "Values": senarioes}
semantic_specification["Name"] = {"Type": "String", "Values": name}
semantic_specification["Description"] = {"Type": "String", "Values": description}
semantic_specification["Description"] = {"Type": "String", "Values": description}
return semantic_specification
def list_semantic_specification_values(self, key: SemanticSpecificationKey):
url = f"{self.host}/engine/semantic_specification"
response = requests.get(url, headers=self.headers)
result = response.json()
semantic_conf = result['data']['semantic_specification']
semantic_conf = result["data"]["semantic_specification"]

return semantic_conf[key.value]['Values']
return semantic_conf[key.value]["Values"]

def load_learnware(self, learnware_file: str, load_model: bool=True):
with tempfile.TemporaryDirectory(prefix='learnware_') as tempdir:
def load_learnware(self, learnware_file: str, load_model: bool = True):
with tempfile.TemporaryDirectory(prefix="learnware_") as tempdir:
with zipfile.ZipFile(learnware_file, "r") as z_file:
z_file.extractall(tempdir)
pass
@@ -295,17 +317,17 @@ class LearnwareClient:
learnware_info = yaml.safe_load(fin)
pass

learnware_id = learnware_info.get('id')
learnware_id = learnware_info.get("id")
if learnware_id is None:
learnware_id = "test_id"
pass

semantic_specification = learnware_info.get('semantic_specification')
semantic_specification = learnware_info.get("semantic_specification")
if semantic_specification is None:
semantic_specification = {}
pass
else:
semantic_file = semantic_specification.get('file_name')
semantic_file = semantic_specification.get("file_name")

with open(os.path.join(tempdir, semantic_file), "r") as fin:
semantic_specification = json.load(fin)
@@ -328,45 +350,55 @@ class LearnwareClient:
raise RuntimeError(f"Command {command} failed with return code {retcd}")
pass


def install_environment(self, zip_path, conda_env=None):
'''install environment of a learnware

@param: zip_path: path of the learnware zip file
@param: conda_env: if it is not None, a new conda environment will be created with the given name
if it is None, use current environment
'''
with tempfile.TemporaryDirectory(prefix='learnware_') as tempdir:
"""Install environment of a learnware

Parameters
----------
zip_path : str
Path of the learnware zip file
conda_env : optional
If it is not None, a new conda environment will be created with the given name;
If it is None, use current environment.

Raises
------
Exception
Lack of the environment configuration file.
"""
with tempfile.TemporaryDirectory(prefix="learnware_") as tempdir:
with zipfile.ZipFile(zip_path, "r") as z_file:
print(z_file.namelist)
if 'environment.yaml' in z_file.namelist():
z_file.extract('environment.yaml', tempdir)
yaml_path = os.path.join(tempdir, 'environment.yaml')
yaml_path_filter = os.path.join(tempdir, 'environment_filter.yaml')
logger.info(f"zip_file namelist: {z_file.namelist}")
if "environment.yaml" in z_file.namelist():
z_file.extract("environment.yaml", tempdir)
yaml_path = os.path.join(tempdir, "environment.yaml")
yaml_path_filter = os.path.join(tempdir, "environment_filter.yaml")
package_utils.filter_nonexist_conda_packages_file(yaml_path, yaml_path_filter)
# create environment
if conda_env is not None:
self.system(f'conda env update --name {conda_env} --file {yaml_path_filter}')
self.system(f"conda env update --name {conda_env} --file {yaml_path_filter}")
pass
else:
self.system(f'conda env update --file {yaml_path_filter}')
self.system(f"conda env update --file {yaml_path_filter}")
pass
pass
elif 'requirements.txt' in z_file.namelist():
z_file.extract('requirements.txt', tempdir)
requirements_path = os.path.join(tempdir, 'requirements.txt')
requirements_path_filter = os.path.join(tempdir, 'requirements_filter.txt')
elif "requirements.txt" in z_file.namelist():
z_file.extract("requirements.txt", tempdir)
requirements_path = os.path.join(tempdir, "requirements.txt")
requirements_path_filter = os.path.join(tempdir, "requirements_filter.txt")
package_utils.filter_nonexist_pip_packages_file(requirements_path, requirements_path_filter)

if conda_env is not None:
self.system(f'conda create --name {conda_env}')
self.system(f'conda run --no-capture-output python3 -m pip install -r {requirements_path_filter}')
self.system(f"conda create --name {conda_env}")
self.system(
f"conda run --no-capture-output python3 -m pip install -r {requirements_path_filter}"
)
else:
self.system(f'python3 -m pip install -r {requirements_path_filter}')
self.system(f"python3 -m pip install -r {requirements_path_filter}")
pass
pass
else:
raise Exception("environment.yaml or requirements.txt not found in the learnware zip file.")
raise Exception("Environment.yaml or requirements.txt not found in the learnware zip file.")
pass
pass
pass
@@ -376,16 +408,16 @@ class LearnwareClient:
semantic_specification = dict()
pass

with tempfile.TemporaryDirectory(prefix='learnware_') as tempdir:
with tempfile.TemporaryDirectory(prefix="learnware_") as tempdir:
with zipfile.ZipFile(zip_path, "r") as z_file:
z_file.extractall(tempdir)
pass
learnware_obj = learnware.get_learnware_from_dirpath('test_id', semantic_specification, tempdir)
learnware_obj = learnware.get_learnware_from_dirpath("test_id", semantic_specification, tempdir)

if learnware_obj is None:
raise Exception("The learnware is not valid.")
learnware_obj.instantiate_model()

if len(semantic_specification) > 0:
@@ -393,6 +425,6 @@ class LearnwareClient:
raise Exception("The learnware is not usable.")
pass
pass
print('test ok')
pass
pass
logger.info("test ok")
pass

+ 54
- 43
learnware/client/package_utils.py View File

@@ -1,8 +1,12 @@
from typing import List, Tuple
import subprocess
import yaml
import os
import yaml
import time
import subprocess
from typing import List, Tuple


from ..logger import get_module_logger
logger = get_module_logger("package_utils")


def try_to_run(args, timeout=5, retry=5):
@@ -18,21 +22,22 @@ def try_to_run(args, timeout=5, retry=5):

if not sucess:
raise subprocess.TimeoutExpired(args, timeout)
pass

def parse_pip_requirement(line: str):
'''parse pip requirement line to package name
'''
"""Parse pip requirement line to package name
"""
line = line.strip()

if len(line) == 0:
return None

if line[0] in ('#', '-'):
if line[0] in ("#", "-"):
return None

package_str = line
for split_ch in ('=', '>', '<', '!', '~', ' '):
for split_ch in ("=", ">", "<", "!", "~", " "):
split_ch_index = package_str.find(split_ch)
if split_ch_index != -1:
package_str = package_str[:split_ch_index]
@@ -41,13 +46,14 @@ def parse_pip_requirement(line: str):

return package_str


def read_pip_packages_from_requirements(requirements_file: str) -> List[str]:
'''read requiremnts.txt and parse it to list
'''
"""Read requiremnts.txt and parse it to list
"""

packages = []
lines = []
with open(requirements_file, 'r') as fin:
with open(requirements_file, "r") as fin:
for line in fin:
package_str = parse_pip_requirement(line)
packages.append(package_str)
@@ -58,23 +64,25 @@ def read_pip_packages_from_requirements(requirements_file: str) -> List[str]:


def filter_nonexist_pip_packages(packages: list) -> Tuple[List[str], List[str]]:
'''filter non-exist pip requirements
"""Filter non-exist pip requirements

Returns:
Returns
-------
Tuple[List[str], List[str]]
exist_packages: list of exist packages
nonexist_packages: list of non-exist packages
'''
"""

exist_packages = []
nonexist_packages = []
for package in packages:
try:
# os.system("python3 -m pip index versions {0}".format(package))
print('check package existence: {0}'.format(package))
logger.info("check package existence: {0}".format(package))
try_to_run(args=["python3", "-m", "pip", "index", "versions", package], timeout=5)
exist_packages.append(package)
except Exception as e:
print(e)
logger.error(e)
nonexist_packages.append(package)
pass
pass
@@ -83,12 +91,14 @@ def filter_nonexist_pip_packages(packages: list) -> Tuple[List[str], List[str]]:


def filter_nonexist_conda_packages(packages: list) -> Tuple[List[str], List[str]]:
'''filter non-exist conda requirements
"""Filter non-exist conda requirements

Returns:
Returns
-------
Tuple[List[str], List[str]]
exist_packages: list of exist packages
nonexist_packages: list of non-exist packages
'''
"""

exist_packages = []
nonexist_packages = []
@@ -104,17 +114,22 @@ def filter_nonexist_conda_packages(packages: list) -> Tuple[List[str], List[str]
return exist_packages, nonexist_packages


def read_conda_packages_from_dict(
env_desc: dict) -> Tuple[List[str], List[str]]:
'''
def read_conda_packages_from_dict(env_desc: dict) -> Tuple[List[str], List[str]]:
"""Read conda packages

:param env_desc: dict of environment description
Parameters
----------
env_desc : dict
Dict of environment description

:return conda packages: list of conda packages
:return pip packages: list of pip packages
'''
Returns
-------
Tuple[List[str], List[str]]
conda_packages: list of conda packages
pip_packages: list of pip packages
"""

conda_packages = env_desc.get('dependencies')
conda_packages = env_desc.get("dependencies")
if conda_packages is None:
conda_packages = []
pip_packages = []
@@ -123,8 +138,8 @@ def read_conda_packages_from_dict(
pip_packages = []
conda_packages_ = []
for package in conda_packages:
if isinstance(package, dict) and 'pip' in package:
pip_packages = package['pip']
if isinstance(package, dict) and "pip" in package:
pip_packages = package["pip"]
pip_packages = [parse_pip_requirement(line) for line in pip_packages]
pass
elif isinstance(package, str):
@@ -136,11 +151,10 @@ def read_conda_packages_from_dict(
pass

return conda_packages, pip_packages
pass


def filter_nonexist_conda_packages_file(yaml_file: str, output_yaml_file: str):
with open(yaml_file, 'r') as fin:
with open(yaml_file, "r") as fin:
env_desc = yaml.safe_load(fin)
pass

@@ -149,36 +163,33 @@ def filter_nonexist_conda_packages_file(yaml_file: str, output_yaml_file: str):
conda_packages, nonexist_conda_packages = filter_nonexist_conda_packages(conda_packages)
pip_packages, nonexist_pip_packages = filter_nonexist_pip_packages(pip_packages)

env_desc['dependencies'] = conda_packages
env_desc["dependencies"] = conda_packages
if len(pip_packages) > 0:
env_desc['dependencies'].append({'pip': pip_packages})
env_desc["dependencies"].append({"pip": pip_packages})
pass

with open(output_yaml_file, 'w') as fout:
yaml.safe_dump(env_desc, fout)
with open(output_yaml_file, "w") as fout:
yaml.safe_dump(env_desc, fout)
pass

return conda_packages, pip_packages, nonexist_conda_packages, nonexist_pip_packages
pass


def filter_nonexist_pip_packages_file(requirements_file: str, output_file: str):
packages, lines = read_pip_packages_from_requirements(requirements_file)

exist_packages, nonexist_packages = filter_nonexist_pip_packages(packages)

exist_packages = set(exist_packages)

with open(output_file, 'w') as fout:
with open(output_file, "w") as fout:
for package, line in zip(packages, lines):
if package is not None and package in exist_packages:
fout.write(line + '\n')
fout.write(line + "\n")
pass
pass
pass
pass

print(f"exist packages: {packages}")
return exist_packages, nonexist_packages
pass
logger.info(f"exist packages: {packages}")
return exist_packages, nonexist_packages

Write Preview
Loading…
Cancel
Save