diff --git a/learnware/client/learnware_client.py b/learnware/client/learnware_client.py index c829a35..7767dfb 100644 --- a/learnware/client/learnware_client.py +++ b/learnware/client/learnware_client.py @@ -104,13 +104,8 @@ class LearnwareClient: for chunk in file_chunks(learnware_file): response = requests.post( url_upload, - files={ - "chunk_file": chunk, - }, - data={ - "file_hash": file_hash, - "chunk_begin": begin, - }, + files={"chunk_file": chunk,}, + data={"file_hash": file_hash, "chunk_begin": begin,}, headers=self.headers, ) @@ -128,10 +123,7 @@ class LearnwareClient: response = requests.post( url_add, - json={ - "file_hash": file_hash, - "semantic_specification": json.dumps(semantic_specification), - }, + json={"file_hash": file_hash, "semantic_specification": json.dumps(semantic_specification),}, headers=self.headers, ) @@ -145,14 +137,7 @@ class LearnwareClient: def download_learnware(self, learnware_id, save_path): url = f"{self.host}/engine/download_learnware" - response = requests.get( - url, - params={ - "learnware_id": learnware_id, - }, - headers=self.headers, - stream=True, - ) + response = requests.get(url, params={"learnware_id": learnware_id,}, headers=self.headers, stream=True,) if response.status_code != 200: raise Exception("download failed: " + json.dumps(response.json())) @@ -425,6 +410,6 @@ class LearnwareClient: raise Exception("The learnware is not usable.") pass pass - + logger.info("test ok") - pass \ No newline at end of file + pass diff --git a/learnware/client/package_utils.py b/learnware/client/package_utils.py index 02b92f9..3caff09 100644 --- a/learnware/client/package_utils.py +++ b/learnware/client/package_utils.py @@ -6,6 +6,7 @@ from typing import List, Tuple from ..logger import get_module_logger + logger = get_module_logger("package_utils") @@ -27,7 +28,7 @@ def try_to_run(args, timeout=5, retry=5): def parse_pip_requirement(line: str): """Parse pip requirement line to package name """ - + line = line.strip() if len(line) == 0: @@ -192,4 +193,4 @@ def filter_nonexist_pip_packages_file(requirements_file: str, output_file: str): pass logger.info(f"exist packages: {packages}") - return exist_packages, nonexist_packages \ No newline at end of file + return exist_packages, nonexist_packages diff --git a/learnware/config.py b/learnware/config.py index e66140d..760598d 100644 --- a/learnware/config.py +++ b/learnware/config.py @@ -11,10 +11,10 @@ class Config: config_file = os.path.join(self.root_path, "config.json") if os.path.exists(config_file): - with open(config_file, "r") as f: - self.__dict__["_config"].update(json.load(f)) - pass - pass + with open(config_file, "r") as f: + self.__dict__["_config"].update(json.load(f)) + pass + pass def __getitem__(self, key): return self.__dict__["_config"][key] @@ -71,10 +71,7 @@ os.makedirs(ROOT_DIRPATH, exist_ok=True) os.makedirs(DATABASE_PATH, exist_ok=True) semantic_config = { - "Data": { - "Values": ["Table", "Image", "Video", "Text", "Audio"], - "Type": "Class", - }, # Choose only one class + "Data": {"Values": ["Table", "Image", "Video", "Text", "Audio"], "Type": "Class",}, # Choose only one class "Task": { "Values": [ "Classification", @@ -115,14 +112,8 @@ semantic_config = { ], "Type": "Tag", # Choose one or more tags }, - "Description": { - "Values": None, - "Type": "String", - }, - "Name": { - "Values": None, - "Type": "String", - }, + "Description": {"Values": None, "Type": "String",}, + "Name": {"Values": None, "Type": "String",}, } _DEFAULT_CONFIG = { @@ -134,13 +125,10 @@ _DEFAULT_CONFIG = { "learnware_pool_path": LEARNWARE_POOL_PATH, "learnware_zip_pool_path": LEARNWARE_ZIP_POOL_PATH, "learnware_folder_pool_path": LEARNWARE_FOLDER_POOL_PATH, - "learnware_folder_config": { - "yaml_file": "learnware.yaml", - "module_file": "__init__.py", - }, + "learnware_folder_config": {"yaml_file": "learnware.yaml", "module_file": "__init__.py",}, "database_url": f"sqlite:///{DATABASE_PATH}", "max_reduced_set_size": 1310720, - "backend_host": "http://www.lamda.nju.edu.cn/learnware/api" + "backend_host": "http://www.lamda.nju.edu.cn/learnware/api", } C = Config(_DEFAULT_CONFIG) diff --git a/learnware/learnware/__init__.py b/learnware/learnware/__init__.py index 06a489d..ddc0062 100644 --- a/learnware/learnware/__init__.py +++ b/learnware/learnware/__init__.py @@ -31,10 +31,7 @@ def get_learnware_from_dirpath(id: str, semantic_spec: dict, learnware_dirpath: The contructed learnware object, return None if build failed """ learnware_config = { - "model": { - "class_name": "Model", - "kwargs": {}, - }, + "model": {"class_name": "Model", "kwargs": {},}, "stat_specifications": [ { "module_path": "learnware.specification", diff --git a/learnware/market/database_ops.py b/learnware/market/database_ops.py index 1306260..a985b02 100644 --- a/learnware/market/database_ops.py +++ b/learnware/market/database_ops.py @@ -1,8 +1,6 @@ from sqlalchemy.ext.declarative import declarative_base from sqlalchemy import create_engine, text -from sqlalchemy import ( - Column, Integer, Text, DateTime, String -) +from sqlalchemy import Column, Integer, Text, DateTime, String import os import json from ..learnware import get_learnware_from_dirpath @@ -13,7 +11,7 @@ DeclarativeBase = declarative_base() class Learnware(DeclarativeBase): - __tablename__ = 'tb_learnware' + __tablename__ = "tb_learnware" id = Column(String(10), primary_key=True, nullable=False) semantic_spec = Column(Text, nullable=False) @@ -25,7 +23,6 @@ class Learnware(DeclarativeBase): class DatabaseOperations(object): - def __init__(self, url: str, database_name: str): if url.startswith("sqlite"): url = os.path.join(url, f"{database_name}.db") @@ -38,14 +35,13 @@ class DatabaseOperations(object): pass - def create_database_if_not_exists(self, url): database_exists = True if url.startswith("sqlite"): # it is sqlite start = url.find(":///") - path = url[start+4:] + path = url[start + 4 :] if os.path.exists(path): database_exists = True pass @@ -57,7 +53,7 @@ class DatabaseOperations(object): elif self.url.startswith("postgresql"): # it is postgresql dbname_start = url.rfind("/") - dbname = url[dbname_start+1:] + dbname = url[dbname_start + 1 :] url_no_dbname = url[:dbname_start] + "/postgres" engine = create_engine(url_no_dbname) @@ -65,14 +61,15 @@ class DatabaseOperations(object): result = conn.execute(text("SELECT datname FROM pg_database;")) db_list = set() - for row in result.fetchall(): + for row in result.fetchall(): db_list.add(row[0].lower()) pass if dbname.lower() not in db_list: database_exists = False conn.execution_options(isolation_level="AUTOCOMMIT").execute( - text("CREATE DATABASE {0};".format(dbname))) + text("CREATE DATABASE {0};".format(dbname)) + ) pass else: database_exists = True @@ -83,7 +80,7 @@ class DatabaseOperations(object): else: raise Exception(f"Unsupported database url: {self.url}") pass - + self.engine = create_engine(url, future=True) if not database_exists: @@ -103,22 +100,26 @@ class DatabaseOperations(object): semantic_spec_str = json.dumps(semantic_spec) conn.execute( text( - ("INSERT INTO tb_learnware (id, semantic_spec, zip_path, folder_path, use_flag)" - "VALUES (:id, :semantic_spec, :zip_path, :folder_path, :use_flag);") + ( + "INSERT INTO tb_learnware (id, semantic_spec, zip_path, folder_path, use_flag)" + "VALUES (:id, :semantic_spec, :zip_path, :folder_path, :use_flag);" + ) + ), + dict( + id=id, + semantic_spec=semantic_spec_str, + zip_path=zip_path, + folder_path=folder_path, + use_flag=use_flag, ), - dict(id=id, semantic_spec=semantic_spec_str, zip_path=zip_path, - folder_path=folder_path, use_flag=use_flag) ) conn.commit() pass pass - + def delete_learnware(self, id: str): with self.engine.connect() as conn: - conn.execute( - text("DELETE FROM tb_learnware WHERE id=:id;"), - dict(id=id) - ) + conn.execute(text("DELETE FROM tb_learnware WHERE id=:id;"), dict(id=id)) conn.commit() pass pass @@ -128,7 +129,7 @@ class DatabaseOperations(object): semantic_spec_str = json.dumps(semantic_spec) r = conn.execute( text("UPDATE tb_learnware SET semantic_spec=:semantic_spec WHERE id=:id;"), - dict(id=id, semantic_spec=semantic_spec_str) + dict(id=id, semantic_spec=semantic_spec_str), ) conn.commit() pass @@ -160,4 +161,4 @@ class DatabaseOperations(object): return learnware_list, zip_list, folder_list, max_count + 1 pass - pass \ No newline at end of file + pass diff --git a/learnware/market/easy.py b/learnware/market/easy.py index f106c14..15f7434 100644 --- a/learnware/market/easy.py +++ b/learnware/market/easy.py @@ -55,7 +55,7 @@ class EasyMarket(BaseMarket): self.learnware_folder_list = {} self.count = 0 self.semantic_spec_list = conf.semantic_specs - self.dbops = DatabaseOperations(conf.database_url, 'market_' + self.market_id) + self.dbops = DatabaseOperations(conf.database_url, "market_" + self.market_id) self.reload_market(rebuild=rebuild) # Automatically reload the market logger.info("Market Initialized!") @@ -105,8 +105,8 @@ class EasyMarket(BaseMarket): learnware_model = learnware.get_model() # check input shape - if semantic_spec['Data']['Values'][0] == 'Table': - input_shape = (semantic_spec['Input']['Dimension'], ) + if semantic_spec["Data"]["Values"][0] == "Table": + input_shape = (semantic_spec["Input"]["Dimension"],) else: input_shape = learnware_model.input_shape pass @@ -126,17 +126,17 @@ class EasyMarket(BaseMarket): if outputs.ndim == 1: outputs = outputs.reshape(-1, 1) pass - - if semantic_spec['Task']['Values'][0] in ('Classification', 'Regression', 'Feature Extraction'): + + if semantic_spec["Task"]["Values"][0] in ("Classification", "Regression", "Feature Extraction"): # check output type if isinstance(outputs, torch.Tensor): outputs = outputs.detach().cpu().numpy() if not isinstance(outputs, np.ndarray): logger.warning(f"The learnware [{learnware.id}] output must be np.ndarray or torch.Tensor") return cls.NONUSABLE_LEARNWARE - + # check output shape - output_dim = int(semantic_spec['Output']['Dimension']) + output_dim = int(semantic_spec["Output"]["Dimension"]) if outputs[0].shape[0] != output_dim: logger.warning(f"The learnware [{learnware.id}] input and output dimention is error") return cls.NONUSABLE_LEARNWARE @@ -236,7 +236,7 @@ class EasyMarket(BaseMarket): if new_learnware is None: return None, self.INVALID_LEARNWARE - + check_flag = self.check_learnware(new_learnware) self.dbops.add_learnware( @@ -617,10 +617,10 @@ class EasyMarket(BaseMarket): def _search_by_semantic_spec(self, learnware_list: List[Learnware], user_info: BaseUserInfo) -> List[Learnware]: def match_semantic_spec(semantic_spec1, semantic_spec2): - ''' + """ semantic_spec1: semantic spec input by user semantic_spec2: semantic spec in database - ''' + """ if semantic_spec1.keys() != semantic_spec2.keys(): # sematic spec in database may contain more keys than user input pass diff --git a/learnware/specification/rkme.py b/learnware/specification/rkme.py index 60b8446..6f9471a 100644 --- a/learnware/specification/rkme.py +++ b/learnware/specification/rkme.py @@ -215,16 +215,24 @@ class RKMEStatSpecification(BaseStatSpecification): grad_Z = torch.zeros_like(Z) for i in range(0, Z.shape[0], batch_size): - Z_ = Z[i: i + batch_size] - term_1 = torch.bmm(torch.unsqueeze((torch.unsqueeze(beta, dim=0) * torch_rbf_kernel(Z_, Z, gamma)), dim=1), - torch.unsqueeze(Z_, dim=1) - torch.unsqueeze(Z, dim=0)) + Z_ = Z[i : i + batch_size] + term_1 = torch.bmm( + torch.unsqueeze((torch.unsqueeze(beta, dim=0) * torch_rbf_kernel(Z_, Z, gamma)), dim=1), + torch.unsqueeze(Z_, dim=1) - torch.unsqueeze(Z, dim=0), + ) if alpha is not None: - term_2 = -2 * torch.bmm(torch.unsqueeze(alpha * torch_rbf_kernel(Z_, X, gamma), dim=1), - torch.unsqueeze(Z_, dim=1) - torch.unsqueeze(X, dim=0)) + term_2 = -2 * torch.bmm( + torch.unsqueeze(alpha * torch_rbf_kernel(Z_, X, gamma), dim=1), + torch.unsqueeze(Z_, dim=1) - torch.unsqueeze(X, dim=0), + ) else: - term_2 = -2 * torch.bmm(torch.unsqueeze(torch_rbf_kernel(Z_, X, gamma) / self.num_points, dim=1), - torch.unsqueeze(Z_, dim=1) - torch.unsqueeze(X, dim=0)) - grad_Z[i: i + batch_size] = -2 * gamma * torch.unsqueeze(beta[i: i + batch_size], dim=1) * torch.squeeze(term_1 + term_2) + term_2 = -2 * torch.bmm( + torch.unsqueeze(torch_rbf_kernel(Z_, X, gamma) / self.num_points, dim=1), + torch.unsqueeze(Z_, dim=1) - torch.unsqueeze(X, dim=0), + ) + grad_Z[i : i + batch_size] = ( + -2 * gamma * torch.unsqueeze(beta[i : i + batch_size], dim=1) * torch.squeeze(term_1 + term_2) + ) Z = Z - step_size * grad_Z self.z = Z @@ -420,9 +428,7 @@ class RKMEStatSpecification(BaseStatSpecification): rkme_to_save["beta"] = rkme_to_save["beta"].tolist() rkme_to_save["device"] = "gpu" if rkme_to_save["cuda_idx"] != -1 else "cpu" json.dump( - rkme_to_save, - codecs.open(save_path, "w", encoding="utf-8"), - separators=(",", ":"), + rkme_to_save, codecs.open(save_path, "w", encoding="utf-8"), separators=(",", ":"), ) def load(self, filepath: str) -> bool: @@ -515,7 +521,7 @@ def torch_rbf_kernel(x1, x2, gamma) -> torch.Tensor: """ x1 = x1.double() x2 = x2.double() - X12norm = torch.sum(x1**2, 1, keepdim=True) - 2 * x1 @ x2.T + torch.sum(x2**2, 1, keepdim=True).T + X12norm = torch.sum(x1 ** 2, 1, keepdim=True) - 2 * x1 @ x2.T + torch.sum(x2 ** 2, 1, keepdim=True).T return torch.exp(-X12norm * gamma) diff --git a/learnware/specification/utils.py b/learnware/specification/utils.py index b9bd9fb..c9a00be 100644 --- a/learnware/specification/utils.py +++ b/learnware/specification/utils.py @@ -28,7 +28,9 @@ def convert_to_numpy(data: Union[np.ndarray, pd.DataFrame, torch.Tensor]): elif isinstance(data, torch.Tensor): return data.detach().cpu().numpy() else: - raise TypeError("Unsupported data format. Please provide a NumPy array, a Pandas DataFrame, or a PyTorch Tensor.") + raise TypeError( + "Unsupported data format. Please provide a NumPy array, a Pandas DataFrame, or a PyTorch Tensor." + ) def generate_rkme_spec( @@ -77,12 +79,12 @@ def generate_rkme_spec( # Convert data type X = convert_to_numpy(X) X = np.ascontiguousarray(X).astype(np.float32) - + # Check reduced_set_size max_reduced_set_size = C.max_reduced_set_size if reduced_set_size * X[0].size > max_reduced_set_size: reduced_set_size = max(20, max_reduced_set_size // X[0].size) - + # Check cuda_idx if not torch.cuda.is_available() or cuda_idx == -1: cuda_idx = -1 @@ -90,7 +92,7 @@ def generate_rkme_spec( num_cuda_devices = torch.cuda.device_count() if cuda_idx is None or not (cuda_idx >= 0 and cuda_idx < num_cuda_devices): cuda_idx = 0 - + # Generate rkme spec rkme_spec = RKMEStatSpecification(gamma=gamma, cuda_idx=cuda_idx) rkme_spec.generate_stat_spec_from_data(X, reduced_set_size, step_size, steps, nonnegative_beta, reduce) @@ -113,4 +115,4 @@ def generate_stat_spec(X: np.ndarray) -> BaseStatSpecification: StatSpecification A StatSpecification object """ - return None \ No newline at end of file + return None