Merge pull request #4 from Learnware-LAMDA/bixd/dev

[MNT] black format

learnware/client/learnware_client.py

@@ -104,13 +104,8 @@ class LearnwareClient:
         for chunk in file_chunks(learnware_file):
             response = requests.post(
                 url_upload,
-                files={
-                    "chunk_file": chunk,
-                },
-                data={
-                    "file_hash": file_hash,
-                    "chunk_begin": begin,
-                },
+                files={"chunk_file": chunk,},
+                data={"file_hash": file_hash, "chunk_begin": begin,},
                 headers=self.headers,
             )
 
@@ -128,10 +123,7 @@ class LearnwareClient:
 
         response = requests.post(
             url_add,
-            json={
-                "file_hash": file_hash,
-                "semantic_specification": json.dumps(semantic_specification),
-            },
+            json={"file_hash": file_hash, "semantic_specification": json.dumps(semantic_specification),},
             headers=self.headers,
         )
 
@@ -145,14 +137,7 @@ class LearnwareClient:
     def download_learnware(self, learnware_id, save_path):
         url = f"{self.host}/engine/download_learnware"
 
-        response = requests.get(
-            url,
-            params={
-                "learnware_id": learnware_id,
-            },
-            headers=self.headers,
-            stream=True,
-        )
+        response = requests.get(url, params={"learnware_id": learnware_id,}, headers=self.headers, stream=True,)
 
         if response.status_code != 200:
             raise Exception("download failed: " + json.dumps(response.json()))
@@ -425,6 +410,6 @@ class LearnwareClient:
                     raise Exception("The learnware is not usable.")
                 pass
             pass
-        
+
         logger.info("test ok")
-        pass
+        pass

learnware/client/package_utils.py

@@ -6,6 +6,7 @@ from typing import List, Tuple
 
 
 from ..logger import get_module_logger
+
 logger = get_module_logger("package_utils")
 
 
@@ -27,7 +28,7 @@ def try_to_run(args, timeout=5, retry=5):
 def parse_pip_requirement(line: str):
     """Parse pip requirement line to package name
     """
-    
+
     line = line.strip()
 
     if len(line) == 0:
@@ -192,4 +193,4 @@ def filter_nonexist_pip_packages_file(requirements_file: str, output_file: str):
         pass
 
     logger.info(f"exist packages: {packages}")
-    return exist_packages, nonexist_packages
+    return exist_packages, nonexist_packages

learnware/config.py

@@ -11,10 +11,10 @@ class Config:
 
         config_file = os.path.join(self.root_path, "config.json")
         if os.path.exists(config_file):
-           with open(config_file, "r") as f:
-               self.__dict__["_config"].update(json.load(f))
-               pass
-           pass
+            with open(config_file, "r") as f:
+                self.__dict__["_config"].update(json.load(f))
+                pass
+            pass
 
     def __getitem__(self, key):
         return self.__dict__["_config"][key]
@@ -71,10 +71,7 @@ os.makedirs(ROOT_DIRPATH, exist_ok=True)
 os.makedirs(DATABASE_PATH, exist_ok=True)
 
 semantic_config = {
-    "Data": {
-        "Values": ["Table", "Image", "Video", "Text", "Audio"],
-        "Type": "Class",
-    },  # Choose only one class
+    "Data": {"Values": ["Table", "Image", "Video", "Text", "Audio"], "Type": "Class",},  # Choose only one class
     "Task": {
         "Values": [
             "Classification",
@@ -115,14 +112,8 @@ semantic_config = {
         ],
         "Type": "Tag",  # Choose one or more tags
     },
-    "Description": {
-        "Values": None,
-        "Type": "String",
-    },
-    "Name": {
-        "Values": None,
-        "Type": "String",
-    },
+    "Description": {"Values": None, "Type": "String",},
+    "Name": {"Values": None, "Type": "String",},
 }
 
 _DEFAULT_CONFIG = {
@@ -134,13 +125,10 @@ _DEFAULT_CONFIG = {
     "learnware_pool_path": LEARNWARE_POOL_PATH,
     "learnware_zip_pool_path": LEARNWARE_ZIP_POOL_PATH,
     "learnware_folder_pool_path": LEARNWARE_FOLDER_POOL_PATH,
-    "learnware_folder_config": {
-        "yaml_file": "learnware.yaml",
-        "module_file": "__init__.py",
-    },
+    "learnware_folder_config": {"yaml_file": "learnware.yaml", "module_file": "__init__.py",},
     "database_url": f"sqlite:///{DATABASE_PATH}",
     "max_reduced_set_size": 1310720,
-    "backend_host": "http://www.lamda.nju.edu.cn/learnware/api"
+    "backend_host": "http://www.lamda.nju.edu.cn/learnware/api",
 }
 
 C = Config(_DEFAULT_CONFIG)

learnware/learnware/__init__.py

@@ -31,10 +31,7 @@ def get_learnware_from_dirpath(id: str, semantic_spec: dict, learnware_dirpath:
         The contructed learnware object, return None if build failed
     """
     learnware_config = {
-        "model": {
-            "class_name": "Model",
-            "kwargs": {},
-        },
+        "model": {"class_name": "Model", "kwargs": {},},
         "stat_specifications": [
             {
                 "module_path": "learnware.specification",

learnware/market/database_ops.py

@@ -1,8 +1,6 @@
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy import create_engine, text
-from sqlalchemy import (
-    Column, Integer, Text, DateTime, String
-)
+from sqlalchemy import Column, Integer, Text, DateTime, String
 import os
 import json
 from ..learnware import get_learnware_from_dirpath
@@ -13,7 +11,7 @@ DeclarativeBase = declarative_base()
 
 
 class Learnware(DeclarativeBase):
-    __tablename__ = 'tb_learnware'
+    __tablename__ = "tb_learnware"
 
     id = Column(String(10), primary_key=True, nullable=False)
     semantic_spec = Column(Text, nullable=False)
@@ -25,7 +23,6 @@ class Learnware(DeclarativeBase):
 
 
 class DatabaseOperations(object):
-
     def __init__(self, url: str, database_name: str):
         if url.startswith("sqlite"):
             url = os.path.join(url, f"{database_name}.db")
@@ -38,14 +35,13 @@ class DatabaseOperations(object):
 
         pass
 
-    
     def create_database_if_not_exists(self, url):
         database_exists = True
 
         if url.startswith("sqlite"):
             # it is sqlite
             start = url.find(":///")
-            path = url[start+4:]
+            path = url[start + 4 :]
             if os.path.exists(path):
                 database_exists = True
                 pass
@@ -57,7 +53,7 @@ class DatabaseOperations(object):
         elif self.url.startswith("postgresql"):
             # it is postgresql
             dbname_start = url.rfind("/")
-            dbname = url[dbname_start+1:]
+            dbname = url[dbname_start + 1 :]
             url_no_dbname = url[:dbname_start] + "/postgres"
             engine = create_engine(url_no_dbname)
 
@@ -65,14 +61,15 @@ class DatabaseOperations(object):
                 result = conn.execute(text("SELECT datname FROM pg_database;"))
                 db_list = set()
 
-                for row in  result.fetchall():
+                for row in result.fetchall():
                     db_list.add(row[0].lower())
                     pass
 
                 if dbname.lower() not in db_list:
                     database_exists = False
                     conn.execution_options(isolation_level="AUTOCOMMIT").execute(
-                    text("CREATE DATABASE {0};".format(dbname)))
+                        text("CREATE DATABASE {0};".format(dbname))
+                    )
                     pass
                 else:
                     database_exists = True
@@ -83,7 +80,7 @@ class DatabaseOperations(object):
         else:
             raise Exception(f"Unsupported database url: {self.url}")
             pass
-        
+
         self.engine = create_engine(url, future=True)
 
         if not database_exists:
@@ -103,22 +100,26 @@ class DatabaseOperations(object):
             semantic_spec_str = json.dumps(semantic_spec)
             conn.execute(
                 text(
-                ("INSERT INTO tb_learnware (id, semantic_spec, zip_path, folder_path, use_flag)"
-                 "VALUES (:id, :semantic_spec, :zip_path, :folder_path, :use_flag);")
+                    (
+                        "INSERT INTO tb_learnware (id, semantic_spec, zip_path, folder_path, use_flag)"
+                        "VALUES (:id, :semantic_spec, :zip_path, :folder_path, :use_flag);"
+                    )
+                ),
+                dict(
+                    id=id,
+                    semantic_spec=semantic_spec_str,
+                    zip_path=zip_path,
+                    folder_path=folder_path,
+                    use_flag=use_flag,
                 ),
-                dict(id=id, semantic_spec=semantic_spec_str, zip_path=zip_path,
-                folder_path=folder_path, use_flag=use_flag)
             )
             conn.commit()
             pass
         pass
-    
+
     def delete_learnware(self, id: str):
         with self.engine.connect() as conn:
-            conn.execute(
-                text("DELETE FROM tb_learnware WHERE id=:id;"),
-                dict(id=id)
-            )
+            conn.execute(text("DELETE FROM tb_learnware WHERE id=:id;"), dict(id=id))
             conn.commit()
             pass
         pass
@@ -128,7 +129,7 @@ class DatabaseOperations(object):
             semantic_spec_str = json.dumps(semantic_spec)
             r = conn.execute(
                 text("UPDATE tb_learnware SET semantic_spec=:semantic_spec WHERE id=:id;"),
-                dict(id=id, semantic_spec=semantic_spec_str)
+                dict(id=id, semantic_spec=semantic_spec_str),
             )
             conn.commit()
             pass
@@ -160,4 +161,4 @@ class DatabaseOperations(object):
         return learnware_list, zip_list, folder_list, max_count + 1
         pass
 
-    pass
+    pass

learnware/market/easy.py

@@ -55,7 +55,7 @@ class EasyMarket(BaseMarket):
         self.learnware_folder_list = {}
         self.count = 0
         self.semantic_spec_list = conf.semantic_specs
-        self.dbops = DatabaseOperations(conf.database_url, 'market_' + self.market_id)
+        self.dbops = DatabaseOperations(conf.database_url, "market_" + self.market_id)
         self.reload_market(rebuild=rebuild)  # Automatically reload the market
         logger.info("Market Initialized!")
 
@@ -105,8 +105,8 @@ class EasyMarket(BaseMarket):
             learnware_model = learnware.get_model()
 
             # check input shape
-            if semantic_spec['Data']['Values'][0] == 'Table':
-                input_shape = (semantic_spec['Input']['Dimension'], )
+            if semantic_spec["Data"]["Values"][0] == "Table":
+                input_shape = (semantic_spec["Input"]["Dimension"],)
             else:
                 input_shape = learnware_model.input_shape
                 pass
@@ -126,17 +126,17 @@ class EasyMarket(BaseMarket):
             if outputs.ndim == 1:
                 outputs = outputs.reshape(-1, 1)
                 pass
-            
-            if semantic_spec['Task']['Values'][0] in ('Classification', 'Regression', 'Feature Extraction'):
+
+            if semantic_spec["Task"]["Values"][0] in ("Classification", "Regression", "Feature Extraction"):
                 # check output type
                 if isinstance(outputs, torch.Tensor):
                     outputs = outputs.detach().cpu().numpy()
                 if not isinstance(outputs, np.ndarray):
                     logger.warning(f"The learnware [{learnware.id}] output must be np.ndarray or torch.Tensor")
                     return cls.NONUSABLE_LEARNWARE
-                
+
                 # check output shape
-                output_dim = int(semantic_spec['Output']['Dimension'])
+                output_dim = int(semantic_spec["Output"]["Dimension"])
                 if outputs[0].shape[0] != output_dim:
                     logger.warning(f"The learnware [{learnware.id}] input and output dimention is error")
                     return cls.NONUSABLE_LEARNWARE
@@ -236,7 +236,7 @@ class EasyMarket(BaseMarket):
 
         if new_learnware is None:
             return None, self.INVALID_LEARNWARE
-        
+
         check_flag = self.check_learnware(new_learnware)
 
         self.dbops.add_learnware(
@@ -617,10 +617,10 @@ class EasyMarket(BaseMarket):
 
     def _search_by_semantic_spec(self, learnware_list: List[Learnware], user_info: BaseUserInfo) -> List[Learnware]:
         def match_semantic_spec(semantic_spec1, semantic_spec2):
-            '''
+            """
             semantic_spec1: semantic spec input by user
             semantic_spec2: semantic spec in database
-            '''
+            """
             if semantic_spec1.keys() != semantic_spec2.keys():
                 # sematic spec in database may contain more keys than user input
                 pass

learnware/specification/rkme.py

@@ -215,16 +215,24 @@ class RKMEStatSpecification(BaseStatSpecification):
 
         grad_Z = torch.zeros_like(Z)
         for i in range(0, Z.shape[0], batch_size):
-            Z_ = Z[i: i + batch_size]
-            term_1 = torch.bmm(torch.unsqueeze((torch.unsqueeze(beta, dim=0) * torch_rbf_kernel(Z_, Z, gamma)), dim=1),
-                               torch.unsqueeze(Z_, dim=1) - torch.unsqueeze(Z, dim=0))
+            Z_ = Z[i : i + batch_size]
+            term_1 = torch.bmm(
+                torch.unsqueeze((torch.unsqueeze(beta, dim=0) * torch_rbf_kernel(Z_, Z, gamma)), dim=1),
+                torch.unsqueeze(Z_, dim=1) - torch.unsqueeze(Z, dim=0),
+            )
             if alpha is not None:
-                term_2 = -2 * torch.bmm(torch.unsqueeze(alpha * torch_rbf_kernel(Z_, X, gamma), dim=1),
-                                        torch.unsqueeze(Z_, dim=1) - torch.unsqueeze(X, dim=0))
+                term_2 = -2 * torch.bmm(
+                    torch.unsqueeze(alpha * torch_rbf_kernel(Z_, X, gamma), dim=1),
+                    torch.unsqueeze(Z_, dim=1) - torch.unsqueeze(X, dim=0),
+                )
             else:
-                term_2 = -2 * torch.bmm(torch.unsqueeze(torch_rbf_kernel(Z_, X, gamma) / self.num_points, dim=1),
-                                        torch.unsqueeze(Z_, dim=1) - torch.unsqueeze(X, dim=0))
-            grad_Z[i: i + batch_size] = -2 * gamma * torch.unsqueeze(beta[i: i + batch_size], dim=1) * torch.squeeze(term_1 + term_2)
+                term_2 = -2 * torch.bmm(
+                    torch.unsqueeze(torch_rbf_kernel(Z_, X, gamma) / self.num_points, dim=1),
+                    torch.unsqueeze(Z_, dim=1) - torch.unsqueeze(X, dim=0),
+                )
+            grad_Z[i : i + batch_size] = (
+                -2 * gamma * torch.unsqueeze(beta[i : i + batch_size], dim=1) * torch.squeeze(term_1 + term_2)
+            )
 
         Z = Z - step_size * grad_Z
         self.z = Z
@@ -420,9 +428,7 @@ class RKMEStatSpecification(BaseStatSpecification):
         rkme_to_save["beta"] = rkme_to_save["beta"].tolist()
         rkme_to_save["device"] = "gpu" if rkme_to_save["cuda_idx"] != -1 else "cpu"
         json.dump(
-            rkme_to_save,
-            codecs.open(save_path, "w", encoding="utf-8"),
-            separators=(",", ":"),
+            rkme_to_save, codecs.open(save_path, "w", encoding="utf-8"), separators=(",", ":"),
         )
 
     def load(self, filepath: str) -> bool:
@@ -515,7 +521,7 @@ def torch_rbf_kernel(x1, x2, gamma) -> torch.Tensor:
     """
     x1 = x1.double()
     x2 = x2.double()
-    X12norm = torch.sum(x1**2, 1, keepdim=True) - 2 * x1 @ x2.T + torch.sum(x2**2, 1, keepdim=True).T
+    X12norm = torch.sum(x1 ** 2, 1, keepdim=True) - 2 * x1 @ x2.T + torch.sum(x2 ** 2, 1, keepdim=True).T
     return torch.exp(-X12norm * gamma)
 
 

learnware/specification/utils.py

@@ -28,7 +28,9 @@ def convert_to_numpy(data: Union[np.ndarray, pd.DataFrame, torch.Tensor]):
     elif isinstance(data, torch.Tensor):
         return data.detach().cpu().numpy()
     else:
-        raise TypeError("Unsupported data format. Please provide a NumPy array, a Pandas DataFrame, or a PyTorch Tensor.")
+        raise TypeError(
+            "Unsupported data format. Please provide a NumPy array, a Pandas DataFrame, or a PyTorch Tensor."
+        )
 
 
 def generate_rkme_spec(
@@ -77,12 +79,12 @@ def generate_rkme_spec(
     # Convert data type
     X = convert_to_numpy(X)
     X = np.ascontiguousarray(X).astype(np.float32)
-    
+
     # Check reduced_set_size
     max_reduced_set_size = C.max_reduced_set_size
     if reduced_set_size * X[0].size > max_reduced_set_size:
         reduced_set_size = max(20, max_reduced_set_size // X[0].size)
-    
+
     # Check cuda_idx
     if not torch.cuda.is_available() or cuda_idx == -1:
         cuda_idx = -1
@@ -90,7 +92,7 @@ def generate_rkme_spec(
         num_cuda_devices = torch.cuda.device_count()
         if cuda_idx is None or not (cuda_idx >= 0 and cuda_idx < num_cuda_devices):
             cuda_idx = 0
-    
+
     # Generate rkme spec
     rkme_spec = RKMEStatSpecification(gamma=gamma, cuda_idx=cuda_idx)
     rkme_spec.generate_stat_spec_from_data(X, reduced_set_size, step_size, steps, nonnegative_beta, reduce)
@@ -113,4 +115,4 @@ def generate_stat_spec(X: np.ndarray) -> BaseStatSpecification:
     StatSpecification
         A StatSpecification object
     """
-    return None
+    return None