From 41bd28fb0c68b5f31a24747a0ff3e54dd552c536 Mon Sep 17 00:00:00 2001 From: bxdd Date: Tue, 18 Apr 2023 17:05:15 +0800 Subject: [PATCH] [MNT] Black --- examples/example_image/get_data.py | 7 ++++-- examples/example_image/main.py | 20 ++++++++++++--- examples/example_m5/m5/utils.py | 6 ++--- examples/example_m5/main.py | 20 ++++++++++++--- examples/example_market_db/example_db.py | 5 +++- examples/example_pfs/main.py | 20 ++++++++++++--- .../example_pfs/pfs/pfs_cross_transfer.py | 4 +-- examples/workflow_by_code/main.py | 20 ++++++++++++--- learnware/config.py | 25 +++++++++++++++---- learnware/learnware/__init__.py | 5 +++- learnware/market/easy.py | 5 +++- learnware/specification/rkme.py | 6 +++-- 12 files changed, 110 insertions(+), 33 deletions(-) diff --git a/examples/example_image/get_data.py b/examples/example_image/get_data.py index e0ee9bc..c3af534 100644 --- a/examples/example_image/get_data.py +++ b/examples/example_image/get_data.py @@ -192,7 +192,7 @@ def get_zca_matrix(X, reg_coef=0.1): def layernorm_data(X): X_processed = X - torch.mean(X, [1, 2, 3], keepdim=True) - X_processed = X_processed / torch.sqrt(torch.sum(X_processed ** 2, [1, 2, 3], keepdim=True)) + X_processed = X_processed / torch.sqrt(torch.sum(X_processed**2, [1, 2, 3], keepdim=True)) return X_processed @@ -240,7 +240,10 @@ def augment(images, dc_aug_param, device): def scalefun(i): h = int((np.random.uniform(1 - scale, 1 + scale)) * shape[2]) w = int((np.random.uniform(1 - scale, 1 + scale)) * shape[2]) - tmp = F.interpolate(images[i : i + 1], [h, w],)[0] + tmp = F.interpolate( + images[i : i + 1], + [h, w], + )[0] mhw = max(h, w, shape[2], shape[3]) im_ = torch.zeros(shape[1], mhw, mhw, dtype=torch.float, device=device) r = int((mhw - h) / 2) diff --git a/examples/example_image/main.py b/examples/example_image/main.py index 813b8df..48b4a26 100644 --- a/examples/example_image/main.py +++ b/examples/example_image/main.py @@ -38,7 +38,10 @@ os.makedirs(model_save_root, exist_ok=True) semantic_specs = [ { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": {"Values": ["Classification"], "Type": "Class",}, + "Task": { + "Values": ["Classification"], + "Type": "Class", + }, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Nature"], "Type": "Tag"}, "Description": {"Values": "", "Type": "Description"}, @@ -46,7 +49,10 @@ semantic_specs = [ }, { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": {"Values": ["Classification"], "Type": "Class",}, + "Task": { + "Values": ["Classification"], + "Type": "Class", + }, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business", "Nature"], "Type": "Tag"}, "Description": {"Values": "", "Type": "Description"}, @@ -54,7 +60,10 @@ semantic_specs = [ }, { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": {"Values": ["Classification"], "Type": "Class",}, + "Task": { + "Values": ["Classification"], + "Type": "Class", + }, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "Description"}, @@ -64,7 +73,10 @@ semantic_specs = [ user_senmantic = { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": {"Values": ["Classification"], "Type": "Class",}, + "Task": { + "Values": ["Classification"], + "Type": "Class", + }, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "Description"}, diff --git a/examples/example_m5/m5/utils.py b/examples/example_m5/m5/utils.py index c21a297..721eee2 100644 --- a/examples/example_m5/m5/utils.py +++ b/examples/example_m5/m5/utils.py @@ -70,7 +70,7 @@ def measure_aux_algo(idx, test_sample, model): # Simple "Memory profilers" to see memory usage def get_memory_usage(): - return np.round(psutil.Process(os.getpid()).memory_info()[0] / 2.0 ** 30, 2) + return np.round(psutil.Process(os.getpid()).memory_info()[0] / 2.0**30, 2) def sizeof_fmt(num, suffix="B"): @@ -84,7 +84,7 @@ def sizeof_fmt(num, suffix="B"): # Memory Reducer def reduce_mem_usage(df, float16_flag=True, verbose=True): numerics = ["int16", "int32", "int64", "float16", "float32", "float64"] - start_mem = df.memory_usage().sum() / 1024 ** 2 + start_mem = df.memory_usage().sum() / 1024**2 for col in df.columns: col_type = df[col].dtypes if col_type in numerics: @@ -106,7 +106,7 @@ def reduce_mem_usage(df, float16_flag=True, verbose=True): df[col] = df[col].astype(np.float32) else: df[col] = df[col].astype(np.float64) - end_mem = df.memory_usage().sum() / 1024 ** 2 + end_mem = df.memory_usage().sum() / 1024**2 if verbose: print( "Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)".format( diff --git a/examples/example_m5/main.py b/examples/example_m5/main.py index 426241d..b14f194 100644 --- a/examples/example_m5/main.py +++ b/examples/example_m5/main.py @@ -15,7 +15,10 @@ from m5 import DataLoader semantic_specs = [ { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": {"Values": ["Classification"], "Type": "Class",}, + "Task": { + "Values": ["Classification"], + "Type": "Class", + }, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Nature"], "Type": "Tag"}, "Description": {"Values": "", "Type": "Description"}, @@ -23,7 +26,10 @@ semantic_specs = [ }, { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": {"Values": ["Classification"], "Type": "Class",}, + "Task": { + "Values": ["Classification"], + "Type": "Class", + }, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business", "Nature"], "Type": "Tag"}, "Description": {"Values": "", "Type": "Description"}, @@ -31,7 +37,10 @@ semantic_specs = [ }, { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": {"Values": ["Classification"], "Type": "Class",}, + "Task": { + "Values": ["Classification"], + "Type": "Class", + }, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "Description"}, @@ -41,7 +50,10 @@ semantic_specs = [ user_senmantic = { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": {"Values": ["Classification"], "Type": "Class",}, + "Task": { + "Values": ["Classification"], + "Type": "Class", + }, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "Description"}, diff --git a/examples/example_market_db/example_db.py b/examples/example_market_db/example_db.py index 771cf10..a7b0585 100644 --- a/examples/example_market_db/example_db.py +++ b/examples/example_market_db/example_db.py @@ -40,7 +40,10 @@ semantic_specs = [ user_senmantic = { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": {"Values": ["Classification"], "Type": "Class",}, + "Task": { + "Values": ["Classification"], + "Type": "Class", + }, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "Description"}, diff --git a/examples/example_pfs/main.py b/examples/example_pfs/main.py index d2db901..adbd211 100644 --- a/examples/example_pfs/main.py +++ b/examples/example_pfs/main.py @@ -15,7 +15,10 @@ from pfs import Dataloader semantic_specs = [ { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": {"Values": ["Classification"], "Type": "Class",}, + "Task": { + "Values": ["Classification"], + "Type": "Class", + }, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Nature"], "Type": "Tag"}, "Description": {"Values": "", "Type": "Description"}, @@ -23,7 +26,10 @@ semantic_specs = [ }, { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": {"Values": ["Classification"], "Type": "Class",}, + "Task": { + "Values": ["Classification"], + "Type": "Class", + }, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business", "Nature"], "Type": "Tag"}, "Description": {"Values": "", "Type": "Description"}, @@ -31,7 +37,10 @@ semantic_specs = [ }, { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": {"Values": ["Classification"], "Type": "Class",}, + "Task": { + "Values": ["Classification"], + "Type": "Class", + }, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "Description"}, @@ -41,7 +50,10 @@ semantic_specs = [ user_senmantic = { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": {"Values": ["Classification"], "Type": "Class",}, + "Task": { + "Values": ["Classification"], + "Type": "Class", + }, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "Description"}, diff --git a/examples/example_pfs/pfs/pfs_cross_transfer.py b/examples/example_pfs/pfs/pfs_cross_transfer.py index a106fb7..93a3fa3 100644 --- a/examples/example_pfs/pfs/pfs_cross_transfer.py +++ b/examples/example_pfs/pfs/pfs_cross_transfer.py @@ -67,7 +67,7 @@ def get_split_errs(algo): for tmp in range(len(proportion_list)): model = lgb.LGBMModel( boosting_type="gbdt", - num_leaves=2 ** 7 - 1, + num_leaves=2**7 - 1, learning_rate=0.01, objective="rmse", metric="rmse", @@ -119,7 +119,7 @@ def get_errors(algo): if algo == "lgb": model = lgb.LGBMModel( boosting_type="gbdt", - num_leaves=2 ** 7 - 1, + num_leaves=2**7 - 1, learning_rate=0.01, objective="rmse", metric="rmse", diff --git a/examples/workflow_by_code/main.py b/examples/workflow_by_code/main.py index 321530c..842d9aa 100644 --- a/examples/workflow_by_code/main.py +++ b/examples/workflow_by_code/main.py @@ -18,7 +18,10 @@ curr_root = os.path.dirname(os.path.abspath(__file__)) semantic_specs = [ { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": {"Values": ["Classification"], "Type": "Class",}, + "Task": { + "Values": ["Classification"], + "Type": "Class", + }, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Nature"], "Type": "Tag"}, "Description": {"Values": "", "Type": "Description"}, @@ -26,7 +29,10 @@ semantic_specs = [ }, { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": {"Values": ["Classification"], "Type": "Class",}, + "Task": { + "Values": ["Classification"], + "Type": "Class", + }, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business", "Nature"], "Type": "Tag"}, "Description": {"Values": "", "Type": "Description"}, @@ -34,7 +40,10 @@ semantic_specs = [ }, { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": {"Values": ["Classification"], "Type": "Class",}, + "Task": { + "Values": ["Classification"], + "Type": "Class", + }, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "Description"}, @@ -44,7 +53,10 @@ semantic_specs = [ user_senmantic = { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": {"Values": ["Classification"], "Type": "Class",}, + "Task": { + "Values": ["Classification"], + "Type": "Class", + }, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "Description"}, diff --git a/learnware/config.py b/learnware/config.py index d80798f..2580b5c 100644 --- a/learnware/config.py +++ b/learnware/config.py @@ -66,7 +66,10 @@ os.makedirs(LEARNWARE_FOLDER_POOL_PATH, exist_ok=True) os.makedirs(DATABASE_PATH, exist_ok=True) semantic_config = { - "Data": {"Values": ["Tabular", "Image", "Video", "Text", "Audio"], "Type": "Class",}, # Choose only one class + "Data": { + "Values": ["Tabular", "Image", "Video", "Text", "Audio"], + "Type": "Class", + }, # Choose only one class "Task": { "Values": [ "Classification", @@ -79,7 +82,10 @@ semantic_config = { ], "Type": "Class", # Choose only one class }, - "Device": {"Values": ["CPU", "GPU"], "Type": "Tag",}, # Choose one or more tags + "Device": { + "Values": ["CPU", "GPU"], + "Type": "Tag", + }, # Choose one or more tags "Scenario": { "Values": [ "Business", @@ -99,8 +105,14 @@ semantic_config = { ], "Type": "Tag", # Choose one or more tags }, - "Description": {"Values": None, "Type": "Description",}, - "Name": {"Values": None, "Type": "Name",}, + "Description": { + "Values": None, + "Type": "Description", + }, + "Name": { + "Values": None, + "Type": "Name", + }, } _DEFAULT_CONFIG = { @@ -111,7 +123,10 @@ _DEFAULT_CONFIG = { "learnware_pool_path": LEARNWARE_POOL_PATH, "learnware_zip_pool_path": LEARNWARE_ZIP_POOL_PATH, "learnware_folder_pool_path": LEARNWARE_FOLDER_POOL_PATH, - "learnware_folder_config": {"yaml_file": "learnware.yaml", "module_file": "__init__.py",}, + "learnware_folder_config": { + "yaml_file": "learnware.yaml", + "module_file": "__init__.py", + }, "database_path": DATABASE_PATH, "max_reduced_set_size": 1000000, } diff --git a/learnware/learnware/__init__.py b/learnware/learnware/__init__.py index 189bef3..c60630b 100644 --- a/learnware/learnware/__init__.py +++ b/learnware/learnware/__init__.py @@ -31,7 +31,10 @@ def get_learnware_from_dirpath(id: str, semantic_spec: dict, learnware_dirpath: The contructed learnware object, return None if build failed """ learnware_config = { - "model": {"class_name": "Model", "kwargs": {},}, + "model": { + "class_name": "Model", + "kwargs": {}, + }, "stat_specifications": [ { "module_path": "learnware.specification", diff --git a/learnware/market/easy.py b/learnware/market/easy.py index 7fa1ccf..0b4630a 100644 --- a/learnware/market/easy.py +++ b/learnware/market/easy.py @@ -144,7 +144,10 @@ class EasyMarket(BaseMarket): self.learnware_folder_list[id] = target_folder_dir self.count += 1 add_learnware_to_db( - id, semantic_spec=semantic_spec, zip_path=target_zip_dir, folder_path=target_folder_dir, + id, + semantic_spec=semantic_spec, + zip_path=target_zip_dir, + folder_path=target_folder_dir, ) return id, True diff --git a/learnware/specification/rkme.py b/learnware/specification/rkme.py index cb9d963..9370c96 100644 --- a/learnware/specification/rkme.py +++ b/learnware/specification/rkme.py @@ -354,7 +354,9 @@ class RKMEStatSpecification(BaseStatSpecification): rkme_to_save["beta"] = rkme_to_save["beta"].tolist() rkme_to_save["device"] = "gpu" if rkme_to_save["cuda_idx"] != -1 else "cpu" json.dump( - rkme_to_save, codecs.open(save_path, "w", encoding="utf-8"), separators=(",", ":"), + rkme_to_save, + codecs.open(save_path, "w", encoding="utf-8"), + separators=(",", ":"), ) def load(self, filepath: str) -> bool: @@ -442,7 +444,7 @@ def torch_rbf_kernel(x1, x2, gamma) -> torch.Tensor: """ x1 = x1.double() x2 = x2.double() - X12norm = torch.sum(x1 ** 2, 1, keepdim=True) - 2 * x1 @ x2.T + torch.sum(x2 ** 2, 1, keepdim=True).T + X12norm = torch.sum(x1**2, 1, keepdim=True) - 2 * x1 @ x2.T + torch.sum(x2**2, 1, keepdim=True).T return torch.exp(-X12norm * gamma)