From 3244404cbfc09d1cf3d9f37385048fe6af56f9cd Mon Sep 17 00:00:00 2001 From: xiey Date: Tue, 18 Apr 2023 16:59:54 +0800 Subject: [PATCH] [FIX] fix bug in _selector_grid_search: Unknown parameter: booster --- examples/example_image/get_data.py | 7 ++---- examples/example_image/main.py | 20 +++------------ examples/example_m5/m5/utils.py | 6 ++--- examples/example_m5/main.py | 20 +++------------ examples/example_market_db/example_db.py | 5 +--- examples/example_pfs/main.py | 20 +++------------ .../example_pfs/pfs/pfs_cross_transfer.py | 4 +-- examples/workflow_by_code/main.py | 20 +++------------ learnware/config.py | 25 ++++--------------- learnware/learnware/__init__.py | 5 +--- learnware/learnware/reuse.py | 4 +-- learnware/market/easy.py | 5 +--- learnware/specification/rkme.py | 6 ++--- 13 files changed, 35 insertions(+), 112 deletions(-) diff --git a/examples/example_image/get_data.py b/examples/example_image/get_data.py index c3af534..e0ee9bc 100644 --- a/examples/example_image/get_data.py +++ b/examples/example_image/get_data.py @@ -192,7 +192,7 @@ def get_zca_matrix(X, reg_coef=0.1): def layernorm_data(X): X_processed = X - torch.mean(X, [1, 2, 3], keepdim=True) - X_processed = X_processed / torch.sqrt(torch.sum(X_processed**2, [1, 2, 3], keepdim=True)) + X_processed = X_processed / torch.sqrt(torch.sum(X_processed ** 2, [1, 2, 3], keepdim=True)) return X_processed @@ -240,10 +240,7 @@ def augment(images, dc_aug_param, device): def scalefun(i): h = int((np.random.uniform(1 - scale, 1 + scale)) * shape[2]) w = int((np.random.uniform(1 - scale, 1 + scale)) * shape[2]) - tmp = F.interpolate( - images[i : i + 1], - [h, w], - )[0] + tmp = F.interpolate(images[i : i + 1], [h, w],)[0] mhw = max(h, w, shape[2], shape[3]) im_ = torch.zeros(shape[1], mhw, mhw, dtype=torch.float, device=device) r = int((mhw - h) / 2) diff --git a/examples/example_image/main.py b/examples/example_image/main.py index 298d7d8..1487d38 100644 --- a/examples/example_image/main.py +++ b/examples/example_image/main.py @@ -38,10 +38,7 @@ os.makedirs(model_save_root, exist_ok=True) semantic_specs = [ { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class",}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Nature"], "Type": "Tag"}, "Description": {"Values": "", "Type": "Description"}, @@ -49,10 +46,7 @@ semantic_specs = [ }, { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class",}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business", "Nature"], "Type": "Tag"}, "Description": {"Values": "", "Type": "Description"}, @@ -60,10 +54,7 @@ semantic_specs = [ }, { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class",}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "Description"}, @@ -73,10 +64,7 @@ semantic_specs = [ user_senmantic = { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class",}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "Description"}, diff --git a/examples/example_m5/m5/utils.py b/examples/example_m5/m5/utils.py index 721eee2..c21a297 100644 --- a/examples/example_m5/m5/utils.py +++ b/examples/example_m5/m5/utils.py @@ -70,7 +70,7 @@ def measure_aux_algo(idx, test_sample, model): # Simple "Memory profilers" to see memory usage def get_memory_usage(): - return np.round(psutil.Process(os.getpid()).memory_info()[0] / 2.0**30, 2) + return np.round(psutil.Process(os.getpid()).memory_info()[0] / 2.0 ** 30, 2) def sizeof_fmt(num, suffix="B"): @@ -84,7 +84,7 @@ def sizeof_fmt(num, suffix="B"): # Memory Reducer def reduce_mem_usage(df, float16_flag=True, verbose=True): numerics = ["int16", "int32", "int64", "float16", "float32", "float64"] - start_mem = df.memory_usage().sum() / 1024**2 + start_mem = df.memory_usage().sum() / 1024 ** 2 for col in df.columns: col_type = df[col].dtypes if col_type in numerics: @@ -106,7 +106,7 @@ def reduce_mem_usage(df, float16_flag=True, verbose=True): df[col] = df[col].astype(np.float32) else: df[col] = df[col].astype(np.float64) - end_mem = df.memory_usage().sum() / 1024**2 + end_mem = df.memory_usage().sum() / 1024 ** 2 if verbose: print( "Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)".format( diff --git a/examples/example_m5/main.py b/examples/example_m5/main.py index d10b129..353cc6d 100644 --- a/examples/example_m5/main.py +++ b/examples/example_m5/main.py @@ -15,10 +15,7 @@ from m5 import DataLoader semantic_specs = [ { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class",}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Nature"], "Type": "Tag"}, "Description": {"Values": "", "Type": "Description"}, @@ -26,10 +23,7 @@ semantic_specs = [ }, { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class",}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business", "Nature"], "Type": "Tag"}, "Description": {"Values": "", "Type": "Description"}, @@ -37,10 +31,7 @@ semantic_specs = [ }, { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class",}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "Description"}, @@ -50,10 +41,7 @@ semantic_specs = [ user_senmantic = { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class",}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "Description"}, diff --git a/examples/example_market_db/example_db.py b/examples/example_market_db/example_db.py index a7b0585..771cf10 100644 --- a/examples/example_market_db/example_db.py +++ b/examples/example_market_db/example_db.py @@ -40,10 +40,7 @@ semantic_specs = [ user_senmantic = { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class",}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "Description"}, diff --git a/examples/example_pfs/main.py b/examples/example_pfs/main.py index 42bd298..0c14a4a 100644 --- a/examples/example_pfs/main.py +++ b/examples/example_pfs/main.py @@ -15,10 +15,7 @@ from pfs import Dataloader semantic_specs = [ { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class",}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Nature"], "Type": "Tag"}, "Description": {"Values": "", "Type": "Description"}, @@ -26,10 +23,7 @@ semantic_specs = [ }, { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class",}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business", "Nature"], "Type": "Tag"}, "Description": {"Values": "", "Type": "Description"}, @@ -37,10 +31,7 @@ semantic_specs = [ }, { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class",}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "Description"}, @@ -50,10 +41,7 @@ semantic_specs = [ user_senmantic = { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class",}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "Description"}, diff --git a/examples/example_pfs/pfs/pfs_cross_transfer.py b/examples/example_pfs/pfs/pfs_cross_transfer.py index 93a3fa3..a106fb7 100644 --- a/examples/example_pfs/pfs/pfs_cross_transfer.py +++ b/examples/example_pfs/pfs/pfs_cross_transfer.py @@ -67,7 +67,7 @@ def get_split_errs(algo): for tmp in range(len(proportion_list)): model = lgb.LGBMModel( boosting_type="gbdt", - num_leaves=2**7 - 1, + num_leaves=2 ** 7 - 1, learning_rate=0.01, objective="rmse", metric="rmse", @@ -119,7 +119,7 @@ def get_errors(algo): if algo == "lgb": model = lgb.LGBMModel( boosting_type="gbdt", - num_leaves=2**7 - 1, + num_leaves=2 ** 7 - 1, learning_rate=0.01, objective="rmse", metric="rmse", diff --git a/examples/workflow_by_code/main.py b/examples/workflow_by_code/main.py index 36acaeb..fe5524d 100644 --- a/examples/workflow_by_code/main.py +++ b/examples/workflow_by_code/main.py @@ -18,10 +18,7 @@ curr_root = os.path.dirname(os.path.abspath(__file__)) semantic_specs = [ { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class",}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Nature"], "Type": "Tag"}, "Description": {"Values": "", "Type": "Description"}, @@ -29,10 +26,7 @@ semantic_specs = [ }, { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class",}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business", "Nature"], "Type": "Tag"}, "Description": {"Values": "", "Type": "Description"}, @@ -40,10 +34,7 @@ semantic_specs = [ }, { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class",}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "Description"}, @@ -53,10 +44,7 @@ semantic_specs = [ user_senmantic = { "Data": {"Values": ["Tabular"], "Type": "Class"}, - "Task": { - "Values": ["Classification"], - "Type": "Class", - }, + "Task": {"Values": ["Classification"], "Type": "Class",}, "Device": {"Values": ["GPU"], "Type": "Tag"}, "Scenario": {"Values": ["Business"], "Type": "Tag"}, "Description": {"Values": "", "Type": "Description"}, diff --git a/learnware/config.py b/learnware/config.py index 2580b5c..d80798f 100644 --- a/learnware/config.py +++ b/learnware/config.py @@ -66,10 +66,7 @@ os.makedirs(LEARNWARE_FOLDER_POOL_PATH, exist_ok=True) os.makedirs(DATABASE_PATH, exist_ok=True) semantic_config = { - "Data": { - "Values": ["Tabular", "Image", "Video", "Text", "Audio"], - "Type": "Class", - }, # Choose only one class + "Data": {"Values": ["Tabular", "Image", "Video", "Text", "Audio"], "Type": "Class",}, # Choose only one class "Task": { "Values": [ "Classification", @@ -82,10 +79,7 @@ semantic_config = { ], "Type": "Class", # Choose only one class }, - "Device": { - "Values": ["CPU", "GPU"], - "Type": "Tag", - }, # Choose one or more tags + "Device": {"Values": ["CPU", "GPU"], "Type": "Tag",}, # Choose one or more tags "Scenario": { "Values": [ "Business", @@ -105,14 +99,8 @@ semantic_config = { ], "Type": "Tag", # Choose one or more tags }, - "Description": { - "Values": None, - "Type": "Description", - }, - "Name": { - "Values": None, - "Type": "Name", - }, + "Description": {"Values": None, "Type": "Description",}, + "Name": {"Values": None, "Type": "Name",}, } _DEFAULT_CONFIG = { @@ -123,10 +111,7 @@ _DEFAULT_CONFIG = { "learnware_pool_path": LEARNWARE_POOL_PATH, "learnware_zip_pool_path": LEARNWARE_ZIP_POOL_PATH, "learnware_folder_pool_path": LEARNWARE_FOLDER_POOL_PATH, - "learnware_folder_config": { - "yaml_file": "learnware.yaml", - "module_file": "__init__.py", - }, + "learnware_folder_config": {"yaml_file": "learnware.yaml", "module_file": "__init__.py",}, "database_path": DATABASE_PATH, "max_reduced_set_size": 1000000, } diff --git a/learnware/learnware/__init__.py b/learnware/learnware/__init__.py index c505595..e4b06ed 100644 --- a/learnware/learnware/__init__.py +++ b/learnware/learnware/__init__.py @@ -30,10 +30,7 @@ def get_learnware_from_dirpath(id: str, semantic_spec: dict, learnware_dirpath: The contructed learnware object, return None if build failed """ learnware_config = { - "model": { - "class_name": "Model", - "kwargs": {}, - }, + "model": {"class_name": "Model", "kwargs": {},}, "stat_specifications": [ { "module_path": "learnware.specification", diff --git a/learnware/learnware/reuse.py b/learnware/learnware/reuse.py index eb20d7f..2a0836e 100644 --- a/learnware/learnware/reuse.py +++ b/learnware/learnware/reuse.py @@ -187,7 +187,7 @@ class ReuseBaseline: n_estimators=2000, # objective="multiclass", # num_class=num_class, - booster="gbtree", + boosting_type="gbdt", seed=0, ) train_y = train_y.astype(np.int) @@ -205,7 +205,7 @@ class ReuseBaseline: n_estimators=2000, # objective="multiclass", # num_class=num_class, - booster="gbtree", + boosting_type="gbdt", seed=0, ) model.fit( diff --git a/learnware/market/easy.py b/learnware/market/easy.py index 5a9014f..3d5b259 100644 --- a/learnware/market/easy.py +++ b/learnware/market/easy.py @@ -141,10 +141,7 @@ class EasyMarket(BaseMarket): self.learnware_folder_list[id] = target_folder_dir self.count += 1 add_learnware_to_db( - id, - semantic_spec=semantic_spec, - zip_path=target_zip_dir, - folder_path=target_folder_dir, + id, semantic_spec=semantic_spec, zip_path=target_zip_dir, folder_path=target_folder_dir, ) return id, True diff --git a/learnware/specification/rkme.py b/learnware/specification/rkme.py index 9370c96..cb9d963 100644 --- a/learnware/specification/rkme.py +++ b/learnware/specification/rkme.py @@ -354,9 +354,7 @@ class RKMEStatSpecification(BaseStatSpecification): rkme_to_save["beta"] = rkme_to_save["beta"].tolist() rkme_to_save["device"] = "gpu" if rkme_to_save["cuda_idx"] != -1 else "cpu" json.dump( - rkme_to_save, - codecs.open(save_path, "w", encoding="utf-8"), - separators=(",", ":"), + rkme_to_save, codecs.open(save_path, "w", encoding="utf-8"), separators=(",", ":"), ) def load(self, filepath: str) -> bool: @@ -444,7 +442,7 @@ def torch_rbf_kernel(x1, x2, gamma) -> torch.Tensor: """ x1 = x1.double() x2 = x2.double() - X12norm = torch.sum(x1**2, 1, keepdim=True) - 2 * x1 @ x2.T + torch.sum(x2**2, 1, keepdim=True).T + X12norm = torch.sum(x1 ** 2, 1, keepdim=True) - 2 * x1 @ x2.T + torch.sum(x2 ** 2, 1, keepdim=True).T return torch.exp(-X12norm * gamma)