From cf319e164fb6b2cdcb34b50a1733df34346a7ac3 Mon Sep 17 00:00:00 2001 From: Frozenmad Date: Thu, 30 Dec 2021 20:10:41 +0800 Subject: [PATCH] update lp pyg test file --- .../pyg/{link_prediction_base.py => base.py} | 10 +-- .../performance/link_prediction/pyg/helper.py | 16 ++-- .../{link_prediction_model.py => model.py} | 10 +-- .../link_prediction/pyg/model_decouple.py | 53 ++++------- .../performance/link_prediction/pyg/solver.py | 83 +++++++++++++++++ ...{link_prediction_trainer.py => trainer.py} | 12 ++- .../link_prediction/pyg/trainer_dataset.py | 90 +++++++++++++++++++ 7 files changed, 211 insertions(+), 63 deletions(-) rename test/performance/link_prediction/pyg/{link_prediction_base.py => base.py} (96%) rename test/performance/link_prediction/pyg/{link_prediction_model.py => model.py} (95%) create mode 100644 test/performance/link_prediction/pyg/solver.py rename test/performance/link_prediction/pyg/{link_prediction_trainer.py => trainer.py} (92%) create mode 100644 test/performance/link_prediction/pyg/trainer_dataset.py diff --git a/test/performance/link_prediction/pyg/link_prediction_base.py b/test/performance/link_prediction/pyg/base.py similarity index 96% rename from test/performance/link_prediction/pyg/link_prediction_base.py rename to test/performance/link_prediction/pyg/base.py index ff07e3d..230824a 100644 --- a/test/performance/link_prediction/pyg/link_prediction_base.py +++ b/test/performance/link_prediction/pyg/base.py @@ -1,3 +1,4 @@ +import time import torch import torch.nn.functional as F import numpy as np @@ -58,14 +59,11 @@ class GAT(torch.nn.Module): def __init__(self, num_features, hidden_features, heads): super(GAT, self).__init__() self.conv1 = GATConv(num_features, hidden_features, heads, dropout=0.0) - self.conv2 = GATConv(hidden_features * heads, hidden_features * heads//2, heads=8, concat=True, dropout=0.0) + self.conv2 = GATConv(hidden_features * heads, hidden_features, heads=8, concat=True, dropout=0.0) def encode(self, data): x, edge_index = data.x, data.train_pos_edge_index - # x = F.dropout(x, p=0.0, training=self.training) x = F.relu(self.conv1(x, edge_index)) - # x = F.dropout(x, p=0.6, training=self.training) x = self.conv2(x, edge_index) - # print(x.shape,"!!!!!!") # torch.Size([3327, 64]) return x def decode(self, z, pos_edge_index, neg_edge_index): @@ -161,6 +159,8 @@ def test(): perfs.append(roc_auc_score(link_labels.cpu(), link_probs.cpu())) return perfs +begin_time = time.time() + res = [] for seed in tqdm(range(1234, 1234+args.repeat)): set_seed(seed) @@ -189,4 +189,4 @@ for seed in tqdm(range(1234, 1234+args.repeat)): test_perf = tmp_test_perf res.append(test_perf) -print(np.mean(res), np.std(res)) +print("{:.2f} ~ {:.2f} ({:.2f}s/it)".format(np.mean(res) * 100, np.std(res) * 100, (time.time() - begin_time) / args.repeat)) diff --git a/test/performance/link_prediction/pyg/helper.py b/test/performance/link_prediction/pyg/helper.py index d1a7382..aa5c37e 100644 --- a/test/performance/link_prediction/pyg/helper.py +++ b/test/performance/link_prediction/pyg/helper.py @@ -1,14 +1,12 @@ def get_encoder_decoder_hp(model='gat', decoder='lpdecoder'): if model == 'gat': model_hp = { - # hp from model - "num_layers": 3, - "hidden": [16,64], - "heads": 8, + "num_layers": 2, + "hidden": [16, 16], "dropout": 0.0, "act": "relu", - 'add_self_loops': 'False', - 'normalize': 'False', + "num_hidden_heads": 8, + "num_output_heads": 8 } elif model == 'gcn': model_hp = { @@ -22,9 +20,7 @@ def get_encoder_decoder_hp(model='gat', decoder='lpdecoder'): "hidden": [128,64], "dropout": 0.0, "act": "relu", - "agg": "mean", - 'add_self_loops': 'False', - 'normalize': 'False', + "agg": "mean" } - + return model_hp, {} diff --git a/test/performance/link_prediction/pyg/link_prediction_model.py b/test/performance/link_prediction/pyg/model.py similarity index 95% rename from test/performance/link_prediction/pyg/link_prediction_model.py rename to test/performance/link_prediction/pyg/model.py index 4df4202..8480381 100644 --- a/test/performance/link_prediction/pyg/link_prediction_model.py +++ b/test/performance/link_prediction/pyg/model.py @@ -1,6 +1,7 @@ import os os.environ["AUTOGL_BACKEND"] = "pyg" +import time import torch import torch.nn.functional as F import numpy as np @@ -90,6 +91,7 @@ def test(data): return perfs res = [] +begin_time = time.time() for seed in tqdm(range(1234, 1234+args.repeat)): setup_seed(seed) data = dataset[0].to(device) @@ -116,7 +118,7 @@ for seed in tqdm(range(1234, 1234+args.repeat)): init=False ).from_hyper_parameter({ 'num_layers': 3, - 'hidden': [16,64], + 'hidden': [16,16], "heads": 8, 'dropout': 0.0, 'act': 'relu' @@ -133,9 +135,7 @@ for seed in tqdm(range(1234, 1234+args.repeat)): 'hidden': [128,64], 'dropout': 0.0, 'act': 'relu', - 'agg': 'mean', - 'add_self_loops': 'False', - 'normalize': 'False', + 'agg': 'mean' }).model else: assert False @@ -151,4 +151,4 @@ for seed in tqdm(range(1234, 1234+args.repeat)): test_perf = tmp_test_perf res.append(test_perf) -print(np.mean(res), np.std(res)) +print("{:.2f} ~ {:.2f} ({:.2f}s/it)".format(np.mean(res) * 100, np.std(res) * 100, (time.time() - begin_time) / args.repeat)) diff --git a/test/performance/link_prediction/pyg/model_decouple.py b/test/performance/link_prediction/pyg/model_decouple.py index cb4ba45..3f82f06 100644 --- a/test/performance/link_prediction/pyg/model_decouple.py +++ b/test/performance/link_prediction/pyg/model_decouple.py @@ -1,6 +1,7 @@ import os os.environ["AUTOGL_BACKEND"] = "pyg" +import time import torch import torch.nn.functional as F import numpy as np @@ -15,6 +16,7 @@ from torch_geometric.utils import train_test_split_edges from torch_geometric.utils import negative_sampling from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter from tqdm import tqdm +from helper import get_encoder_decoder_hp from sklearn.metrics import roc_auc_score @@ -107,6 +109,10 @@ def test(): res = [] +begin_time = time.time() + +model_hp, _ = get_encoder_decoder_hp(args.model) + for seed in tqdm(range(1234, 1234+args.repeat)): setup_seed(seed) data = dataset[0].to(device) @@ -116,50 +122,25 @@ for seed in tqdm(range(1234, 1234+args.repeat)): data.edge_index = data.train_pos_edge_index if args.model == 'gcn': encoder = GCNEncoderMaintainer( - dataset.num_features, 64, args.device - ).from_hyper_parameter({ - "hidden": [128], - "dropout": 0.0, - "act": "relu" - }).encoder + dataset.num_features, "auto", args.device + ).from_hyper_parameter(model_hp).encoder model = DummyModel(encoder).to(args.device) elif args.model == 'gat': - model = AutoGAT(dataset=dataset, - num_features=dataset.num_features, - num_classes=2, - device=args.device, - init=False - ).from_hyper_parameter({ - 'num_layers': 3, - 'hidden': [16,64], - "heads": 8, - 'dropout': 0.0, - 'act': 'relu' - }).model - # print(model) + encoder = GATEncoderMaintainer( + dataset.num_features, "auto", args.device + ).from_hyper_parameter(model_hp).encoder + model = DummyModel(encoder).to(args.device) elif args.model == 'sage': - model = AutoSAGE(dataset=dataset, - num_features=dataset.num_features, - num_classes=2, - device=args.device, - init=False - ).from_hyper_parameter({ - 'num_layers': 3, - 'hidden': [128,64], - 'dropout': 0.0, - 'act': 'relu', - 'agg': 'mean', - 'add_self_loops': 'False', - 'normalize': 'False', - }).model + encoder = SAGEEncoderMaintainer( + dataset.num_features, "auto", args.device + ).from_hyper_parameter(model_hp).encoder + model = DummyModel(encoder).to(args.device) else: assert False optimizer = torch.optim.Adam(params=model.parameters(), lr=0.01) - import pdb - pdb.set_trace() best_val_perf = test_perf = 0 for epoch in range(100): @@ -170,4 +151,4 @@ for seed in tqdm(range(1234, 1234+args.repeat)): test_perf = tmp_test_perf res.append(test_perf) -print(np.mean(res), np.std(res)) +print("{:.2f} ~ {:.2f} ({:.2f}s/it)".format(np.mean(res) * 100, np.std(res) * 100, (time.time() - begin_time) / args.repeat)) diff --git a/test/performance/link_prediction/pyg/solver.py b/test/performance/link_prediction/pyg/solver.py new file mode 100644 index 0000000..12df34b --- /dev/null +++ b/test/performance/link_prediction/pyg/solver.py @@ -0,0 +1,83 @@ +import os +os.environ["AUTOGL_BACKEND"] = "pyg" +import time +from tqdm import tqdm +import numpy as np +from helper import get_encoder_decoder_hp +from autogl.solver import AutoLinkPredictor +from autogl.datasets import build_dataset_from_name + +def fixed(**kwargs): + return [{ + 'parameterName': k, + "type": "FIXED", + "value": v + } for k, v in kwargs.items()] + +if __name__ == "__main__": + + from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter + + parser = ArgumentParser( + "auto link prediction", formatter_class=ArgumentDefaultsHelpFormatter + ) + parser.add_argument( + "--dataset", + default="Cora", + type=str, + help="dataset to use", + choices=[ + "Cora", + "CiteSeer", + "PubMed", + ], + ) + parser.add_argument( + "--model", + default="sage", + type=str, + help="model to use", + choices=[ + "gcn", + "gat", + "sage", + ], + ) + parser.add_argument("--seed", type=int, default=0, help="random seed") + parser.add_argument('--repeat', type=int, default=10) + parser.add_argument("--device", default=0, type=int, help="GPU device") + + args = parser.parse_args() + + if args.device < 0: + device = args.device = "cpu" + else: + device = args.device = f"cuda:{args.device}" + + dataset = build_dataset_from_name(args.dataset.lower()) + + res = [] + begin_time = time.time() + for seed in tqdm(range(1234, 1234+args.repeat)): + model_hp, decoder_hp = get_encoder_decoder_hp(args.model) + + solver = AutoLinkPredictor( + feature_module="NormalizeFeatures", + graph_models=(args.model, ), + hpo_module="random", + ensemble_module=None, + max_evals=1, + trainer_hp_space=fixed(**{ + "max_epoch": 100, + "early_stopping_round": 101, + "lr": 1e-2, + "weight_decay": 0.0, + }), + model_hp_spaces=[{"encoder": fixed(**model_hp), "decoder": fixed(**decoder_hp)}] + ) + + solver.fit(dataset, train_split=0.85, val_split=0.05, evaluation_method=["auc"], seed=seed) + pre = solver.evaluate(metric="auc") + res.append(pre) + + print("{:.2f} ~ {:.2f} ({:.2f}s/it)".format(np.mean(res) * 100, np.std(res) * 100, (time.time() - begin_time) / args.repeat)) diff --git a/test/performance/link_prediction/pyg/link_prediction_trainer.py b/test/performance/link_prediction/pyg/trainer.py similarity index 92% rename from test/performance/link_prediction/pyg/link_prediction_trainer.py rename to test/performance/link_prediction/pyg/trainer.py index 6d7e455..af299b5 100644 --- a/test/performance/link_prediction/pyg/link_prediction_trainer.py +++ b/test/performance/link_prediction/pyg/trainer.py @@ -1,5 +1,6 @@ import os os.environ["AUTOGL_BACKEND"] = "pyg" +import time from tqdm import tqdm from autogl.module.train.evaluation import Auc import random @@ -10,7 +11,6 @@ import os.path as osp import torch_geometric.transforms as T from torch_geometric.datasets import Planetoid from torch_geometric.utils import train_test_split_edges -from autogl.datasets.utils import split_edges from autogl.module.train.link_prediction_full import LinkPredictionTrainer def setup_seed(seed): @@ -66,16 +66,13 @@ if __name__ == "__main__": dataset = Planetoid(osp.expanduser('~/.cache-autogl'), args.dataset, transform=T.NormalizeFeatures()) res = [] + begin_time = time.time() for seed in tqdm(range(1234, 1234+args.repeat)): setup_seed(seed) data = dataset[0].to(device) # use train_test_split_edges to create neg and positive edges data.train_mask = data.val_mask = data.test_mask = data.y = None - - if args.use_our_split_edges: - data = split_edges(dataset, 0.85, 0.05)[0] - else: - data = train_test_split_edges(data).to(device) + data = train_test_split_edges(data).to(device) model_hp, decoder_hp = get_encoder_decoder_hp(args.model) @@ -106,4 +103,5 @@ if __name__ == "__main__": pre = trainer.evaluate([data], mask="test", feval=Auc) res.append(pre) - print(np.mean(res), np.std(res)) + print("{:.2f} ~ {:.2f} ({:.2f}s/it)".format(np.mean(res) * 100, np.std(res) * 100, (time.time() - begin_time) / args.repeat)) + diff --git a/test/performance/link_prediction/pyg/trainer_dataset.py b/test/performance/link_prediction/pyg/trainer_dataset.py new file mode 100644 index 0000000..a59af96 --- /dev/null +++ b/test/performance/link_prediction/pyg/trainer_dataset.py @@ -0,0 +1,90 @@ +import os +os.environ["AUTOGL_BACKEND"] = "pyg" +import time +from tqdm import tqdm +from autogl.module.train.evaluation import Auc +import numpy as np +from helper import get_encoder_decoder_hp +from autogl.module.train.link_prediction_full import LinkPredictionTrainer +from autogl.datasets.utils import split_edges +from autogl.solver.utils import set_seed +from autogl.datasets import build_dataset_from_name +from autogl.module.feature import NormalizeFeatures + +if __name__ == "__main__": + + + from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter + + parser = ArgumentParser( + "auto link prediction", formatter_class=ArgumentDefaultsHelpFormatter + ) + parser.add_argument( + "--dataset", + default="Cora", + type=str, + help="dataset to use", + choices=[ + "Cora", + "CiteSeer", + "PubMed", + ], + ) + parser.add_argument( + "--model", + default="sage", + type=str, + help="model to use", + choices=[ + "gcn", + "gat", + "sage", + ], + ) + parser.add_argument("--seed", type=int, default=0, help="random seed") + parser.add_argument('--repeat', type=int, default=10) + parser.add_argument("--device", default=0, type=int, help="GPU device") + + args = parser.parse_args() + + if args.device < 0: + device = args.device = "cpu" + else: + device = args.device = f"cuda:{args.device}" + + dataset = build_dataset_from_name(args.dataset.lower()) + dataset = NormalizeFeatures().fit_transform(dataset) + + res = [] + begin_time = time.time() + for seed in tqdm(range(1234, 1234+args.repeat)): + set_seed(seed) + data = split_edges(dataset, 0.85, 0.05)[0] + + model_hp, decoder_hp = get_encoder_decoder_hp(args.model) + + trainer = LinkPredictionTrainer( + model = args.model, + num_features = data.x.size(1), + lr = 1e-2, + max_epoch = 100, + early_stopping_round = 101, + weight_decay = 0.0, + device = args.device, + feval = [Auc], + loss = "binary_cross_entropy_with_logits", + init = False + ).duplicate_from_hyper_parameter( + { + "trainer": {}, + "encoder": model_hp, + "decoder": decoder_hp + }, + restricted=False + ) + + trainer.train([data], False) + pre = trainer.evaluate([data], mask="test", feval=Auc) + res.append(pre) + + print("{:.2f} ~ {:.2f} ({:.2f}s/it)".format(np.mean(res) * 100, np.std(res) * 100, (time.time() - begin_time) / args.repeat))