update lp pyg test file

test/performance/link_prediction/pyg/link_prediction_base.py → test/performance/link_prediction/pyg/base.py

@@ -1,3 +1,4 @@
+import time
 import torch
 import torch.nn.functional as F
 import numpy as np
@@ -58,14 +59,11 @@ class GAT(torch.nn.Module):
     def __init__(self, num_features, hidden_features, heads):
         super(GAT, self).__init__()
         self.conv1 = GATConv(num_features, hidden_features, heads, dropout=0.0)
-        self.conv2 = GATConv(hidden_features * heads, hidden_features * heads//2, heads=8, concat=True, dropout=0.0)
+        self.conv2 = GATConv(hidden_features * heads, hidden_features, heads=8, concat=True, dropout=0.0)
     def encode(self, data):
         x, edge_index = data.x, data.train_pos_edge_index
-        # x = F.dropout(x, p=0.0, training=self.training)
         x = F.relu(self.conv1(x, edge_index))
-        # x = F.dropout(x, p=0.6, training=self.training)
         x = self.conv2(x, edge_index)
-        # print(x.shape,"!!!!!!") # torch.Size([3327, 64])
         return x
     
     def decode(self, z, pos_edge_index, neg_edge_index):
@@ -161,6 +159,8 @@ def test():
         perfs.append(roc_auc_score(link_labels.cpu(), link_probs.cpu()))
     return perfs
 
+begin_time = time.time()
+
 res = []
 for seed in tqdm(range(1234, 1234+args.repeat)):
     set_seed(seed)
@@ -189,4 +189,4 @@ for seed in tqdm(range(1234, 1234+args.repeat)):
             test_perf = tmp_test_perf
     res.append(test_perf)
 
-print(np.mean(res), np.std(res))
+print("{:.2f} ~ {:.2f} ({:.2f}s/it)".format(np.mean(res) * 100, np.std(res) * 100, (time.time() - begin_time) / args.repeat))

test/performance/link_prediction/pyg/helper.py

@@ -1,14 +1,12 @@
 def get_encoder_decoder_hp(model='gat', decoder='lpdecoder'):
     if model == 'gat':
         model_hp = {
-            # hp from model
-            "num_layers": 3,
-            "hidden": [16,64],
-            "heads": 8,
+            "num_layers": 2,
+            "hidden": [16, 16],
             "dropout": 0.0,
             "act": "relu",
-            'add_self_loops': 'False',
-            'normalize': 'False',
+            "num_hidden_heads": 8,
+            "num_output_heads": 8
         }
     elif model == 'gcn':
         model_hp = {
@@ -22,9 +20,7 @@ def get_encoder_decoder_hp(model='gat', decoder='lpdecoder'):
             "hidden": [128,64],
             "dropout": 0.0,
             "act": "relu",
-            "agg": "mean",
-            'add_self_loops': 'False',
-            'normalize': 'False',
+            "agg": "mean"
         }
-        
+
     return model_hp, {}

test/performance/link_prediction/pyg/link_prediction_model.py → test/performance/link_prediction/pyg/model.py

@@ -1,6 +1,7 @@
 import os
 os.environ["AUTOGL_BACKEND"] = "pyg"
 
+import time
 import torch
 import torch.nn.functional as F
 import numpy as np
@@ -90,6 +91,7 @@ def test(data):
     return perfs
 
 res = []
+begin_time = time.time()
 for seed in tqdm(range(1234, 1234+args.repeat)):
     setup_seed(seed)
     data = dataset[0].to(device)
@@ -116,7 +118,7 @@ for seed in tqdm(range(1234, 1234+args.repeat)):
                 init=False
             ).from_hyper_parameter({
                 'num_layers': 3,
-                'hidden': [16,64],
+                'hidden': [16,16],
                 "heads": 8,
                 'dropout': 0.0,
                 'act': 'relu'
@@ -133,9 +135,7 @@ for seed in tqdm(range(1234, 1234+args.repeat)):
                 'hidden': [128,64],
                 'dropout': 0.0,
                 'act': 'relu',
-                'agg': 'mean',
-                'add_self_loops': 'False',
-                'normalize': 'False',
+                'agg': 'mean'
             }).model
     else:
         assert False
@@ -151,4 +151,4 @@ for seed in tqdm(range(1234, 1234+args.repeat)):
             test_perf = tmp_test_perf
     res.append(test_perf)
 
-print(np.mean(res), np.std(res))
+print("{:.2f} ~ {:.2f} ({:.2f}s/it)".format(np.mean(res) * 100, np.std(res) * 100, (time.time() - begin_time) / args.repeat))

test/performance/link_prediction/pyg/model_decouple.py

@@ -1,6 +1,7 @@
 import os
 os.environ["AUTOGL_BACKEND"] = "pyg"
 
+import time
 import torch
 import torch.nn.functional as F
 import numpy as np
@@ -15,6 +16,7 @@ from torch_geometric.utils import train_test_split_edges
 from torch_geometric.utils import negative_sampling
 from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
 from tqdm import tqdm
+from helper import get_encoder_decoder_hp
 
 from sklearn.metrics import roc_auc_score
 
@@ -107,6 +109,10 @@ def test():
 
 
 res = []
+begin_time = time.time()
+
+model_hp, _ = get_encoder_decoder_hp(args.model)
+
 for seed in tqdm(range(1234, 1234+args.repeat)):
     setup_seed(seed)
     data = dataset[0].to(device)
@@ -116,50 +122,25 @@ for seed in tqdm(range(1234, 1234+args.repeat)):
     data.edge_index = data.train_pos_edge_index
     if args.model == 'gcn':
         encoder = GCNEncoderMaintainer(
-            dataset.num_features, 64, args.device
-        ).from_hyper_parameter({
-            "hidden": [128],
-            "dropout": 0.0,
-            "act": "relu"
-        }).encoder
+            dataset.num_features, "auto", args.device
+        ).from_hyper_parameter(model_hp).encoder
         model = DummyModel(encoder).to(args.device)
 
     elif args.model == 'gat':
-        model = AutoGAT(dataset=dataset,
-                num_features=dataset.num_features,
-                num_classes=2,
-                device=args.device,
-                init=False
-            ).from_hyper_parameter({
-                'num_layers': 3,
-                'hidden': [16,64],
-                "heads": 8,
-                'dropout': 0.0,
-                'act': 'relu'
-            }).model
-        # print(model)
+        encoder = GATEncoderMaintainer(
+            dataset.num_features, "auto", args.device
+        ).from_hyper_parameter(model_hp).encoder
+        model = DummyModel(encoder).to(args.device)
     elif args.model == 'sage':
-        model = AutoSAGE(dataset=dataset,
-                num_features=dataset.num_features,
-                num_classes=2,
-                device=args.device,
-                init=False
-            ).from_hyper_parameter({
-                'num_layers': 3,
-                'hidden': [128,64],
-                'dropout': 0.0,
-                'act': 'relu',
-                'agg': 'mean',
-                'add_self_loops': 'False',
-                'normalize': 'False',
-            }).model
+        encoder = SAGEEncoderMaintainer(
+            dataset.num_features, "auto", args.device
+        ).from_hyper_parameter(model_hp).encoder
+        model = DummyModel(encoder).to(args.device)
     else:
         assert False
 
     optimizer = torch.optim.Adam(params=model.parameters(), lr=0.01)
 
-    import pdb
-    pdb.set_trace()
     best_val_perf = test_perf = 0
 
     for epoch in range(100):
@@ -170,4 +151,4 @@ for seed in tqdm(range(1234, 1234+args.repeat)):
             test_perf = tmp_test_perf
     res.append(test_perf)
 
-print(np.mean(res), np.std(res))
+print("{:.2f} ~ {:.2f} ({:.2f}s/it)".format(np.mean(res) * 100, np.std(res) * 100, (time.time() - begin_time) / args.repeat))

test/performance/link_prediction/pyg/solver.py

@@ -0,0 +1,83 @@
+import os
+os.environ["AUTOGL_BACKEND"] = "pyg"
+import time
+from tqdm import tqdm
+import numpy as np
+from helper import get_encoder_decoder_hp
+from autogl.solver import AutoLinkPredictor
+from autogl.datasets import build_dataset_from_name
+
+def fixed(**kwargs):
+    return [{
+        'parameterName': k,
+        "type": "FIXED",
+        "value": v
+    } for k, v in kwargs.items()]
+
+if __name__ == "__main__":
+
+    from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
+
+    parser = ArgumentParser(
+        "auto link prediction", formatter_class=ArgumentDefaultsHelpFormatter
+    )
+    parser.add_argument(
+        "--dataset",
+        default="Cora",
+        type=str,
+        help="dataset to use",
+        choices=[
+            "Cora",
+            "CiteSeer",
+            "PubMed",
+        ],
+    )
+    parser.add_argument(
+        "--model",
+        default="sage",
+        type=str,
+        help="model to use",
+        choices=[
+            "gcn",
+            "gat",
+            "sage",
+        ],
+    )
+    parser.add_argument("--seed", type=int, default=0, help="random seed")
+    parser.add_argument('--repeat', type=int, default=10)
+    parser.add_argument("--device", default=0, type=int, help="GPU device")
+
+    args = parser.parse_args()
+
+    if args.device < 0:
+        device = args.device = "cpu"
+    else:
+        device = args.device = f"cuda:{args.device}"
+
+    dataset = build_dataset_from_name(args.dataset.lower())
+
+    res = []
+    begin_time = time.time()
+    for seed in tqdm(range(1234, 1234+args.repeat)):
+        model_hp, decoder_hp = get_encoder_decoder_hp(args.model)
+
+        solver = AutoLinkPredictor(
+            feature_module="NormalizeFeatures",
+            graph_models=(args.model, ),
+            hpo_module="random",
+            ensemble_module=None,
+            max_evals=1,
+            trainer_hp_space=fixed(**{
+                "max_epoch": 100,
+                "early_stopping_round": 101,
+                "lr": 1e-2,
+                "weight_decay": 0.0,
+            }),
+            model_hp_spaces=[{"encoder": fixed(**model_hp), "decoder": fixed(**decoder_hp)}]
+        )
+
+        solver.fit(dataset, train_split=0.85, val_split=0.05, evaluation_method=["auc"], seed=seed)
+        pre = solver.evaluate(metric="auc")
+        res.append(pre)
+
+    print("{:.2f} ~ {:.2f} ({:.2f}s/it)".format(np.mean(res) * 100, np.std(res) * 100, (time.time() - begin_time) / args.repeat))

test/performance/link_prediction/pyg/link_prediction_trainer.py → test/performance/link_prediction/pyg/trainer.py

@@ -1,5 +1,6 @@
 import os
 os.environ["AUTOGL_BACKEND"] = "pyg"
+import time
 from tqdm import tqdm
 from autogl.module.train.evaluation import Auc
 import random
@@ -10,7 +11,6 @@ import os.path as osp
 import torch_geometric.transforms as T
 from torch_geometric.datasets import Planetoid
 from torch_geometric.utils import train_test_split_edges
-from autogl.datasets.utils import split_edges
 from autogl.module.train.link_prediction_full import LinkPredictionTrainer
 
 def setup_seed(seed):
@@ -66,16 +66,13 @@ if __name__ == "__main__":
     dataset = Planetoid(osp.expanduser('~/.cache-autogl'), args.dataset, transform=T.NormalizeFeatures())
 
     res = []
+    begin_time = time.time()
     for seed in tqdm(range(1234, 1234+args.repeat)):
         setup_seed(seed)
         data = dataset[0].to(device)
         # use train_test_split_edges to create neg and positive edges
         data.train_mask = data.val_mask = data.test_mask = data.y = None
-
-        if args.use_our_split_edges:
-            data = split_edges(dataset, 0.85, 0.05)[0]
-        else:
-            data = train_test_split_edges(data).to(device)
+        data = train_test_split_edges(data).to(device)
 
         model_hp, decoder_hp = get_encoder_decoder_hp(args.model)
 
@@ -106,4 +103,5 @@ if __name__ == "__main__":
         pre = trainer.evaluate([data], mask="test", feval=Auc)
         res.append(pre)
 
-    print(np.mean(res), np.std(res))
+    print("{:.2f} ~ {:.2f} ({:.2f}s/it)".format(np.mean(res) * 100, np.std(res) * 100, (time.time() - begin_time) / args.repeat))
+

test/performance/link_prediction/pyg/trainer_dataset.py

@@ -0,0 +1,90 @@
+import os
+os.environ["AUTOGL_BACKEND"] = "pyg"
+import time
+from tqdm import tqdm
+from autogl.module.train.evaluation import Auc
+import numpy as np
+from helper import get_encoder_decoder_hp
+from autogl.module.train.link_prediction_full import LinkPredictionTrainer
+from autogl.datasets.utils import split_edges
+from autogl.solver.utils import set_seed
+from autogl.datasets import build_dataset_from_name
+from autogl.module.feature import NormalizeFeatures
+
+if __name__ == "__main__":
+
+
+    from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
+
+    parser = ArgumentParser(
+        "auto link prediction", formatter_class=ArgumentDefaultsHelpFormatter
+    )
+    parser.add_argument(
+        "--dataset",
+        default="Cora",
+        type=str,
+        help="dataset to use",
+        choices=[
+            "Cora",
+            "CiteSeer",
+            "PubMed",
+        ],
+    )
+    parser.add_argument(
+        "--model",
+        default="sage",
+        type=str,
+        help="model to use",
+        choices=[
+            "gcn",
+            "gat",
+            "sage",
+        ],
+    )
+    parser.add_argument("--seed", type=int, default=0, help="random seed")
+    parser.add_argument('--repeat', type=int, default=10)
+    parser.add_argument("--device", default=0, type=int, help="GPU device")
+
+    args = parser.parse_args()
+
+    if args.device < 0:
+        device = args.device = "cpu"
+    else:
+        device = args.device = f"cuda:{args.device}"
+
+    dataset = build_dataset_from_name(args.dataset.lower())
+    dataset = NormalizeFeatures().fit_transform(dataset)
+
+    res = []
+    begin_time = time.time()
+    for seed in tqdm(range(1234, 1234+args.repeat)):
+        set_seed(seed)
+        data = split_edges(dataset, 0.85, 0.05)[0]
+
+        model_hp, decoder_hp = get_encoder_decoder_hp(args.model)
+
+        trainer = LinkPredictionTrainer(
+            model = args.model,
+            num_features = data.x.size(1),
+            lr = 1e-2,
+            max_epoch = 100,
+            early_stopping_round = 101,
+            weight_decay = 0.0,
+            device = args.device,
+            feval = [Auc],
+            loss = "binary_cross_entropy_with_logits",
+            init = False
+        ).duplicate_from_hyper_parameter(
+            {
+                "trainer": {},
+                "encoder": model_hp,
+                "decoder": decoder_hp
+            },
+            restricted=False
+        )
+
+        trainer.train([data], False)
+        pre = trainer.evaluate([data], mask="test", feval=Auc)
+        res.append(pre)
+
+    print("{:.2f} ~ {:.2f} ({:.2f}s/it)".format(np.mean(res) * 100, np.std(res) * 100, (time.time() - begin_time) / args.repeat))