From eca9d0567a02592bcdb837e99dce3f54afb020b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=94=A1=E5=A9=95?= Date: Wed, 9 Nov 2022 22:19:54 +0800 Subject: [PATCH] gnnguard and tutorial for link-prediction --- autogl/module/model/pyg/__init__.py | 6 +- autogl/module/model/pyg/robust/gcn_svd.py | 279 ---------------- autogl/module/model/pyg/robust/gnnguard.py | 304 ++++++++++++++++++ .../model/pyg/robust/nn/conv/gcn_conv.py | 196 +++++++++++ .../tutorial/t_homo_link_prediction.rst | 204 ++++++++++++ .../tutorial_cn/t_homo_link_prediction.rst | 212 ++++++++++++ test/performance/robust/model_gcnsvd.py | 130 -------- test/performance/robust/model_gnnguard.py | 192 ----------- .../performance/robust/model_gnnguard_meta.py | 204 ++++++++++++ 9 files changed, 1123 insertions(+), 604 deletions(-) delete mode 100644 autogl/module/model/pyg/robust/gcn_svd.py create mode 100644 autogl/module/model/pyg/robust/nn/conv/gcn_conv.py create mode 100644 docs/docfile/tutorial/t_homo_link_prediction.rst create mode 100644 docs/docfile/tutorial_cn/t_homo_link_prediction.rst delete mode 100644 test/performance/robust/model_gcnsvd.py delete mode 100644 test/performance/robust/model_gnnguard.py create mode 100644 test/performance/robust/model_gnnguard_meta.py diff --git a/autogl/module/model/pyg/__init__.py b/autogl/module/model/pyg/__init__.py index ab26d3e..3d05241 100644 --- a/autogl/module/model/pyg/__init__.py +++ b/autogl/module/model/pyg/__init__.py @@ -9,8 +9,7 @@ from .gcn import AutoGCN from .gat import AutoGAT from .gin import AutoGIN -from .robust.gcn_svd import AutoGCNSVD -from .robust.gnnguard import AutoGNNGuard, GCN4GNNGuard +from .robust.gnnguard import AutoGNNGuard, AutoGNNGuard_attack, GCN4GNNGuard, GCN4GNNGuard_attack __all__ = [ "ModelUniversalRegistry", @@ -22,7 +21,8 @@ __all__ = [ "AutoGCN", "AutoGAT", "AutoGIN", - "AutoGCNSVD", "AutoGNNGuard", + "AutoGNNGuard_attack", "GCN4GNNGuard", + "GCN4GNNGuard_attack", ] diff --git a/autogl/module/model/pyg/robust/gcn_svd.py b/autogl/module/model/pyg/robust/gcn_svd.py deleted file mode 100644 index cd1c408..0000000 --- a/autogl/module/model/pyg/robust/gcn_svd.py +++ /dev/null @@ -1,279 +0,0 @@ -from tkinter import TRUE -import torch -import torch.optim as optim -from torch.nn.parameter import Parameter -from torch.nn.modules.module import Module -import torch.nn as nn -import torch.nn.functional as F -import typing as _typing -import math -from tqdm import tqdm -import scipy.sparse as sp -import numpy as np -from copy import deepcopy -from numba import njit - -from .. import register_model -from . import utils -from ..gcn import GCN -from ..base import BaseAutoModel -from .....utils import get_logger - -LOGGER = get_logger("GCNSVDModel") - - -### ========================== ### - -class GCN4Robust(GCN): - # 在已有gcn的基础上增加robust的部分 - def __init__(self, nfeat, nclass, nhid, activation, dropout=0.5, lr=0.01, weight_decay=5e-4, with_relu=True, with_bias=True, add_self_loops = True, normalize = TRUE): - super(GCN4Robust, self).__init__(nfeat, nclass, nhid, activation, dropout=dropout, add_self_loops = add_self_loops, normalize = normalize) - - def fit(self, features, adj, labels, idx_train, idx_val=None, train_iters=200, initialize=True, verbose=False, normalize=True, patience=500, **kwargs): - self.device = self.gc1.weight.device - if initialize: - self.initialize() - - if type(adj) is not torch.Tensor: - features, adj, labels = utils.to_tensor(features, adj, labels, device=self.device) - else: - features = features.to(self.device) - adj = adj.to(self.device) - labels = labels.to(self.device) - - if normalize: - if utils.is_sparse_tensor(adj): - adj_norm = utils.normalize_adj_tensor(adj, sparse=True) - else: - adj_norm = utils.normalize_adj_tensor(adj) - else: - adj_norm = adj - - self.adj_norm = adj_norm - self.features = features - self.labels = labels - - if idx_val is None: - self._train_without_val(labels, idx_train, train_iters, verbose) - else: - if patience < train_iters: - self._train_with_early_stopping(labels, idx_train, idx_val, train_iters, patience, verbose) - else: - self._train_with_val(labels, idx_train, idx_val, train_iters, verbose) - - def _train_without_val(self, labels, idx_train, train_iters, verbose): - self.train() - optimizer = optim.Adam(self.parameters(), lr=self.lr, weight_decay=self.weight_decay) - for i in range(train_iters): - optimizer.zero_grad() - output = self.forward(self.features, self.adj_norm) - loss_train = F.nll_loss(output[idx_train], labels[idx_train]) - loss_train.backward() - optimizer.step() - if verbose and i % 10 == 0: - print('Epoch {}, training loss: {}'.format(i, loss_train.item())) - - self.eval() - output = self.forward(self.features, self.adj_norm) - self.output = output - - def _train_with_val(self, labels, idx_train, idx_val, train_iters, verbose): - if verbose: - print('=== training gcn model ===') - optimizer = optim.Adam(self.parameters(), lr=self.lr, weight_decay=self.weight_decay) - - best_loss_val = 100 - best_acc_val = 0 - - for i in range(train_iters): - self.train() - optimizer.zero_grad() - output = self.forward(self.features, self.adj_norm) - loss_train = F.nll_loss(output[idx_train], labels[idx_train]) - loss_train.backward() - optimizer.step() - - if verbose and i % 10 == 0: - print('Epoch {}, training loss: {}'.format(i, loss_train.item())) - - self.eval() - output = self.forward(self.features, self.adj_norm) - loss_val = F.nll_loss(output[idx_val], labels[idx_val]) - acc_val = utils.accuracy(output[idx_val], labels[idx_val]) - - if best_loss_val > loss_val: - best_loss_val = loss_val - self.output = output - weights = deepcopy(self.state_dict()) - - if acc_val > best_acc_val: - best_acc_val = acc_val - self.output = output - weights = deepcopy(self.state_dict()) - - if verbose: - print('=== picking the best model according to the performance on validation ===') - self.load_state_dict(weights) - - def _train_with_early_stopping(self, labels, idx_train, idx_val, train_iters, patience, verbose): - if verbose: - print('=== training gcn model ===') - optimizer = optim.Adam(self.parameters(), lr=self.lr, weight_decay=self.weight_decay) - - early_stopping = patience - best_loss_val = 100 - - for i in range(train_iters): - self.train() - optimizer.zero_grad() - output = self.forward(self.features, self.adj_norm) - loss_train = F.nll_loss(output[idx_train], labels[idx_train]) - loss_train.backward() - optimizer.step() - - if verbose and i % 10 == 0: - print('Epoch {}, training loss: {}'.format(i, loss_train.item())) - - self.eval() - output = self.forward(self.features, self.adj_norm) - - # def eval_class(output, labels): - # preds = output.max(1)[1].type_as(labels) - # return f1_score(labels.cpu().numpy(), preds.cpu().numpy(), average='micro') + \ - # f1_score(labels.cpu().numpy(), preds.cpu().numpy(), average='macro') - - # perf_sum = eval_class(output[idx_val], labels[idx_val]) - loss_val = F.nll_loss(output[idx_val], labels[idx_val]) - - if best_loss_val > loss_val: - best_loss_val = loss_val - self.output = output - weights = deepcopy(self.state_dict()) - patience = early_stopping - else: - patience -= 1 - if i > early_stopping and patience <= 0: - break - - if verbose: - print('=== early stopping at {0}, loss_val = {1} ==='.format(i, best_loss_val) ) - self.load_state_dict(weights) - -class GCNSVD(GCN4Robust): - def __init__(self, nfeat, nclass, nhid, activation, dropout=0.5, lr=0.01, weight_decay=5e-4, with_relu=True, with_bias=True, add_self_loops = True, normalize = True): - super(GCNSVD, self).__init__(nfeat, nclass, nhid, activation, dropout, lr, weight_decay, with_relu, with_bias, add_self_loops, normalize) - - def fit(self, features, adj, labels, idx_train, idx_val=None, k=50, train_iters=200, initialize=True, verbose=True, **kwargs): - modified_adj = self.truncatedSVD(adj, k=k) - self.k = k - # modified_adj_tensor = utils.sparse_mx_to_torch_sparse_tensor(self.modified_adj) - features, modified_adj, labels = utils.to_tensor(features, modified_adj, labels, device=self.device) - - self.modified_adj = modified_adj - self.features = features - self.labels = labels - super().fit(features, modified_adj, labels, idx_train, idx_val, train_iters=train_iters, initialize=initialize, verbose=verbose) - - def truncatedSVD(self, data, k=50): - print('=== GCN-SVD: rank={} ==='.format(k)) - if sp.issparse(data): - data = data.asfptype() - U, S, V = sp.linalg.svds(data, k=k) - print("rank_after = {}".format(len(S.nonzero()[0]))) - diag_S = np.diag(S) - else: - U, S, V = np.linalg.svd(data) - U = U[:, :k] - S = S[:k] - V = V[:k, :] - print("rank_before = {}".format(len(S.nonzero()[0]))) - diag_S = np.diag(S) - print("rank_after = {}".format(len(diag_S.nonzero()[0]))) - - return U @ diag_S @ V - - def predict(self, features=None, adj=None): - - self.eval() - if features is None and adj is None: - return self.forward(self.features, self.adj_norm) - else: - adj = self.truncatedSVD(adj, k=self.k) - if type(adj) is not torch.Tensor: - features, adj = utils.to_tensor(features, adj, device=self.device) - - self.features = features - if utils.is_sparse_tensor(adj): - self.adj_norm = utils.normalize_adj_tensor(adj, sparse=True) - else: - self.adj_norm = utils.normalize_adj_tensor(adj) - return self.forward(self.features, self.adj_norm) - -@register_model("gcnsvd-model") -class AutoGCNSVD(BaseAutoModel): - def __init__( - self, - num_features: int = ..., - num_classes: int = ..., - device: _typing.Union[str, torch.device] = ..., - **kwargs - ) -> None: - super().__init__(num_features, num_classes, device, **kwargs) - self.hyper_parameter_space = [ - { - "parameterName": "add_self_loops", - "type": "CATEGORICAL", - "feasiblePoints": [1], - }, - { - "parameterName": "normalize", - "type": "CATEGORICAL", - "feasiblePoints": [1], - }, - { - "parameterName": "num_layers", - "type": "DISCRETE", - "feasiblePoints": "2,3,4", - }, - { - "parameterName": "hidden", - "type": "NUMERICAL_LIST", - "numericalType": "INTEGER", - "length": 3, - "minValue": [8, 8, 8], - "maxValue": [128, 128, 128], - "scalingType": "LOG", - "cutPara": ("num_layers",), - "cutFunc": lambda x: x[0] - 1, - }, - { - "parameterName": "dropout", - "type": "DOUBLE", - "maxValue": 0.8, - "minValue": 0.2, - "scalingType": "LINEAR", - }, - { - "parameterName": "act", - "type": "CATEGORICAL", - "feasiblePoints": ["leaky_relu", "relu", "elu", "tanh"], - }, - ] - - self.hyper_parameters = { - "num_layers": 3, - "hidden": [128, 64], - "dropout": 0, - "act": "relu", - } - - def _initialize(self): - self._model = GCNSVD( - self.input_dimension, - self.output_dimension, - self.hyper_parameters.get("hidden"), - self.hyper_parameters.get("act"), - self.hyper_parameters.get("dropout", None), - bool(self.hyper_parameters.get("add_self_loops", True)), - bool(self.hyper_parameters.get("normalize", True)), - ).to(self.device) diff --git a/autogl/module/model/pyg/robust/gnnguard.py b/autogl/module/model/pyg/robust/gnnguard.py index 8171545..42e7d23 100644 --- a/autogl/module/model/pyg/robust/gnnguard.py +++ b/autogl/module/model/pyg/robust/gnnguard.py @@ -39,6 +39,12 @@ class GCN4GNNGuard(GCN): self.nfeat = nfeat self.hidden_sizes = nhid self.drop = drop + if not with_relu: + self.weight_decay = 0 + else: + self.weight_decay = weight_decay + self.with_relu = with_relu + self.with_bias = with_bias self.gc1 = GCNConv(nfeat, nhid[0], bias=True,) self.gc2 = GCNConv(nhid[0], nclass, bias=True, ) @@ -345,6 +351,235 @@ class GCN4GNNGuard(GCN): self.adj_norm = utils.normalize_adj_tensor(adj) return self.forward(self.features, self.adj_norm) +class GCN4GNNGuard_attack(GCN): + # Based on the existing GCN, add the robust part. + def __init__(self, nfeat, nclass, nhid, activation, dropout=0.5, lr=0.01, drop=False, weight_decay=5e-4, with_relu=True, with_bias=True, add_self_loops = True, normalize = True): + super(GCN4GNNGuard_attack, self).__init__(nfeat, nclass, nhid, activation, dropout=dropout, add_self_loops = add_self_loops, normalize = normalize) + + self.lr = lr + self.weight_decay = weight_decay + self.dropout = dropout + self.nclass = nclass + self.nfeat = nfeat + self.hidden_sizes = nhid + self.drop = drop + if not with_relu: + self.weight_decay = 0 + else: + self.weight_decay = weight_decay + self.with_relu = with_relu + self.with_bias = with_bias + + self.gc1 = GCNConv(nfeat, nhid[0], bias=True,) + self.gc2 = GCNConv(nhid[0], nclass, bias=True, ) + + def forward(self, x, adj_lil): + """we don't change the edge_index, just update the edge_weight; + some edge_weight are regarded as removed if it equals to zero""" + x = x.to_dense() + adj = adj_lil.coalesce().indices() + edge_weight = adj_lil.coalesce().values() + + x = F.relu(self.gc1(x, adj, edge_weight=edge_weight)) + x = F.dropout(x, self.dropout, training=self.training) + x = self.gc2(x, adj, edge_weight=edge_weight) + + return F.log_softmax(x, dim=1) + + def add_loop_sparse(self, adj, fill_value=1): + # make identify sparse tensor + row = torch.range(0, int(adj.shape[0]-1), dtype=torch.int64) + i = torch.stack((row, row), dim=0) + v = torch.ones(adj.shape[0], dtype=torch.float32) + shape = adj.shape + I_n = torch.sparse.FloatTensor(i, v, shape) + return adj + I_n.to(self.device) + + def initialize(self): + self.gc1.reset_parameters() + self.gc2.reset_parameters() + + def fit(self, features, adj, labels, idx_train, idx_val=None, idx_test=None, train_iters=81, att_0=None, attention=False, model_name=None, initialize=True, verbose=False, normalize=False, patience=510, ): + ''' + train the gcn model, when idx_val is not None, pick the best model + according to the validation loss + ''' + sd = self.state_dict() + for v in sd.values(): + self.device = v.device + break + + self.sim = None + self.attention = attention + if self.attention: + att_0 = self.att_coef_1(features, adj) + adj = att_0 # update adj + self.sim = att_0 # update att_0 + + self.idx_test = idx_test + + if initialize: + self.initialize() + + if type(adj) is not torch.Tensor: + features, adj, labels = utils.to_tensor(features, adj, labels, device=self.device) + else: + features = features.to(self.device) + adj = adj.to(self.device) + labels = labels.to(self.device) + + normalize = False # we don't need normalize here, the norm is conducted in the GCN (self.gcn1) model + if normalize: + if utils.is_sparse_tensor(adj): + adj_norm = utils.normalize_adj_tensor(adj, sparse=True) + else: + adj_norm = utils.normalize_adj_tensor(adj) + else: + adj_norm = adj + # add self loop + # adj = self.add_loop_sparse(adj) + + + """Make the coefficient D^{-1/2}(A+I)D^{-1/2}""" + self.adj_norm = adj_norm + self.features = features + self.labels = labels + + if idx_val is None: + self._train_without_val(labels, idx_train, train_iters, verbose) + else: + if patience < train_iters: + self._train_with_early_stopping(labels, idx_train, idx_val, train_iters, patience, verbose) + else: + self._train_with_val(labels, idx_train, idx_val, train_iters, verbose) + + def _train_without_val(self, labels, idx_train, train_iters, verbose): + self.train() + optimizer = optim.Adam(self.parameters(), lr=self.lr, weight_decay=self.weight_decay) + for i in range(train_iters): + optimizer.zero_grad() + output = self.forward(self.features, self.adj_norm) + loss_train = F.nll_loss(output[idx_train], labels[idx_train], weight=None) # this weight is the weight of each training nodes + loss_train.backward() + optimizer.step() + if verbose and i % 10 == 0: + print('Epoch {}, training loss: {}'.format(i, loss_train.item())) + + self.eval() + output = self.forward(self.features, self.adj_norm) + self.output = output + + def _train_with_val(self, labels, idx_train, idx_val, train_iters, verbose): + if verbose: + print('=== training gcn model ===') + optimizer = optim.Adam(self.parameters(), lr=self.lr, weight_decay=self.weight_decay) + + best_loss_val = 100 + best_acc_val = 0 + + for i in range(train_iters): + # print('epoch', i) + self.train() + optimizer.zero_grad() + output = self.forward(self.features, self.adj_norm) + loss_train = F.nll_loss(output[idx_train], labels[idx_train]) + loss_train.backward() + optimizer.step() + + acc_test =accuracy(output[self.idx_test], labels[self.idx_test]) + + self.eval() + output = self.forward(self.features, self.adj_norm) + loss_val = F.nll_loss(output[idx_val], labels[idx_val]) + acc_val = utils.accuracy(output[idx_val], labels[idx_val]) + + if verbose and i % 200 == 0: + print('Epoch {}, training loss: {}, test acc: {}'.format(i, loss_train.item(), acc_test)) + + if best_loss_val > loss_val: + best_loss_val = loss_val + self.output = output + weights = deepcopy(self.state_dict()) + + if acc_val > best_acc_val: + best_acc_val = acc_val + self.output = output + weights = deepcopy(self.state_dict()) + + if verbose: + print('=== picking the best model according to the performance on validation ===') + self.load_state_dict(weights) + + + def _train_with_early_stopping(self, labels, idx_train, idx_val, train_iters, patience, verbose): + if verbose: + print('=== training gcn model ===') + optimizer = optim.Adam(self.parameters(), lr=self.lr, weight_decay=self.weight_decay) + + early_stopping = patience + best_loss_val = 100 + + for i in range(train_iters): + self.train() + optimizer.zero_grad() + output = self.forward(self.features, self.adj_norm) + loss_train = F.nll_loss(output[idx_train], labels[idx_train]) + loss_train.backward() + optimizer.step() + + self.eval() + output = self.forward(self.features, self.adj_norm) + + if verbose and i % 10 == 0: + print('Epoch {}, training loss: {}'.format(i, loss_train.item())) + + + loss_val = F.nll_loss(output[idx_val], labels[idx_val]) + + if best_loss_val > loss_val: + best_loss_val = loss_val + self.output = output + weights = deepcopy(self.state_dict()) + patience = early_stopping + else: + patience -= 1 + if i > early_stopping and patience <= 0: + break + + if verbose: + print('=== early stopping at {0}, loss_val = {1} ==='.format(i, best_loss_val) ) + self.load_state_dict(weights) + + def test(self, idx_test): + self.eval() + output = self.predict() # here use the self.features and self.adj_norm in training stage + loss_test = F.nll_loss(output[idx_test], self.labels[idx_test]) + acc_test = utils.accuracy(output[idx_test], self.labels[idx_test]) + print("Test set results:", + "loss= {:.4f}".format(loss_test.item()), + "accuracy= {:.4f}".format(acc_test.item())) + return acc_test, output + + def _set_parameters(self): + # TODO + pass + + def predict(self, features=None, adj=None): + '''By default, inputs are unnormalized data''' + # self.eval() + if features is None and adj is None: + return self.forward(self.features, self.adj_norm) + else: + if type(adj) is not torch.Tensor: + features, adj = utils.to_tensor(features, adj, device=self.device) + + self.features = features + if utils.is_sparse_tensor(adj): + self.adj_norm = utils.normalize_adj_tensor(adj, sparse=True) + else: + self.adj_norm = utils.normalize_adj_tensor(adj) + return self.forward(self.features, self.adj_norm) + @register_model("gnnguard-model") class AutoGNNGuard(BaseAutoModel): @@ -414,3 +649,72 @@ class AutoGNNGuard(BaseAutoModel): add_self_loops = bool(self.hyper_parameters.get("add_self_loops", True)), normalize = bool(self.hyper_parameters.get("normalize", True)), ).to(self.device) + +@register_model("gnnguard-attack-model") +class AutoGNNGuard_attack(BaseAutoModel): + def __init__( + self, + num_features: int = ..., + num_classes: int = ..., + device: _typing.Union[str, torch.device] = ..., + **kwargs + ) -> None: + super().__init__(num_features, num_classes, device, **kwargs) + self.hyper_parameter_space = [ + { + "parameterName": "add_self_loops", + "type": "CATEGORICAL", + "feasiblePoints": [1], + }, + { + "parameterName": "normalize", + "type": "CATEGORICAL", + "feasiblePoints": [1], + }, + { + "parameterName": "num_layers", + "type": "DISCRETE", + "feasiblePoints": "2,3,4", + }, + { + "parameterName": "hidden", + "type": "NUMERICAL_LIST", + "numericalType": "INTEGER", + "length": 3, + "minValue": [8, 8, 8], + "maxValue": [128, 128, 128], + "scalingType": "LOG", + "cutPara": ("num_layers",), + "cutFunc": lambda x: x[0] - 1, + }, + { + "parameterName": "dropout", + "type": "DOUBLE", + "maxValue": 0.8, + "minValue": 0.2, + "scalingType": "LINEAR", + }, + { + "parameterName": "act", + "type": "CATEGORICAL", + "feasiblePoints": ["leaky_relu", "relu", "elu", "tanh"], + }, + ] + + self.hyper_parameters = { + "num_layers": 3, + "hidden": [128, 64], + "dropout": 0, + "act": "relu", + } + + def _initialize(self): + self._model = GCN4GNNGuard_attack( + nfeat = self.input_dimension, + nclass = self.output_dimension, + nhid = self.hyper_parameters.get("hidden"), + activation = self.hyper_parameters.get("act"), + dropout = self.hyper_parameters.get("dropout", None), + add_self_loops = bool(self.hyper_parameters.get("add_self_loops", True)), + normalize = bool(self.hyper_parameters.get("normalize", True)), + ).to(self.device) \ No newline at end of file diff --git a/autogl/module/model/pyg/robust/nn/conv/gcn_conv.py b/autogl/module/model/pyg/robust/nn/conv/gcn_conv.py new file mode 100644 index 0000000..e82176b --- /dev/null +++ b/autogl/module/model/pyg/robust/nn/conv/gcn_conv.py @@ -0,0 +1,196 @@ +import torch +from torch.nn import Parameter +from torch_scatter import scatter_add +from torch_geometric.nn.conv import MessagePassing +from torch_geometric.utils import add_remaining_self_loops, to_undirected + +from ..inits import glorot, zeros + +@torch.jit._overload +def gcn_norm(edge_index, edge_weight=None, num_nodes=None, improved=False, + add_self_loops=True, dtype=None): + # type: (Tensor, OptTensor, Optional[int], bool, bool, Optional[int]) -> PairTensor # noqa + pass + + +@torch.jit._overload +def gcn_norm(edge_index, edge_weight=None, num_nodes=None, improved=False, + add_self_loops=True, dtype=None): + # type: (SparseTensor, OptTensor, Optional[int], bool, bool, Optional[int]) -> SparseTensor # noqa + pass + + +def gcn_norm(edge_index, edge_weight=None, num_nodes=None, improved=False, + add_self_loops=True, dtype=None): + + fill_value = 2. if improved else 1. + + if isinstance(edge_index, SparseTensor): + adj_t = edge_index + if not adj_t.has_value(): + adj_t = adj_t.fill_value(1., dtype=dtype) + if add_self_loops: + adj_t = fill_diag(adj_t, fill_value) + deg = sum(adj_t, dim=1) + deg_inv_sqrt = deg.pow_(-0.5) + deg_inv_sqrt.masked_fill_(deg_inv_sqrt == float('inf'), 0.) + adj_t = mul(adj_t, deg_inv_sqrt.view(-1, 1)) + adj_t = mul(adj_t, deg_inv_sqrt.view(1, -1)) + return adj_t + + else: + num_nodes = maybe_num_nodes(edge_index, num_nodes) + + if edge_weight is None: + edge_weight = torch.ones((edge_index.size(1), ), dtype=dtype, + device=edge_index.device) + + if add_self_loops: + edge_index, tmp_edge_weight = add_remaining_self_loops( + edge_index, edge_weight, fill_value, num_nodes) + assert tmp_edge_weight is not None + edge_weight = tmp_edge_weight + + row, col = edge_index[0], edge_index[1] + deg = scatter_add(edge_weight, col, dim=0, dim_size=num_nodes) + deg_inv_sqrt = deg.pow_(-0.5) + deg_inv_sqrt.masked_fill_(deg_inv_sqrt == float('inf'), 0) + return edge_index, deg_inv_sqrt[row] * edge_weight * deg_inv_sqrt[col] + + + +class GCNConv(MessagePassing): + r"""The graph convolutional operator from the `"Semi-supervised + Classification with Graph Convolutional Networks" + `_ paper + + .. math:: + \mathbf{X}^{\prime} = \mathbf{\hat{D}}^{-1/2} \mathbf{\hat{A}} + \mathbf{\hat{D}}^{-1/2} \mathbf{X} \mathbf{\Theta}, + + where :math:`\mathbf{\hat{A}} = \mathbf{A} + \mathbf{I}` denotes the + adjacency matrix with inserted self-loops and + :math:`\hat{D}_{ii} = \sum_{j=0} \hat{A}_{ij}` its diagonal degree matrix. + + Args: + in_channels (int): Size of each input sample. + out_channels (int): Size of each output sample. + improved (bool, optional): If set to :obj:`True`, the layer computes + :math:`\mathbf{\hat{A}}` as :math:`\mathbf{A} + 2\mathbf{I}`. + (default: :obj:`False`) + cached (bool, optional): If set to :obj:`True`, the layer will cache + the computation of :math:`\mathbf{\hat{D}}^{-1/2} \mathbf{\hat{A}} + \mathbf{\hat{D}}^{-1/2}` on first execution, and will use the + cached version for further executions. + This parameter should only be set to :obj:`True` in transductive + learning scenarios. (default: :obj:`False`) + bias (bool, optional): If set to :obj:`False`, the layer will not learn + an additive bias. (default: :obj:`True`) + normalize (bool, optional): Whether to add self-loops and apply + symmetric normalization. (default: :obj:`True`) + **kwargs (optional): Additional arguments of + :class:`torch_geometric.nn.conv.MessagePassing`. + """ + + def __init__(self, in_channels, out_channels, improved=False, cached=False, + bias=True, add_self_loops: bool = True, normalize=True, **kwargs): + super(GCNConv, self).__init__(aggr='add', **kwargs) + + self.in_channels = in_channels + self.out_channels = out_channels + self.improved = improved + self.cached = cached + self.normalize = normalize + + self.weight = Parameter(torch.Tensor(in_channels, out_channels)) + + if bias: + self.bias = Parameter(torch.tensor(out_channels, dtype=torch.float32)) + else: + self.register_parameter('bias', None) + + self.reset_parameters() + + def reset_parameters(self): + glorot(self.weight) + zeros(self.bias) + self.cached_result = None + self.cached_num_edges = None + + # 原来的版本 + # @staticmethod + # def norm(edge_index, num_nodes, edge_weight=None, improved=False, + # dtype=None): + # if edge_weight is None: + # edge_weight = torch.ones((edge_index.size(1), ), dtype=dtype, + # device=edge_index.device) + + # fill_value = 1 if not improved else 2 + # # """Here I removed the self-loop because the self-loop already added in the att_coef function""" + # # edge_index, edge_weight = add_remaining_self_loops( + # # edge_index, edge_weight, fill_value, num_nodes) + + # row, col = edge_index + # deg = scatter_add(edge_weight, row, dim=0, dim_size=num_nodes) + # deg_inv_sqrt = deg.pow(-0.5) + # deg_inv_sqrt[deg_inv_sqrt == float('inf')] = 0 + + # return edge_index, deg_inv_sqrt[row] * edge_weight * deg_inv_sqrt[col] + + # for GNNGuard + @staticmethod + def norm(edge_index, num_nodes, edge_weight=None, improved=False, + dtype=None): + if edge_weight is None: + edge_weight = torch.ones((edge_index.size(1), ), dtype=dtype, + device=edge_index.device) + edge_weight = edge_weight.to(edge_index.device) + fill_value = 1 if not improved else 2 + # """Here I removed the self-loop because the self-loop already added in the att_coef function""" + # edge_index, edge_weight = add_remaining_self_loops( + # edge_index, edge_weight, fill_value, num_nodes) + + row, col = edge_index # for GNNGuard + # row, col = edge_index[0], edge_index[1] + deg = scatter_add(edge_weight, row, dim=0, dim_size=num_nodes) + deg_inv_sqrt = deg.pow(-0.5) + deg_inv_sqrt[deg_inv_sqrt == float('inf')] = 0 + + return edge_index, deg_inv_sqrt[row] * edge_weight * deg_inv_sqrt[col] + + + def forward(self, x, edge_index, edge_weight=None): + """""" + x = torch.matmul(x, self.weight) + + if self.cached and self.cached_result is not None: + if edge_index.size(1) != self.cached_num_edges: + raise RuntimeError( + 'Cached {} number of edges, but found {}. Please ' + 'disable the caching behavior of this layer by removing ' + 'the `cached=True` argument in its constructor.'.format( + self.cached_num_edges, edge_index.size(1))) + # edge_index = to_undirected(edge_index, x.size(0)) # add non-direct edges + if not self.cached or self.cached_result is None: + self.cached_num_edges = edge_index.size(1) + if self.normalize: + edge_index, norm = self.norm(edge_index, x.size(0), edge_weight, self.improved, x.dtype) + else: + norm = edge_weight + self.cached_result = edge_index, norm + + edge_index, norm = self.cached_result + + return self.propagate(edge_index, x=x, norm=norm) + + def message(self, x_j, norm): + return norm.view(-1, 1) * x_j + + def update(self, aggr_out): + if self.bias is not None: + aggr_out = aggr_out + self.bias + return aggr_out + + def __repr__(self): + return '{}({}, {})'.format(self.__class__.__name__, self.in_channels, + self.out_channels) \ No newline at end of file diff --git a/docs/docfile/tutorial/t_homo_link_prediction.rst b/docs/docfile/tutorial/t_homo_link_prediction.rst new file mode 100644 index 0000000..2f092aa --- /dev/null +++ b/docs/docfile/tutorial/t_homo_link_prediction.rst @@ -0,0 +1,204 @@ +========================== +Link Prediction Model +========================== + +Building Link Prediction Modules +===================================== + +In AutoGL, we support three models for link prediction models, ``gcn``, ``gat`` and ``sage``. + +AutoLinkPredictor +>>>>>>> + +Used to automatically solve the link prediction problems. For example, + + +.. code-block:: python + + class AutoGCN(BaseAutoModel): + r""" + AutoGCN. + The model used in this automodel is GCN, i.e., the graph convolutional network from the + `"Semi-supervised Classification with Graph Convolutional + Networks" `_ paper. The layer is + + .. math:: + + \mathbf{X}^{\prime} = \mathbf{\hat{D}}^{-1/2} \mathbf{\hat{A}} + \mathbf{\hat{D}}^{-1/2} \mathbf{X} \mathbf{\Theta}, + + where :math:`\mathbf{\hat{A}} = \mathbf{A} + \mathbf{I}` denotes the + adjacency matrix with inserted self-loops and + :math:`\hat{D}_{ii} = \sum_{j=0} \hat{A}_{ij}` its diagonal degree matrix. + + Parameters + ---------- + num_features: ``int`` + The dimension of features. + + num_classes: ``int`` + The number of classes. + + device: ``torch.device`` or ``str`` + The device where model will be running on. + + init: `bool`. + If True(False), the model will (not) be initialized. + """ + + def __init__( + self, + num_features: int = ..., + num_classes: int = ..., + device: _typing.Union[str, torch.device] = ..., + **kwargs + ) -> None: + super().__init__(num_features, num_classes, device, **kwargs) + self.hyper_parameter_space = [ + { + "parameterName": "add_self_loops", + "type": "CATEGORICAL", + "feasiblePoints": [1], + }, + { + "parameterName": "normalize", + "type": "CATEGORICAL", + "feasiblePoints": [1], + }, + { + "parameterName": "num_layers", + "type": "DISCRETE", + "feasiblePoints": "2,3,4", + }, + { + "parameterName": "hidden", + "type": "NUMERICAL_LIST", + "numericalType": "INTEGER", + "length": 3, + "minValue": [8, 8, 8], + "maxValue": [128, 128, 128], + "scalingType": "LOG", + "cutPara": ("num_layers",), + "cutFunc": lambda x: x[0] - 1, + }, + { + "parameterName": "dropout", + "type": "DOUBLE", + "maxValue": 0.8, + "minValue": 0.2, + "scalingType": "LINEAR", + }, + { + "parameterName": "act", + "type": "CATEGORICAL", + "feasiblePoints": ["leaky_relu", "relu", "elu", "tanh"], + }, + ] + + self.hyper_parameters = { + "num_layers": 3, + "hidden": [128, 64], + "dropout": 0, + "act": "relu", + } + + def _initialize(self): + self._model = GCN( + self.input_dimension, + self.output_dimension, + self.hyper_parameters.get("hidden"), + self.hyper_parameters.get("act"), + self.hyper_parameters.get("dropout", None), + bool(self.hyper_parameters.get("add_self_loops", True)), + bool(self.hyper_parameters.get("normalize", True)), + ).to(self.device) + +You could get define your own ``LinkPrediction`` model by using ``from_hyper_parameter`` function and specify the hyperpameryers. + +.. code-block:: python + + # pyg version + from autogl.module.model.pyg import AutoLinkPredictor + # from autogl.module.model.dgl import AutoLinkPredictor # dgl version + model = AutoLinkPredictor( + feature_module="NormalizeFeatures", + graph_models=(args.model, ), + hpo_module="random", + ensemble_module=None, + max_evals=1, + trainer_hp_space=fixed(**{ + "max_epoch": 100, + "early_stopping_round": 101, + "lr": 1e-2, + "weight_decay": 0.0, + }), + model_hp_spaces=[{"encoder": fixed(**model_hp), "decoder": fixed(**decoder_hp)}] + ).model + +Then you can train the model for 100 epochs. + +.. code-block:: python + + import torch.nn.functional as F + + # Define the loss optimizer. + optimizer = torch.optim.Adam(model.parameters(), lr=0.01) + + # Training + for epoch in range(100): + model.train() + optimizer.zero_grad() + + z = model.lp_encode(splitted[0]) + link_logits = model.lp_decode( + z, torch.stack(splitted[1].edges()), torch.stack(splitted[2].edges()) + ) + link_labels = get_link_labels( + torch.stack(splitted[1].edges()), torch.stack(splitted[2].edges()) + ) + loss = F.binary_cross_entropy_with_logits(link_logits, link_labels) + loss.backward() + optimizer.step() + + auc_val = evaluate(model, splitted, "val") + + if auc_val > best_auc: + best_auc = auc_val + best_parameters = pickle.dumps(model.state_dict()) + +Finally, evaluate the trained model. + +.. code-block:: python + + model.load_state_dict(pickle.loads(best_parameters)) + evaluate(model, splitted, "test") + + +Automatic Search for Link Prediction Tasks +=============================================== + +In AutoGL, we also provide a high-level API Solver to control the overall pipeline. +We encapsulated the training process in the Building GNN Modules part for link prediction tasks +in the solver ``AutoLinkPredictor`` that supports automatic hyperparametric optimization +as well as feature engineering and ensemble. In this part, we will show you how to use +``AutoLinkPredictor``. + +.. code-block:: python + + solver = AutoLinkPredictor( + feature_module="NormalizeFeatures", + graph_models=(args.model, ), + hpo_module="random", + ensemble_module=None, + max_evals=1, + trainer_hp_space=fixed(**{ + "max_epoch": 100, + "early_stopping_round": 101, + "lr": 1e-2, + "weight_decay": 0.0, + }), + model_hp_spaces=[{"encoder": fixed(**model_hp), "decoder": fixed(**decoder_hp)}] + ) + + solver.fit(dataset, train_split=0.85, val_split=0.05, evaluation_method=["auc"], seed=seed) + pre = solver.evaluate(metric="auc") diff --git a/docs/docfile/tutorial_cn/t_homo_link_prediction.rst b/docs/docfile/tutorial_cn/t_homo_link_prediction.rst new file mode 100644 index 0000000..0589ca6 --- /dev/null +++ b/docs/docfile/tutorial_cn/t_homo_link_prediction.rst @@ -0,0 +1,212 @@ +: _homo_cn: + +========================== +链接预测模型 +========================== + +构建链接预测模块 +===================================== +.. In AutoGL, we support two graph classification models, ``gin`` and ``topk``. +在AutoGL中，我们支持三种链接预测模型： ``gcn``, ``gat`` and ``sage`` 。 + +AutoLinkPredictor +>>>>>>> + +.. Used to automatically solve the link prediction problems. For example, +用于自动解决链路预测问题。例如, + +.. code-block:: python + + class AutoGCN(BaseAutoModel): + r""" + AutoGCN. + The model used in this automodel is GCN, i.e., the graph convolutional network from the + `"Semi-supervised Classification with Graph Convolutional + Networks" `_ paper. The layer is + + .. math:: + + \mathbf{X}^{\prime} = \mathbf{\hat{D}}^{-1/2} \mathbf{\hat{A}} + \mathbf{\hat{D}}^{-1/2} \mathbf{X} \mathbf{\Theta}, + + where :math:`\mathbf{\hat{A}} = \mathbf{A} + \mathbf{I}` denotes the + adjacency matrix with inserted self-loops and + :math:`\hat{D}_{ii} = \sum_{j=0} \hat{A}_{ij}` its diagonal degree matrix. + + Parameters + ---------- + num_features: ``int`` + The dimension of features. + + num_classes: ``int`` + The number of classes. + + device: ``torch.device`` or ``str`` + The device where model will be running on. + + init: `bool`. + If True(False), the model will (not) be initialized. + """ + + def __init__( + self, + num_features: int = ..., + num_classes: int = ..., + device: _typing.Union[str, torch.device] = ..., + **kwargs + ) -> None: + super().__init__(num_features, num_classes, device, **kwargs) + self.hyper_parameter_space = [ + { + "parameterName": "add_self_loops", + "type": "CATEGORICAL", + "feasiblePoints": [1], + }, + { + "parameterName": "normalize", + "type": "CATEGORICAL", + "feasiblePoints": [1], + }, + { + "parameterName": "num_layers", + "type": "DISCRETE", + "feasiblePoints": "2,3,4", + }, + { + "parameterName": "hidden", + "type": "NUMERICAL_LIST", + "numericalType": "INTEGER", + "length": 3, + "minValue": [8, 8, 8], + "maxValue": [128, 128, 128], + "scalingType": "LOG", + "cutPara": ("num_layers",), + "cutFunc": lambda x: x[0] - 1, + }, + { + "parameterName": "dropout", + "type": "DOUBLE", + "maxValue": 0.8, + "minValue": 0.2, + "scalingType": "LINEAR", + }, + { + "parameterName": "act", + "type": "CATEGORICAL", + "feasiblePoints": ["leaky_relu", "relu", "elu", "tanh"], + }, + ] + + self.hyper_parameters = { + "num_layers": 3, + "hidden": [128, 64], + "dropout": 0, + "act": "relu", + } + + def _initialize(self): + self._model = GCN( + self.input_dimension, + self.output_dimension, + self.hyper_parameters.get("hidden"), + self.hyper_parameters.get("act"), + self.hyper_parameters.get("dropout", None), + bool(self.hyper_parameters.get("add_self_loops", True)), + bool(self.hyper_parameters.get("normalize", True)), + ).to(self.device) + + +.. You could get define your own ``LinkPrediction`` model by using ``from_hyper_parameter`` function and specify the hyperpameryers. +You could get define your own ``gin`` model by using ``from_hyper_parameter`` function and specify the hyperpameryers. +你可以通过使用 ``from_hyper_parameter`` 函数定义你自己的 ``LinkPrediction`` 模型，并对其指定超参数。 + +.. code-block:: python + + # pyg version + from autogl.module.model.pyg import AutoLinkPredictor + # from autogl.module.model.dgl import AutoLinkPredictor # dgl version + model = AutoLinkPredictor( + feature_module="NormalizeFeatures", + graph_models=(args.model, ), + hpo_module="random", + ensemble_module=None, + max_evals=1, + trainer_hp_space=fixed(**{ + "max_epoch": 100, + "early_stopping_round": 101, + "lr": 1e-2, + "weight_decay": 0.0, + }), + model_hp_spaces=[{"encoder": fixed(**model_hp), "decoder": fixed(**decoder_hp)}] + ).model + + + +.. Then you can train the model for 100 epochs. +然后你可以对模型进行100次的训练： + +.. code-block:: python + + import torch.nn.functional as F + + # Define the loss optimizer. + optimizer = torch.optim.Adam(model.parameters(), lr=0.01) + + # Training + for epoch in range(100): + model.train() + optimizer.zero_grad() + + z = model.lp_encode(splitted[0]) + link_logits = model.lp_decode( + z, torch.stack(splitted[1].edges()), torch.stack(splitted[2].edges()) + ) + link_labels = get_link_labels( + torch.stack(splitted[1].edges()), torch.stack(splitted[2].edges()) + ) + loss = F.binary_cross_entropy_with_logits(link_logits, link_labels) + loss.backward() + optimizer.step() + + auc_val = evaluate(model, splitted, "val") + + if auc_val > best_auc: + best_auc = auc_val + best_parameters = pickle.dumps(model.state_dict()) + + +.. Finally, evaluate the trained model. +最后，你可以评估该模型： + +.. code-block:: python + + model.load_state_dict(pickle.loads(best_parameters)) + evaluate(model, splitted, "test") + +.. Automatic Search for Link Prediction Tasks +链接预测任务的自动搜索 +=============================================== + +.. In AutoGL, we also provide a high-level API Solver to control the overall pipeline. We encapsulated the training process in the Building GNN Modules part for link prediction tasks in the solver ``AutoLinkPredictor`` that supports automatic hyperparametric optimization as well as feature engineering and ensemble. In this part, we will show you how to use ``AutoLinkPredictor``. +在AutoGL中，我们还提供了一个高级的API求解器来控制整个流水线。我们将构建图神经网络模块部分的训练过程封装在求解器 ``AutoLinkPredictor`` 中以用于图分类任务，它支持自动超参数优化，特征工程及集成。 +在这一部分，我们提供了一个例子来指导如何使用 ``AutoLinkPredictor`` ： + +.. code-block:: python + + solver = AutoLinkPredictor( + feature_module="NormalizeFeatures", + graph_models=(args.model, ), + hpo_module="random", + ensemble_module=None, + max_evals=1, + trainer_hp_space=fixed(**{ + "max_epoch": 100, + "early_stopping_round": 101, + "lr": 1e-2, + "weight_decay": 0.0, + }), + model_hp_spaces=[{"encoder": fixed(**model_hp), "decoder": fixed(**decoder_hp)}] + ) + + solver.fit(dataset, train_split=0.85, val_split=0.05, evaluation_method=["auc"], seed=seed) + pre = solver.evaluate(metric="auc") \ No newline at end of file diff --git a/test/performance/robust/model_gcnsvd.py b/test/performance/robust/model_gcnsvd.py deleted file mode 100644 index 6daf039..0000000 --- a/test/performance/robust/model_gcnsvd.py +++ /dev/null @@ -1,130 +0,0 @@ -import os -import pickle -from torchaudio import datasets -from tqdm import tqdm -import torch -import numpy as np -import torch.nn.functional as F -import torch.optim as optim -from deeprobust.graph.defense import GCNSVD -from deeprobust.graph.utils import * -from deeprobust.graph.data import Dataset, PrePtbDataset, Dpr2Pyg, AmazonPyg -import argparse - -os.environ["AUTOGL_BACKEND"] = "pyg" - - -from autogl.module.model.pyg import AutoGCNSVD -from autogl.solver.utils import set_seed - -def test(model, data, mask): - model.eval() - - if hasattr(model, 'cls_forward'): - out = model.cls_forward(data)[mask] - else: - out = model(data)[mask] - pred = out.max(1)[1] - acc = pred.eq(data.y[mask]).sum().item() / mask.sum().item() - return acc - -def train(model, data, args): - optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) - print(data) - parameters = model.state_dict() - best_acc = 0. - for epoch in range(args.epoch): - model.train() - optimizer.zero_grad() - if hasattr(model, 'cls_forward'): - output = model.cls_forward(data) - else: - output = model(data) - loss = F.nll_loss(output[data.train_mask], data.y[data.train_mask]) - loss.backward() - optimizer.step() - - val_acc = test(model, data, data.val_mask) - if val_acc > best_acc: - best_acc = val_acc - parameters = pickle.dumps(model.state_dict()) - - model.load_state_dict(pickle.loads(parameters)) - return model - - -if __name__ == '__main__': - - parser = argparse.ArgumentParser() - parser = argparse.ArgumentParser('pyg model') - parser.add_argument('--seed', type=int, default=15, help='Random seed.') - parser.add_argument('--dataset', type=str, default='cora', choices=['cora', 'cora_ml', 'citeseer', 'polblogs', 'pubmed'], help='dataset') - parser.add_argument('--ptb_rate', type=float, default=0.05, help='pertubation rate') - parser.add_argument('--k', type=int, default=15, help='Truncated Components.') - parser.add_argument('--repeat', type=int, default=1) - parser.add_argument('--device', type=str, default='cuda:0') - parser.add_argument('--lr', type=float, default=0.01) - parser.add_argument('--weight_decay', type=float, default=0.0) - parser.add_argument('--epoch', type=int, default=200) - - args = parser.parse_args() - args.cuda = torch.cuda.is_available() - print('cuda: %s' % args.cuda) - - # make sure you use the same data splits as you generated attacks - np.random.seed(args.seed) - if args.cuda: - torch.cuda.manual_seed(args.seed) - - # Here the random seed is to split the train/val/test data, - # we need to set the random seed to be the same as that when you generate the perturbed graph - # data = Dataset(root='/tmp/', name=args.dataset, setting='nettack', seed=15) - # Or we can just use setting='prognn' to get the splits - data = Dataset(root='/tmp/', name=args.dataset, setting='prognn') - adj, features, labels = data.adj, data.features, data.labels - idx_train, idx_val, idx_test = data.idx_train, data.idx_val, data.idx_test - pyg_data = Dpr2Pyg(data).process().to(args.device) - pyg_data.num_classes = len(set(labels)) - - # load pre-attacked graph - perturbed_data = PrePtbDataset(root='/tmp/', - name=args.dataset, - attack_method='meta', - ptb_rate=args.ptb_rate) - perturbed_adj = perturbed_data.adj - - print('=== testing GCN-SVD on perturbed graph (AutoGL) ===') - model_hp = { - "num_layers": 2, - "hidden": [16], - "dropout": 0.5, - "act": "relu" - } - accs = [] - for seed in tqdm(range(args.repeat)): - set_seed(seed) - model = AutoGCNSVD( - num_features=pyg_data.num_node_features, - num_classes=pyg_data.num_classes, - device=args.device, - init=False - ).from_hyper_parameter(model_hp).model - model.to(args.device) - - train(model, pyg_data, args) - acc = test(model, pyg_data, pyg_data.test_mask) - accs.append(acc) - print('{:.4f} ~ {:.4f}'.format(np.mean(accs), np.std(accs))) - - - print('=== testing GCN-SVD on perturbed graph (deeprobust)===') - model = GCNSVD(nfeat=features.shape[1], nclass=labels.max()+1, - nhid=16, device=args.device) - - model = model.to(args.device) - # Test set results: loss= 0.8541 accuracy= 0.7067 - model.fit(features, perturbed_adj, labels, idx_train, idx_val, k=args.k, verbose=True) - model.eval() - output = model.test(idx_test) - print(output) - \ No newline at end of file diff --git a/test/performance/robust/model_gnnguard.py b/test/performance/robust/model_gnnguard.py deleted file mode 100644 index f7c6be8..0000000 --- a/test/performance/robust/model_gnnguard.py +++ /dev/null @@ -1,192 +0,0 @@ -import os -import torch -# import sys -# sys.path.insert(0, '/n/scratch2/xz204/Dr37/lib/python3.7/site-packages') -from deeprobust.graph.targeted_attack import Nettack -from deeprobust.graph.utils import * -from deeprobust.graph.data import Dataset -import argparse -# from deeprobust.graph.defense import * # GCN, GAT, GIN, JK, GCN_attack,accuracy_1 -from deeprobust.graph.defense import * -from deeprobust.graph.data import Dataset, PrePtbDataset, Dpr2Pyg, AmazonPyg -from tqdm import tqdm -import scipy -import numpy as np -from sklearn.preprocessing import normalize -import pickle - -os.environ["AUTOGL_BACKEND"] = "pyg" - -from autogl.module.model.pyg import AutoGNNGuard -from autogl.solver.utils import set_seed - -parser = argparse.ArgumentParser() -parser.add_argument('--seed', type=int, default=14, help='Random seed.') -# cora and citeseer are binary, pubmed has not binary features -parser.add_argument('--dataset', type=str, default='citeseer', choices=['cora', 'cora_ml', 'citeseer', 'polblogs', 'pubmed'], help='dataset') -parser.add_argument('--ptb_rate', type=float, default=0.05, help='pertubation rate') -parser.add_argument('--modelname', type=str, default='GCN', choices=['GCN', 'GAT','GIN', 'JK']) -parser.add_argument('--defensemodel', type=str, default='GCNJaccard', choices=['GCNJaccard', 'RGCN', 'GCNSVD']) -parser.add_argument('--DPlabel', type=int, default=9, help='0-10') - -args = parser.parse_args() -args.cuda = torch.cuda.is_available() -print('cuda: %s' % args.cuda) -args.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") - -def seed_torch(seed=1029): - np.random.seed(seed) - torch.manual_seed(seed) - torch.cuda.manual_seed(seed) - torch.cuda.manual_seed_all(seed) - torch.backends.cudnn.benchmark = False - torch.backends.cudnn.deterministic = True - # torch.use_deterministic_algorithms(True) -seed_torch(args.seed) - -args.dataset = "cora" -args.modelname = "GCN" - -data = Dataset(root='/tmp/', name=args.dataset) -adj, features, labels = data.adj, data.features, data.labels -if scipy.sparse.issparse(features)==False: - features = scipy.sparse.csr_matrix(features) -"""set the number of training/val/testing nodes""" -idx_train, idx_val, idx_test = data.idx_train, data.idx_val, data.idx_test -"""add undirected edges, orgn-arxiv is directed graph, we transfer it to undirected closely following -https://ogb.stanford.edu/docs/leader_nodeprop/#ogbn-arxiv -""" -adj = adj + adj.T -adj[adj>1] = 1 - -pyg_data = Dpr2Pyg(data).process().to(args.device) -pyg_data.num_classes = len(set(labels)) - - -def main(flag): - - # Setup Surrogate model - surrogate = GCN_attack(nfeat=features.shape[1], nclass=labels.max().item()+1, n_edge=adj.nonzero()[0].shape[0], nhid=16, dropout=0, with_relu=False, with_bias=False, device=args.device, ) - surrogate = surrogate.to(args.device) - surrogate.fit(features, adj, labels, idx_train, train_iters=201) # change this train_iters to 201: train_iters=201 - - # Setup Attack Model - target_node = 859 - - model = Nettack(surrogate, nnodes=adj.shape[0], attack_structure=True, attack_features=False, device=args.device) - model = model.to(args.device) - - degrees = adj.sum(0).A1 - # How many perturbations to perform. Default: Degree of the node - n_perturbations = int(degrees[target_node]) - - # # indirect attack/ influencer attack - model.attack(features, adj, labels, target_node, n_perturbations, direct=True) - modified_adj = model.modified_adj - modified_features = model.modified_features - - print('=== testing GNN on original(clean) graph ===') - test(adj, features, target_node, attention=flag) - - print('=== testing GCN on perturbed graph ===') - test(modified_adj, modified_features, target_node,attention=flag) - -def test(adj, features, target_node, attention=False): - '' - """test on GCN """ - """model_name could be 'GCN', 'GAT', 'GIN','JK' """ - # for orgn-arxiv: nhid =256, layers =3, epoch =500 - - gcn = globals()[args.modelname](nfeat=features.shape[1], nhid=256, nclass=labels.max().item() + 1, dropout=0.5, - device=args.device) - gcn = gcn.to(args.device) - gcn.fit(features, adj, labels, idx_train, idx_val=idx_val, - idx_test=idx_test, - attention=attention, verbose=True, train_iters=81) - gcn.eval() - _, output = gcn.test(idx_test=idx_test) - - probs = torch.exp(output[[target_node]])[0] - print('probs: {}'.format(probs.detach().cpu().numpy())) - acc_test = accuracy(output[idx_test], labels[idx_test]) - - print("Test set results:", - "accuracy= {:.4f}".format(acc_test.item())) - return acc_test.item() - -def main_autogl(flag): - # Setup Surrogate model - surrogate = AutoGNNGuard( - num_features=pyg_data.num_node_features, - num_classes=pyg_data.num_classes, - device=args.device, - init=False - ).from_hyper_parameter(model_hp).model - surrogate = surrogate.to(args.device) - surrogate.fit(features, adj, labels, idx_train, train_iters=201) # change this train_iters to 201: train_iters=201 - - # Setup Attack Model - target_node = 859 - - model = Nettack(surrogate, nnodes=adj.shape[0], attack_structure=True, attack_features=False, device=args.device) - model = model.to(args.device) - - degrees = adj.sum(0).A1 - # How many perturbations to perform. Default: Degree of the node - n_perturbations = int(degrees[target_node]) - - # # indirect attack/ influencer attack - model.attack(features, adj, labels, target_node, n_perturbations, direct=True) - modified_adj = model.modified_adj - modified_features = model.modified_features - - print('=== testing GNN on original(clean) graph (AutoGL) ===') - test_autogl(adj, features, target_node, attention=flag) - - print('=== testing GCN on perturbed graph (AutoGL) ===') - test_autogl(modified_adj, modified_features, target_node,attention=flag) - - -def test_autogl(adj, features, target_node, attention=False): - '' - """test on GCN """ - """model_name could be 'GCN', 'GAT', 'GIN','JK' """ - # for orgn-arxiv: nhid =256, layers =3, epoch =500 - - gcn = AutoGNNGuard( - num_features=pyg_data.num_node_features, - num_classes=pyg_data.num_classes, - device=args.device, - init=False - ).from_hyper_parameter(model_hp).model - gcn = gcn.to(args.device) - gcn.fit(features, adj, labels, idx_train, idx_val=idx_val, - idx_test=idx_test, - attention=attention, verbose=True, train_iters=81) - gcn.eval() - _, output = gcn.test(idx_test=idx_test) - - probs = torch.exp(output[[target_node]])[0] - print('probs: {}'.format(probs.detach().cpu().numpy())) - acc_test = accuracy(output[idx_test], labels[idx_test]) - - print("Test set results:", - "accuracy= {:.4f}".format(acc_test.item())) - return acc_test.item() - -model_hp = { - "num_layers": 2, - "hidden": [16], - "dropout": 0.0, - "act": "relu" - } - -if __name__ == '__main__': - # Check the performance of GCN under directed attack without defense - main(flag=False) - # Use GNNGuard for defense - main(flag=True) - # Check the performance of GCN under directed attack without defense（AutoGL） - main_autogl(flag=False) - # Use GNNGuard for defense（AutoGL） - main_autogl(flag=True) diff --git a/test/performance/robust/model_gnnguard_meta.py b/test/performance/robust/model_gnnguard_meta.py new file mode 100644 index 0000000..ea199df --- /dev/null +++ b/test/performance/robust/model_gnnguard_meta.py @@ -0,0 +1,204 @@ +import os +import torch +# import sys +# sys.path.insert(0, '/n/scratch2/xz204/Dr37/lib/python3.7/site-packages') +from deeprobust.graph.targeted_attack import Nettack +from deeprobust.graph.utils import * +from deeprobust.graph.data import Dataset +from deeprobust.graph.global_attack import Metattack +import argparse +# from deeprobust.graph.defense import * # GCN, GAT, GIN, JK, GCN_attack,accuracy_1 +from deeprobust.graph.defense import * +from deeprobust.graph.data import Dataset, PrePtbDataset, Dpr2Pyg, AmazonPyg +from scipy.sparse import csr_matrix +from tqdm import tqdm +import scipy +import scipy.sparse as sp +import numpy as np +from sklearn.preprocessing import normalize +import pickle + +os.environ["AUTOGL_BACKEND"] = "pyg" + +from autogl.module.model.pyg import AutoGNNGuard, AutoGNNGuard_attack +from autogl.solver.utils import set_seed + +def seed_torch(seed=1029): + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + torch.backends.cudnn.benchmark = False + torch.backends.cudnn.deterministic = True + # torch.use_deterministic_algorithms(True) +seed_torch(2048) + +def main(dataset, adj, features, device): + # from deeprobust.graph.data import PrePtbDataset + # perturbed_data = PrePtbDataset(root='/tmp/', name=dataset, attack_method='meta', ptb_rate=0.2) + # modified_adj = perturbed_data.adj + + # Setup Surrogate model + surrogate = GCN_attack(nfeat=features.shape[1], nclass=labels.max().item()+1, n_edge=adj.nonzero()[0].shape[0], nhid=16, dropout=0, with_relu=False, with_bias=False, device=args.device, ) + + surrogate = surrogate.to(args.device) + surrogate.fit(features, adj, labels, idx_train, train_iters=201) # change this train_iters to 201: train_iters=201 + + # Setup Attack Model + # model = Metattack(model=surrogate, nnodes=adj.shape[0], feature_shape=features.shape, attack_structure=True, attack_features=False, device=args.device, lambda_=0.5) # lambda_ is used to weight the two objectives in Eq. (10) in the paper. + # model = model.to(args.device) + + # """save the mettacked adj""" + # model.attack(features, adj, labels, idx_train, idx_unlabeled, perturbations, ll_constraint=False) + # modified_adj = sp.csr_matrix(model.modified_adj.cpu()) + + # from deeprobust.graph.data import PrePtbDataset + perturbed_data = PrePtbDataset(root='/tmp/', name=dataset, attack_method='meta', ptb_rate=0.2) + modified_adj = perturbed_data.adj + + # Check the performance of GCN under directed attack without defense + flag = False + # print('=== testing GNN on original(clean) graph ===') + # print("acc_test:",test(adj, features, device, attention=flag)) + # print('=== testing GCN on perturbed graph ===') + # print("acc_test:",test(modified_adj, features, device, attention=flag)) + + # Use GNNGuard for defense + flag = True + print('=== testing GNN on original(clean) graph + GNNGuard ===') + print("acc_test:",test(adj, features, device, attention=flag)) + print('=== testing GCN on perturbed graph + GNNGuard ===') + print("acc_test:",test(modified_adj, features, device, attention=flag)) + +def test(adj, features, device, attention): + accs = [] + for seed in tqdm(range(5)): + + classifier = GCN(nfeat=features.shape[1], nhid=16, nclass=labels.max().item() + 1, dropout=0.5, device=device) + + classifier = classifier.to(device) + + print(1) + classifier.fit(features, adj, labels, idx_train, train_iters=201, + idx_val=idx_val, + idx_test=idx_test, + verbose=True, attention=attention) # idx_val=idx_val, idx_test=idx_test , model_name=model_name + classifier.eval() + + # classifier.fit(features, adj, labels, idx_train, idx_val) # train with validation model picking + acc_test, output = classifier.test(idx_test) + accs.append(acc_test.item()) + mean = np.mean(accs) + std = np.std(accs) + return {"mean": mean, "std": std} + +def main_autogl(dataset, model_hp, adj, features, device): + + # Setup Surrogate model + surrogate = AutoGNNGuard_attack( + num_features=pyg_data.num_node_features, + num_classes=pyg_data.num_classes, + device=args.device, + init=False + ).from_hyper_parameter(model_hp).model + surrogate = surrogate.to(args.device) + surrogate.fit(features, adj, labels, idx_train, train_iters=201) # change this train_iters to 201: train_iters=201 + + # Setup Attack Model + # model = Metattack(model=surrogate, nnodes=adj.shape[0], feature_shape=features.shape, attack_structure=True, attack_features=False, device=args.device, lambda_=0.5) # lambda_ is used to weight the two objectives in Eq. (10) in the paper. + # model = model.to(args.device) + + # """save the mettacked adj""" + # model.attack(features, adj, labels, idx_train, idx_unlabeled, perturbations, ll_constraint=False) + # modified_adj = sp.csr_matrix(model.modified_adj.cpu()) + + # from deeprobust.graph.data import PrePtbDataset + perturbed_data = PrePtbDataset(root='/tmp/', name=dataset, attack_method='meta', ptb_rate=0.2) + modified_adj = perturbed_data.adj + + # Check the performance of GCN under directed attack without defense（AutoGL） + # flag = False + # print('=== testing GNN on original(clean) graph (AutoGL) ===') + # print("acc_test:",test_autogl(adj, features, device, attention=flag)) + # print('=== testing GCN on perturbed graph (AutoGL) ===') + # print("acc_test:",test_autogl(modified_adj, features, device, attention=flag)) + + # Use GNNGuard for defense（AutoGL） + flag = True + print('=== testing GNN on original(clean) graph (AutoGL) + GNNGuard ===') + print("acc_test:",test_autogl(adj, features, device, attention=flag)) + print('=== testing GCN on perturbed graph (AutoGL) + GNNGuard ===') + print("acc_test:",test_autogl(modified_adj, features, device, attention=flag)) + +def test_autogl(adj, features, device, attention): + '' + """test on GCN """ + """model_name could be 'GCN', 'GAT', 'GIN','JK' """ + accs = [] + for seed in tqdm(range(5)): + gcn = AutoGNNGuard( + num_features=pyg_data.num_node_features, + num_classes=pyg_data.num_classes, + device=args.device, + init=False + ).from_hyper_parameter(model_hp).model + gcn = gcn.to(device) + gcn.fit(features, adj, labels, idx_train, idx_val=idx_val, + idx_test=idx_test, + attention=attention, verbose=True, train_iters=81) + gcn.eval() + acc_test, output = gcn.test(idx_test=idx_test) + accs.append(acc_test.item()) + mean = np.mean(accs) + std = np.std(accs) + return {"mean": mean, "std": std} + +if __name__ == '__main__': + + model_hp = { + "num_layers": 2, + "hidden": [16], + "dropout": 0.5, + "act": "relu" + } + + parser = argparse.ArgumentParser() + parser.add_argument('--seed', type=int, default=14, help='Random seed.') + # cora and citeseer are binary, pubmed has not binary features + parser.add_argument('--dataset', type=str, default='citeseer', choices=['cora', 'cora_ml', 'citeseer', 'polblogs', 'pubmed'], help='dataset') + parser.add_argument('--ptb_rate', type=float, default=0.2, help='pertubation rate') + parser.add_argument('--modelname', type=str, default='GCN', choices=['GCN', 'GAT','GIN', 'JK']) + parser.add_argument('--defensemodel', type=str, default='GCNJaccard', choices=['GCNJaccard', 'RGCN', 'GCNSVD']) + parser.add_argument('--DPlabel', type=int, default=9, help='0-10') + + args = parser.parse_args() + args.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + print('cuda: %s' % args.device) + + args.dataset = "pubmed" + args.modelname = "GCN" + + data = Dataset(root='/tmp/', name=args.dataset) + pyg_data = Dpr2Pyg(data).process().to(args.device) + pyg_data.num_classes = len(set(data.labels)) + + adj, features, labels = data.adj, data.features, data.labels + idx_train, idx_val, idx_test = data.idx_train, data.idx_val, data.idx_test + idx_unlabeled = np.union1d(idx_val, idx_test) + if scipy.sparse.issparse(features)==False: + features = scipy.sparse.csr_matrix(features) + + perturbations = int(args.ptb_rate * (adj.sum()//2)) ### + adj, features, labels = preprocess(adj, features, labels, preprocess_adj=False) + + # to CSR sparse + adj, features = csr_matrix(adj), csr_matrix(features) + + """add undirected edges, orgn-arxiv is directed graph, we transfer it to undirected closely following + https://ogb.stanford.edu/docs/leader_nodeprop/#ogbn-arxiv + """ + adj = adj + adj.T + adj[adj>1] = 1 + + # main(args.dataset, adj, features, device=args.device) + main_autogl(args.dataset, model_hp, adj, features, device=args.device) \ No newline at end of file