From c26be9e973c389cfc49390a5b59323c8e03d7e08 Mon Sep 17 00:00:00 2001 From: Frozenmad Date: Fri, 26 Nov 2021 06:32:27 +0000 Subject: [PATCH 1/7] initial commit for hetero solver --- .../classifier/hetero/node_classifier.py | 839 ++++++++++++++++++ autogl/solver/classifier/node_classifier.py | 4 +- 2 files changed, 842 insertions(+), 1 deletion(-) create mode 100644 autogl/solver/classifier/hetero/node_classifier.py diff --git a/autogl/solver/classifier/hetero/node_classifier.py b/autogl/solver/classifier/hetero/node_classifier.py new file mode 100644 index 0000000..834d0d5 --- /dev/null +++ b/autogl/solver/classifier/hetero/node_classifier.py @@ -0,0 +1,839 @@ +""" +Auto Classfier for Heterogeneous Node Classification +""" +import time +import json + +from copy import deepcopy + +import torch +import numpy as np +import yaml + +from typing import Iterable + +from ..base import BaseClassifier +from ...base import _parse_hp_space, _initialize_single_model +from ....module.feature import FEATURE_DICT +from ....module.model import MODEL_DICT, BaseModel +from ....module.train import TRAINER_DICT, BaseNodeClassificationTrainer +from ....module.train import get_feval +from ....module.nas.space import NAS_SPACE_DICT +from ....module.nas.algorithm import NAS_ALGO_DICT +from ....module.nas.estimator import NAS_ESTIMATOR_DICT, BaseEstimator +from ...utils import LeaderBoard, get_graph_from_dataset, get_graph_labels, get_graph_masks, get_graph_node_features, get_graph_node_number, set_seed, convert_dataset +from ....datasets import utils +from ....utils import get_logger + +LOGGER = get_logger("HeteroNodeClassifier") + +class AutoHeteroNodeClassifier(BaseClassifier): + """ + Auto Multi-class HeteroGraph Node Classifier. + + Used to automatically solve the heterogeneous node classification problems. + + Parameters + ---------- + feature_module: autogl.module.feature.BaseFeatureEngineer or str or None + The (name of) auto feature engineer used to process the given dataset. Default ``deepgl``. + Disable feature engineer by setting it to ``None``. + + graph_models: list of autogl.module.model.BaseModel or list of str + The (name of) models to be optimized as backbone. Default ``['gat', 'gcn']``. + + hpo_module: autogl.module.hpo.BaseHPOptimizer or str or None + The (name of) hpo module used to search for best hyper parameters. Default ``anneal``. + Disable hpo by setting it to ``None``. + + ensemble_module: autogl.module.ensemble.BaseEnsembler or str or None + The (name of) ensemble module used to ensemble the multi-models found. Default ``voting``. + Disable ensemble by setting it to ``None``. + + max_evals: int (Optional) + If given, will set the number eval times the hpo module will use. + Only be effective when hpo_module is ``str``. Default ``None``. + + trainer_hp_space: list of dict (Optional) + trainer hp space or list of trainer hp spaces configuration. + If a single trainer hp is given, will specify the hp space of trainer for every model. + If a list of trainer hp is given, will specify every model with corrsponding + trainer hp space. + Default ``None``. + + model_hp_spaces: list of list of dict (Optional) + model hp space configuration. + If given, will specify every hp space of every passed model. Default ``None``. + + size: int (Optional) + The max models ensemble module will use. Default ``None``. + + device: torch.device or str + The device where model will be running on. If set to ``auto``, will use gpu when available. + You can also specify the device by directly giving ``gpu`` or ``cuda:0``, etc. + Default ``auto``. + """ + + def __init__( + self, + feature_module=None, + graph_models=("han", "hgt"), + hpo_module="anneal", + ensemble_module="voting", + max_evals=50, + default_trainer="NodeClassificationHet", + trainer_hp_space=None, + model_hp_spaces=None, + size=4, + device="auto", + ): + + super().__init__( + feature_module=feature_module, + graph_models=graph_models, + # currently we do not support nas for heterogeneous node classifier + nas_algorithms=None, + nas_spaces=None, + nas_estimators=None, + hpo_module=hpo_module, + ensemble_module=ensemble_module, + max_evals=max_evals, + default_trainer=default_trainer, + trainer_hp_space=trainer_hp_space, + model_hp_spaces=model_hp_spaces, + size=size, + device=device, + ) + + # data to be kept when fit + self.dataset = None + + def _init_graph_module( + self, graph_models, num_classes, num_features, feval, device, loss + ) -> "AutoHeteroNodeClassifier": + # TODO: fix model initialization because hetero model should receive additional arguments + # load graph network module + self.graph_model_list = [] + if isinstance(graph_models, Iterable): + for model in graph_models: + if isinstance(model, str): + if model in MODEL_DICT: + # TODO: fix model initialization because hetero model should receive additional arguments + self.graph_model_list.append( + MODEL_DICT[model]( + num_classes=num_classes, + num_features=num_features, + device=device, + init=False, + ) + ) + else: + raise KeyError("cannot find model %s" % (model)) + elif isinstance(model, type) and issubclass(model, BaseModel): + # TODO: fix model initialization because hetero model should receive additional arguments + self.graph_model_list.append( + model( + num_classes=num_classes, + num_features=num_features, + device=device, + init=False, + ) + ) + elif isinstance(model, BaseModel): + # setup the hp of num_classes and num_features + # TODO: setup model + model.set_num_classes(num_classes) + model.set_num_features(num_features) + self.graph_model_list.append(model.to(device)) + elif isinstance(model, BaseNodeClassificationTrainer): + # receive a trainer list, put trainer to list + assert ( + model.get_model() is not None + ), "Passed trainer should contain a model" + model.model.set_num_classes(num_classes) + model.model.set_num_features(num_features) + model.update_parameters( + num_classes=num_classes, + num_features=num_features, + loss=loss, + feval=feval, + device=device, + ) + self.graph_model_list.append(model) + else: + raise KeyError("cannot find graph network %s." % (model)) + else: + raise ValueError( + "need graph network to be (list of) str or a BaseModel class/instance, get", + graph_models, + "instead.", + ) + + # wrap all model_cls with specified trainer + for i, model in enumerate(self.graph_model_list): + # set model hp space + if self._model_hp_spaces is not None: + if self._model_hp_spaces[i] is not None: + if isinstance(model, BaseNodeClassificationTrainer): + model.model.hyper_parameter_space = self._model_hp_spaces[i] + else: + model.hyper_parameter_space = self._model_hp_spaces[i] + # initialize trainer if needed + if isinstance(model, BaseModel): + name = ( + self._default_trainer + if isinstance(self._default_trainer, str) + else self._default_trainer[i] + ) + model = TRAINER_DICT[name]( + model=model, + num_features=num_features, + num_classes=num_classes, + loss=loss, + feval=feval, + device=device, + init=False, + ) + # set trainer hp space + if self._trainer_hp_space is not None: + if isinstance(self._trainer_hp_space[0], list): + current_hp_for_trainer = self._trainer_hp_space[i] + else: + current_hp_for_trainer = self._trainer_hp_space + model.hyper_parameter_space = current_hp_for_trainer + self.graph_model_list[i] = model + + return self + + # pylint: disable=arguments-differ + def fit( + self, + dataset, + time_limit=-1, + inplace=False, + train_split=None, + val_split=None, + balanced=True, + evaluation_method="infer", + seed=None, + ) -> "AutoHeteroNodeClassifier": + """ + Fit current solver on given dataset. + + Parameters + ---------- + dataset: autogl.data.Dataset + The dataset needed to fit on. This dataset must have only one graph. + + time_limit: int + The time limit of the whole fit process (in seconds). If set below 0, + will ignore time limit. Default ``-1``. + + inplace: bool + Whether we process the given dataset in inplace manner. Default ``False``. + Set it to True if you want to save memory by modifying the given dataset directly. + + train_split: float or int (Optional) + The train ratio (in ``float``) or number (in ``int``) of dataset. If you want to + use default train/val/test split in dataset, please set this to ``None``. + Default ``None``. + + val_split: float or int (Optional) + The validation ratio (in ``float``) or number (in ``int``) of dataset. If you want + to use default train/val/test split in dataset, please set this to ``None``. + Default ``None``. + + balanced: bool + Wether to create the train/valid/test split in a balanced way. + If set to ``True``, the train/valid will have the same number of different classes. + Default ``True``. + + evaluation_method: (list of) str or autogl.module.train.evaluation + A (list of) evaluation method for current solver. If ``infer``, will automatically + determine. Default ``infer``. + + seed: int (Optional) + The random seed. If set to ``None``, will run everything at random. + Default ``None``. + + Returns + ------- + self: autogl.solver.AutoNodeClassifier + A reference of current solver. + """ + set_seed(seed) + + if time_limit < 0: + time_limit = 3600 * 24 + time_begin = time.time() + + graph_data = get_graph_from_dataset(dataset, 0) + all_labels = get_graph_labels(graph_data) + num_classes = all_labels.max().item() + 1 + + # initialize leaderboard + if evaluation_method == "infer": + if hasattr(dataset, "metric"): + evaluation_method = [dataset.metric] + else: + num_of_label = num_classes + if num_of_label == 2: + evaluation_method = ["auc"] + else: + evaluation_method = ["acc"] + assert isinstance(evaluation_method, list) + evaluator_list = get_feval(evaluation_method) + + self.leaderboard = LeaderBoard( + [e.get_eval_name() for e in evaluator_list], + {e.get_eval_name(): e.is_higher_better() for e in evaluator_list}, + ) + + + # set up the dataset + if train_split is not None and val_split is not None: + size = get_graph_node_number(graph_data) + if balanced: + train_split = ( + train_split if train_split > 1 else int(train_split * size) + ) + val_split = val_split if val_split > 1 else int(val_split * size) + utils.random_splits_mask_class( + dataset, + num_train_per_class=train_split // num_classes, + num_val_per_class=val_split // num_classes, + seed=seed, + ) + else: + train_split = train_split if train_split < 1 else train_split / size + val_split = val_split if val_split < 1 else val_split / size + utils.random_splits_mask( + dataset, train_ratio=train_split, val_ratio=val_split + ) + else: + assert get_graph_masks(graph_data, 'train') is not None and get_graph_masks(graph_data, 'val') is not None, ( + "The dataset has no default train/val split! Please manually pass " + "train and val ratio." + ) + LOGGER.info("Use the default train/val/test ratio in given dataset") + + # feature engineering + if self.feature_module is not None: + dataset = self.feature_module.fit_transform(dataset, inplace=inplace) + + self.dataset = dataset + + # check whether the dataset has features. + # currently we only support graph classification with features. + + feat = get_graph_node_features(graph_data) + assert feat is not None, ( + "Does not support fit on non node-feature dataset!" + " Please add node features to dataset or specify feature engineers that generate" + " node features." + ) + + num_features = feat.size(-1) + + # initialize graph networks + self._init_graph_module( + self.gml, + num_features=num_features, + num_classes=num_classes, + feval=evaluator_list, + device=self.runtime_device, + loss="nll_loss" if not hasattr(dataset, "loss") else self.dataset.loss, + ) + + if self.nas_algorithms is not None: + # perform neural architecture search + self._init_nas_module( + num_features=num_features, + num_classes=num_classes, + feval=evaluator_list, + device=self.runtime_device, + loss="nll_loss" if not hasattr(dataset, "loss") else dataset.loss, + ) + + assert not isinstance(self._default_trainer, list) or len( + self.nas_algorithms + ) == len(self._default_trainer) - len( + self.graph_model_list + ), "length of default trainer should match total graph models and nas models passed" + + # perform nas and add them to model list + idx_trainer = len(self.graph_model_list) + for algo, space, estimator in zip( + self.nas_algorithms, self.nas_spaces, self.nas_estimators + ): + model = algo.search(space, convert_dataset(self.dataset), estimator) + # insert model into default trainer + if isinstance(self._default_trainer, list): + train_name = self._default_trainer[idx_trainer] + idx_trainer += 1 + else: + train_name = self._default_trainer + if isinstance(train_name, str): + trainer = TRAINER_DICT[train_name]( + model=model, + num_features=num_features, + num_classes=num_classes, + loss="nll_loss" + if not hasattr(dataset, "loss") + else dataset.loss, + feval=evaluator_list, + device=self.runtime_device, + init=False, + ) + else: + trainer = train_name + trainer.model = model + trainer.update_parameters( + num_features=num_features, + num_classes=num_classes, + loss="nll_loss" + if not hasattr(dataset, "loss") + else dataset.loss, + feval=evaluator_list, + device=self.runtime_device, + ) + self.graph_model_list.append(trainer) + + # train the models and tune hpo + result_valid = [] + names = [] + for idx, model in enumerate(self.graph_model_list): + time_for_each_model = (time_limit - time.time() + time_begin) / ( + len(self.graph_model_list) - idx + ) + if self.hpo_module is None: + model.initialize() + model.train(convert_dataset(self.dataset), True) + optimized = model + else: + optimized, _ = self.hpo_module.optimize( + trainer=model, dataset=convert_dataset(self.dataset), time_limit=time_for_each_model + ) + # to save memory, all the trainer derived will be mapped to cpu + optimized.to(torch.device("cpu")) + name = str(optimized) + "_idx%d" % (idx) + names.append(name) + performance_on_valid, _ = optimized.get_valid_score(return_major=False) + result_valid.append(optimized.get_valid_predict_proba().cpu().numpy()) + self.leaderboard.insert_model_performance( + name, + dict( + zip( + [e.get_eval_name() for e in evaluator_list], + performance_on_valid, + ) + ), + ) + self.trained_models[name] = optimized + + # fit the ensemble model + if self.ensemble_module is not None: + performance = self.ensemble_module.fit( + result_valid, + all_labels[get_graph_masks(graph_data, 'val')].cpu().numpy(), + names, + evaluator_list, + n_classes=num_classes, + ) + self.leaderboard.insert_model_performance( + "ensemble", + dict(zip([e.get_eval_name() for e in evaluator_list], performance)), + ) + + return self + + def fit_predict( + self, + dataset, + time_limit=-1, + inplace=False, + train_split=None, + val_split=None, + balanced=True, + evaluation_method="infer", + use_ensemble=True, + use_best=True, + name=None, + ) -> np.ndarray: + """ + Fit current solver on given dataset and return the predicted value. + + Parameters + ---------- + dataset: torch_geometric.data.dataset.Dataset + The dataset needed to fit on. This dataset must have only one graph. + + time_limit: int + The time limit of the whole fit process (in seconds). + If set below 0, will ignore time limit. Default ``-1``. + + inplace: bool + Whether we process the given dataset in inplace manner. Default ``False``. + Set it to True if you want to save memory by modifying the given dataset directly. + + train_split: float or int (Optional) + The train ratio (in ``float``) or number (in ``int``) of dataset. If you want to + use default train/val/test split in dataset, please set this to ``None``. + Default ``None``. + + val_split: float or int (Optional) + The validation ratio (in ``float``) or number (in ``int``) of dataset. If you want + to use default train/val/test split in dataset, please set this to ``None``. + Default ``None``. + + balanced: bool + Wether to create the train/valid/test split in a balanced way. + If set to ``True``, the train/valid will have the same number of different classes. + Default ``False``. + + evaluation_method: (list of) str or autogl.module.train.evaluation + A (list of) evaluation method for current solver. If ``infer``, will automatically + determine. Default ``infer``. + + use_ensemble: bool + Whether to use ensemble to do the predict. Default ``True``. + + use_best: bool + Whether to use the best single model to do the predict. Will only be effective when + ``use_ensemble`` is ``False``. + Default ``True``. + + name: str or None + The name of model used to predict. Will only be effective when ``use_ensemble`` and + ``use_best`` both are ``False``. + Default ``None``. + + Returns + ------- + result: np.ndarray + An array of shape ``(N,)``, where ``N`` is the number of test nodes. The prediction + on given dataset. + """ + self.fit( + dataset=dataset, + time_limit=time_limit, + inplace=inplace, + train_split=train_split, + val_split=val_split, + balanced=balanced, + evaluation_method=evaluation_method, + ) + return self.predict( + dataset=dataset, + inplaced=inplace, + inplace=inplace, + use_ensemble=use_ensemble, + use_best=use_best, + name=name, + ) + + def predict_proba( + self, + dataset=None, + inplaced=False, + inplace=False, + use_ensemble=True, + use_best=True, + name=None, + mask="test", + ) -> np.ndarray: + """ + Predict the node probability. + + Parameters + ---------- + dataset: torch_geometric.data.dataset.Dataset or None + The dataset needed to predict. If ``None``, will use the processed dataset passed + to ``fit()`` instead. Default ``None``. + + inplaced: bool + Whether the given dataset is processed. Only be effective when ``dataset`` + is not ``None``. If you pass the dataset to ``fit()`` with ``inplace=True``, and + you pass the dataset again to this method, you should set this argument to ``True``. + Otherwise ``False``. Default ``False``. + + inplace: bool + Whether we process the given dataset in inplace manner. Default ``False``. Set it to + True if you want to save memory by modifying the given dataset directly. + + use_ensemble: bool + Whether to use ensemble to do the predict. Default ``True``. + + use_best: bool + Whether to use the best single model to do the predict. Will only be effective when + ``use_ensemble`` is ``False``. Default ``True``. + + name: str or None + The name of model used to predict. Will only be effective when ``use_ensemble`` and + ``use_best`` both are ``False``. Default ``None``. + + mask: str + The data split to give prediction on. Default ``test``. + + Returns + ------- + result: np.ndarray + An array of shape ``(N,C,)``, where ``N`` is the number of test nodes and ``C`` is + the number of classes. The prediction on given dataset. + """ + if dataset is None: + dataset = self.dataset + assert dataset is not None, ( + "Please execute fit() first before" " predicting on remembered dataset" + ) + elif not inplaced and self.feature_module is not None: + dataset = self.feature_module.transform(dataset, inplace=inplace) + + if use_ensemble: + LOGGER.info("Ensemble argument on, will try using ensemble model.") + + if not use_ensemble and use_best: + LOGGER.info( + "Ensemble argument off and best argument on, will try using best model." + ) + + if (use_ensemble and self.ensemble_module is not None) or ( + not use_best and name == "ensemble" + ): + # we need to get all the prediction of every model trained + predict_result = [] + names = [] + for model_name in self.trained_models: + predict_result.append( + self._predict_proba_by_name(dataset, model_name, mask) + ) + names.append(model_name) + return self.ensemble_module.ensemble(predict_result, names) + + if use_ensemble and self.ensemble_module is None: + LOGGER.warning( + "Cannot use ensemble because no ensebmle module is given." + " Will use best model instead." + ) + + if use_best or (use_ensemble and self.ensemble_module is None): + # just return the best model we have found + name = self.leaderboard.get_best_model() + return self._predict_proba_by_name(dataset, name, mask) + + if name is not None: + # return model performance by name + return self._predict_proba_by_name(dataset, name, mask) + + LOGGER.error( + "No model name is given while ensemble and best arguments are off." + ) + raise ValueError( + "You need to specify a model name if you do not want use ensemble and best model." + ) + + def _predict_proba_by_name(self, dataset, name, mask="test"): + self.trained_models[name].to(self.runtime_device) + predicted = ( + self.trained_models[name].predict_proba(convert_dataset(dataset), mask=mask).cpu().numpy() + ) + self.trained_models[name].to(torch.device("cpu")) + return predicted + + def predict( + self, + dataset=None, + inplaced=False, + inplace=False, + use_ensemble=True, + use_best=True, + name=None, + mask="test", + ) -> np.ndarray: + """ + Predict the node class number. + + Parameters + ---------- + dataset: torch_geometric.data.dataset.Dataset or None + The dataset needed to predict. If ``None``, will use the processed dataset passed + to ``fit()`` instead. Default ``None``. + + inplaced: bool + Whether the given dataset is processed. Only be effective when ``dataset`` + is not ``None``. If you pass the dataset to ``fit()`` with ``inplace=True``, + and you pass the dataset again to this method, you should set this argument + to ``True``. Otherwise ``False``. Default ``False``. + + inplace: bool + Whether we process the given dataset in inplace manner. Default ``False``. + Set it to True if you want to save memory by modifying the given dataset directly. + + use_ensemble: bool + Whether to use ensemble to do the predict. Default ``True``. + + use_best: bool + Whether to use the best single model to do the predict. Will only be effective + when ``use_ensemble`` is ``False``. Default ``True``. + + name: str or None + The name of model used to predict. Will only be effective when ``use_ensemble`` + and ``use_best`` both are ``False``. Default ``None``. + + mask: str + The data split to give prediction on. Default ``test``. + + Returns + ------- + result: np.ndarray + An array of shape ``(N,)``, where ``N`` is the number of test nodes. + The prediction on given dataset. + """ + proba = self.predict_proba( + dataset, inplaced, inplace, use_ensemble, use_best, name, mask + ) + return np.argmax(proba, axis=1) + + @classmethod + def from_config(cls, path_or_dict, filetype="auto") -> "AutoHeteroNodeClassifier": + """ + Load solver from config file. + + You can use this function to directly load a solver from predefined config dict + or config file path. Currently, only support file type of ``json`` or ``yaml``, + if you pass a path. + + Parameters + ---------- + path_or_dict: str or dict + The path to the config file or the config dictionary object + + filetype: str + The filetype the given file if the path is specified. Currently only support + ``json`` or ``yaml``. You can set to ``auto`` to automatically detect the file + type (from file name). Default ``auto``. + + Returns + ------- + solver: autogl.solver.AutoGraphClassifier + The solver that is created from given file or dictionary. + """ + assert filetype in ["auto", "yaml", "json"], ( + "currently only support yaml file or json file type, but get type " + + filetype + ) + if isinstance(path_or_dict, str): + if filetype == "auto": + if path_or_dict.endswith(".yaml") or path_or_dict.endswith(".yml"): + filetype = "yaml" + elif path_or_dict.endswith(".json"): + filetype = "json" + else: + LOGGER.error( + "cannot parse the type of the given file name, " + "please manually set the file type" + ) + raise ValueError( + "cannot parse the type of the given file name, " + "please manually set the file type" + ) + if filetype == "yaml": + path_or_dict = yaml.load( + open(path_or_dict, "r").read(), Loader=yaml.FullLoader + ) + else: + path_or_dict = json.load(open(path_or_dict, "r")) + + path_or_dict = deepcopy(path_or_dict) + solver = cls(None, [], None, None) + fe_list = path_or_dict.pop("feature", None) + if fe_list is not None: + fe_list_ele = [] + for feature_engineer in fe_list: + name = feature_engineer.pop("name") + if name is not None: + fe_list_ele.append(FEATURE_DICT[name](**feature_engineer)) + if fe_list_ele != []: + solver.set_feature_module(fe_list_ele) + + models = path_or_dict.pop("models", [{"name": "gcn"}, {"name": "gat"}]) + model_hp_space = [ + _parse_hp_space(model.pop("hp_space", None)) for model in models + ] + model_list = [ + _initialize_single_model(model.pop("name"), model) for model in models + ] + + trainer = path_or_dict.pop("trainer", None) + default_trainer = "NodeClassificationFull" + trainer_space = None + if isinstance(trainer, dict): + # global default + default_trainer = trainer.pop("name", "NodeClassificationFull") + trainer_space = _parse_hp_space(trainer.pop("hp_space", None)) + default_kwargs = {"num_features": None, "num_classes": None} + default_kwargs.update(trainer) + default_kwargs["init"] = False + for i in range(len(model_list)): + model = model_list[i] + trainer_wrap = TRAINER_DICT[default_trainer]( + model=model, **default_kwargs + ) + model_list[i] = trainer_wrap + elif isinstance(trainer, list): + # sequential trainer definition + assert len(trainer) == len( + model_list + ), "The number of trainer and model does not match" + trainer_space = [] + for i in range(len(model_list)): + train, model = trainer[i], model_list[i] + default_trainer = train.pop("name", "NodeClassificationFull") + trainer_space.append(_parse_hp_space(train.pop("hp_space", None))) + default_kwargs = {"num_features": None, "num_classes": None} + default_kwargs.update(train) + default_kwargs["init"] = False + trainer_wrap = TRAINER_DICT[default_trainer]( + model=model, **default_kwargs + ) + model_list[i] = trainer_wrap + + solver.set_graph_models( + model_list, default_trainer, trainer_space, model_hp_space + ) + + hpo_dict = path_or_dict.pop("hpo", {"name": "anneal"}) + if hpo_dict is not None: + name = hpo_dict.pop("name") + solver.set_hpo_module(name, **hpo_dict) + + ensemble_dict = path_or_dict.pop("ensemble", {"name": "voting"}) + if ensemble_dict is not None: + name = ensemble_dict.pop("name") + solver.set_ensemble_module(name, **ensemble_dict) + + nas_dict = path_or_dict.pop("nas", None) + if nas_dict is not None: + keys: set = set(nas_dict.keys()) + needed = {"space", "algorithm", "estimator"} + if keys != needed: + LOGGER.error("Key mismatch, we need %s, you give %s", needed, keys) + raise KeyError("Key mismatch, we need %s, you give %s" % (needed, keys)) + + spaces, algorithms, estimators = [], [], [] + + for container, indexer, k in zip( + [spaces, algorithms, estimators], + [NAS_SPACE_DICT, NAS_ALGO_DICT, NAS_ESTIMATOR_DICT], + ["space", "algorithm", "estimator"], + ): + configs = nas_dict[k] + if isinstance(configs, list): + for item in configs: + container.append(indexer[item.pop("name")](**item)) + else: + container.append(indexer[configs.pop("name")](**configs)) + + solver.set_nas_module(algorithms, spaces, estimators) + + return solver diff --git a/autogl/solver/classifier/node_classifier.py b/autogl/solver/classifier/node_classifier.py index f9dd636..3c053c2 100644 --- a/autogl/solver/classifier/node_classifier.py +++ b/autogl/solver/classifier/node_classifier.py @@ -10,6 +10,8 @@ import torch import numpy as np import yaml +from typing import Iterable + from .base import BaseClassifier from ..base import _parse_hp_space, _initialize_single_model from ...module.feature import FEATURE_DICT @@ -122,7 +124,7 @@ class AutoNodeClassifier(BaseClassifier): ) -> "AutoNodeClassifier": # load graph network module self.graph_model_list = [] - if isinstance(graph_models, (list, tuple)): + if isinstance(graph_models, Iterable): for model in graph_models: if isinstance(model, str): if model in MODEL_DICT: From a4160da343b13d862934339904ef8ad217fc7ced Mon Sep 17 00:00:00 2001 From: Frozenmad Date: Fri, 10 Dec 2021 04:56:30 +0000 Subject: [PATCH 2/7] add dataset to initialize procedure --- .../classifier/hetero/node_classifier.py | 84 +++++-------------- 1 file changed, 19 insertions(+), 65 deletions(-) diff --git a/autogl/solver/classifier/hetero/node_classifier.py b/autogl/solver/classifier/hetero/node_classifier.py index 834d0d5..a7cb2bf 100644 --- a/autogl/solver/classifier/hetero/node_classifier.py +++ b/autogl/solver/classifier/hetero/node_classifier.py @@ -16,7 +16,7 @@ from ..base import BaseClassifier from ...base import _parse_hp_space, _initialize_single_model from ....module.feature import FEATURE_DICT from ....module.model import MODEL_DICT, BaseModel -from ....module.train import TRAINER_DICT, BaseNodeClassificationTrainer +from ....module.train import TRAINER_DICT, NodeClassificationHetTrainer from ....module.train import get_feval from ....module.nas.space import NAS_SPACE_DICT from ....module.nas.algorithm import NAS_ALGO_DICT @@ -109,18 +109,17 @@ class AutoHeteroNodeClassifier(BaseClassifier): self.dataset = None def _init_graph_module( - self, graph_models, num_classes, num_features, feval, device, loss + self, graph_models, num_classes, num_features, feval, device, loss, dataset ) -> "AutoHeteroNodeClassifier": - # TODO: fix model initialization because hetero model should receive additional arguments # load graph network module self.graph_model_list = [] if isinstance(graph_models, Iterable): for model in graph_models: if isinstance(model, str): if model in MODEL_DICT: - # TODO: fix model initialization because hetero model should receive additional arguments self.graph_model_list.append( MODEL_DICT[model]( + dataset=dataset, num_classes=num_classes, num_features=num_features, device=device, @@ -130,9 +129,9 @@ class AutoHeteroNodeClassifier(BaseClassifier): else: raise KeyError("cannot find model %s" % (model)) elif isinstance(model, type) and issubclass(model, BaseModel): - # TODO: fix model initialization because hetero model should receive additional arguments self.graph_model_list.append( model( + dataset=dataset, num_classes=num_classes, num_features=num_features, device=device, @@ -141,11 +140,11 @@ class AutoHeteroNodeClassifier(BaseClassifier): ) elif isinstance(model, BaseModel): # setup the hp of num_classes and num_features - # TODO: setup model model.set_num_classes(num_classes) model.set_num_features(num_features) + model.from_dataset(dataset) self.graph_model_list.append(model.to(device)) - elif isinstance(model, BaseNodeClassificationTrainer): + elif isinstance(model, NodeClassificationHetTrainer): # receive a trainer list, put trainer to list assert ( model.get_model() is not None @@ -159,6 +158,7 @@ class AutoHeteroNodeClassifier(BaseClassifier): feval=feval, device=device, ) + model.model.from_dataset(dataset) self.graph_model_list.append(model) else: raise KeyError("cannot find graph network %s." % (model)) @@ -174,12 +174,13 @@ class AutoHeteroNodeClassifier(BaseClassifier): # set model hp space if self._model_hp_spaces is not None: if self._model_hp_spaces[i] is not None: - if isinstance(model, BaseNodeClassificationTrainer): + if isinstance(model, NodeClassificationHetTrainer): model.model.hyper_parameter_space = self._model_hp_spaces[i] else: model.hyper_parameter_space = self._model_hp_spaces[i] # initialize trainer if needed if isinstance(model, BaseModel): + # FIXME: seems that currently we only support passing str of trainer? name = ( self._default_trainer if isinstance(self._default_trainer, str) @@ -187,6 +188,7 @@ class AutoHeteroNodeClassifier(BaseClassifier): ) model = TRAINER_DICT[name]( model=model, + dataset=dataset, num_features=num_features, num_classes=num_classes, loss=loss, @@ -298,6 +300,7 @@ class AutoHeteroNodeClassifier(BaseClassifier): train_split if train_split > 1 else int(train_split * size) ) val_split = val_split if val_split > 1 else int(val_split * size) + # FIXME: CAUTION! May have problems utils.random_splits_mask_class( dataset, num_train_per_class=train_split // num_classes, @@ -307,6 +310,7 @@ class AutoHeteroNodeClassifier(BaseClassifier): else: train_split = train_split if train_split < 1 else train_split / size val_split = val_split if val_split < 1 else val_split / size + # FIXME: CAUTION! May have problems utils.random_splits_mask( dataset, train_ratio=train_split, val_ratio=val_split ) @@ -324,8 +328,9 @@ class AutoHeteroNodeClassifier(BaseClassifier): self.dataset = dataset # check whether the dataset has features. - # currently we only support graph classification with features. + # currently we only support hetero graph classification with features. + # FIXME: CAUTION! May have problems feat = get_graph_node_features(graph_data) assert feat is not None, ( "Does not support fit on non node-feature dataset!" @@ -335,6 +340,8 @@ class AutoHeteroNodeClassifier(BaseClassifier): num_features = feat.size(-1) + dataset_converted = convert_dataset(self.dataset) + # initialize graph networks self._init_graph_module( self.gml, @@ -343,62 +350,9 @@ class AutoHeteroNodeClassifier(BaseClassifier): feval=evaluator_list, device=self.runtime_device, loss="nll_loss" if not hasattr(dataset, "loss") else self.dataset.loss, + dataset=dataset_converted ) - if self.nas_algorithms is not None: - # perform neural architecture search - self._init_nas_module( - num_features=num_features, - num_classes=num_classes, - feval=evaluator_list, - device=self.runtime_device, - loss="nll_loss" if not hasattr(dataset, "loss") else dataset.loss, - ) - - assert not isinstance(self._default_trainer, list) or len( - self.nas_algorithms - ) == len(self._default_trainer) - len( - self.graph_model_list - ), "length of default trainer should match total graph models and nas models passed" - - # perform nas and add them to model list - idx_trainer = len(self.graph_model_list) - for algo, space, estimator in zip( - self.nas_algorithms, self.nas_spaces, self.nas_estimators - ): - model = algo.search(space, convert_dataset(self.dataset), estimator) - # insert model into default trainer - if isinstance(self._default_trainer, list): - train_name = self._default_trainer[idx_trainer] - idx_trainer += 1 - else: - train_name = self._default_trainer - if isinstance(train_name, str): - trainer = TRAINER_DICT[train_name]( - model=model, - num_features=num_features, - num_classes=num_classes, - loss="nll_loss" - if not hasattr(dataset, "loss") - else dataset.loss, - feval=evaluator_list, - device=self.runtime_device, - init=False, - ) - else: - trainer = train_name - trainer.model = model - trainer.update_parameters( - num_features=num_features, - num_classes=num_classes, - loss="nll_loss" - if not hasattr(dataset, "loss") - else dataset.loss, - feval=evaluator_list, - device=self.runtime_device, - ) - self.graph_model_list.append(trainer) - # train the models and tune hpo result_valid = [] names = [] @@ -408,11 +362,11 @@ class AutoHeteroNodeClassifier(BaseClassifier): ) if self.hpo_module is None: model.initialize() - model.train(convert_dataset(self.dataset), True) + model.train(dataset_converted, True) optimized = model else: optimized, _ = self.hpo_module.optimize( - trainer=model, dataset=convert_dataset(self.dataset), time_limit=time_for_each_model + trainer=model, dataset=dataset_converted, time_limit=time_for_each_model ) # to save memory, all the trainer derived will be mapped to cpu optimized.to(torch.device("cpu")) From bf154e77de6934535e09679be91611144e882169 Mon Sep 17 00:00:00 2001 From: Frozenmad Date: Fri, 17 Dec 2021 12:17:29 +0800 Subject: [PATCH 3/7] add hetero utils --- autogl/solver/utils.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/autogl/solver/utils.py b/autogl/solver/utils.py index f1b8c52..5c543ba 100644 --- a/autogl/solver/utils.py +++ b/autogl/solver/utils.py @@ -267,3 +267,10 @@ def set_seed(seed=None): torch.cuda.manual_seed_all(seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False + +def get_graph_labels_hetero(graph, target_node_type): + if isinstance(graph, GeneralStaticGraph): + if 'label' in graph.nodes[target_node_type].data and BACKEND == 'dgl': + return graph.nodes[target_node_type].data['label'] + return None + if BACKEND == 'dgl' and 'label' in graph.ndata[target_node_type]: return graph.ndata[target_node_type]['label'] From 092b297ab24fca241e33e555e4ebb0e8e26adce5 Mon Sep 17 00:00:00 2001 From: Frozenmad Date: Sun, 19 Dec 2021 23:53:50 +0800 Subject: [PATCH 4/7] fix bugs for model, trainer, solver --- autogl/module/model/dgl/base.py | 3 + autogl/module/model/dgl/hetero/han.py | 7 +- autogl/module/train/__init__.py | 2 + .../module/train/node_classification_het.py | 2 +- autogl/solver/__init__.py | 3 +- autogl/solver/classifier/__init__.py | 2 + autogl/solver/classifier/hetero/__init__.py | 1 + .../classifier/hetero/node_classifier.py | 272 +++++------------- examples/hetero_node_classification.py | 8 + test/performance/heterogeneous/dgl/solver.py | 25 ++ 10 files changed, 114 insertions(+), 211 deletions(-) create mode 100644 autogl/solver/classifier/hetero/__init__.py create mode 100644 examples/hetero_node_classification.py create mode 100644 test/performance/heterogeneous/dgl/solver.py diff --git a/autogl/module/model/dgl/base.py b/autogl/module/model/dgl/base.py index 5d0840b..6ad7ede 100644 --- a/autogl/module/model/dgl/base.py +++ b/autogl/module/model/dgl/base.py @@ -35,6 +35,9 @@ class BaseAutoModel(AutoModule): self._kwargs = kwargs super(BaseAutoModel, self).__init__(device) + def to(self, device): + return self.to_device(device) + def to_device(self, device): self.device = device if self.model is not None: diff --git a/autogl/module/model/dgl/hetero/han.py b/autogl/module/model/dgl/hetero/han.py index a31ff0a..f8796ec 100644 --- a/autogl/module/model/dgl/hetero/han.py +++ b/autogl/module/model/dgl/hetero/han.py @@ -133,7 +133,7 @@ class HAN(nn.Module): self.layers.append(HANLayer(self.args["meta_paths"], self.args["num_features"], self.args["hidden"][0], self.args["heads"][0], self.args["dropout"], act)) for l in range(1, len(self.args["heads"])): self.layers.append(HANLayer(self.args["meta_paths"], self.args["hidden"][l-1] * self.args["heads"][l-1], - self.args["hidden"], self.args["heads"][l], self.args["dropout"], act)) + self.args["hidden"][l], self.args["heads"][l], self.args["dropout"], act)) self.predict = nn.Linear(self.args["hidden"][-1] * self.args["heads"][-1], self.args["num_class"]) def forward(self, g): @@ -206,7 +206,10 @@ class AutoHAN(BaseHeteroModelMaintainer): "parameterName": "heads", "type": "NUMERICAL_LIST", "numericalType": "INTEGER", - "feasiblePoints": "[8]", + "scalingType": "LOG", + "length": 3, + "minValue": [1, 1, 1], + "maxValue": [16, 16, 16], "cutPara": ("num_layers",), "cutFunc": lambda x: x[0] - 1, }, diff --git a/autogl/module/train/__init__.py b/autogl/module/train/__init__.py index cf428b3..c83f1c3 100644 --- a/autogl/module/train/__init__.py +++ b/autogl/module/train/__init__.py @@ -7,6 +7,7 @@ from .base import ( BaseNodeClassificationTrainer, BaseGraphClassificationTrainer, BaseLinkPredictionTrainer, + BaseNodeClassificationHetTrainer ) @@ -36,6 +37,7 @@ __all__ = [ "Evaluation", "BaseGraphClassificationTrainer", "BaseNodeClassificationTrainer", + "BaseNodeClassificationHetTrainer", "BaseLinkPredictionTrainer", "GraphClassificationFullTrainer", "NodeClassificationFullTrainer", diff --git a/autogl/module/train/node_classification_het.py b/autogl/module/train/node_classification_het.py index aaf4f78..33f0e3a 100644 --- a/autogl/module/train/node_classification_het.py +++ b/autogl/module/train/node_classification_het.py @@ -71,7 +71,7 @@ class NodeClassificationHetTrainer(BaseNodeClassificationHetTrainer): early_stopping_round=100, weight_decay=1e-4, device="auto", - init=True, + init=False, feval=[Logloss], loss="nll_loss", lr_scheduler_type=None, diff --git a/autogl/solver/__init__.py b/autogl/solver/__init__.py index 1cba0b1..5896f5e 100644 --- a/autogl/solver/__init__.py +++ b/autogl/solver/__init__.py @@ -2,12 +2,13 @@ Auto solver for various graph tasks """ -from .classifier import AutoGraphClassifier, AutoNodeClassifier, AutoLinkPredictor +from .classifier import AutoGraphClassifier, AutoNodeClassifier, AutoLinkPredictor, AutoHeteroNodeClassifier from .utils import LeaderBoard __all__ = [ "AutoNodeClassifier", "AutoGraphClassifier", "AutoLinkPredictor", + "AutoHeteroNodeClassifier", "LeaderBoard", ] diff --git a/autogl/solver/classifier/__init__.py b/autogl/solver/classifier/__init__.py index e30c582..5c7137f 100644 --- a/autogl/solver/classifier/__init__.py +++ b/autogl/solver/classifier/__init__.py @@ -6,10 +6,12 @@ from .base import BaseClassifier from .graph_classifier import AutoGraphClassifier from .node_classifier import AutoNodeClassifier from .link_predictor import AutoLinkPredictor +from .hetero import AutoHeteroNodeClassifier __all__ = [ "BaseClassifier", "AutoGraphClassifier", "AutoNodeClassifier", "AutoLinkPredictor", + "AutoHeteroNodeClassifier" ] diff --git a/autogl/solver/classifier/hetero/__init__.py b/autogl/solver/classifier/hetero/__init__.py new file mode 100644 index 0000000..fbffed0 --- /dev/null +++ b/autogl/solver/classifier/hetero/__init__.py @@ -0,0 +1 @@ +from .node_classifier import AutoHeteroNodeClassifier \ No newline at end of file diff --git a/autogl/solver/classifier/hetero/node_classifier.py b/autogl/solver/classifier/hetero/node_classifier.py index a7cb2bf..b719498 100644 --- a/autogl/solver/classifier/hetero/node_classifier.py +++ b/autogl/solver/classifier/hetero/node_classifier.py @@ -10,18 +10,15 @@ import torch import numpy as np import yaml -from typing import Iterable - from ..base import BaseClassifier -from ...base import _parse_hp_space, _initialize_single_model +from ...base import _parse_hp_space, _initialize_single_model, _parse_model_hp from ....module.feature import FEATURE_DICT -from ....module.model import MODEL_DICT, BaseModel -from ....module.train import TRAINER_DICT, NodeClassificationHetTrainer +from ....module.train import TRAINER_DICT, BaseNodeClassificationHetTrainer from ....module.train import get_feval from ....module.nas.space import NAS_SPACE_DICT from ....module.nas.algorithm import NAS_ALGO_DICT -from ....module.nas.estimator import NAS_ESTIMATOR_DICT, BaseEstimator -from ...utils import LeaderBoard, get_graph_from_dataset, get_graph_labels, get_graph_masks, get_graph_node_features, get_graph_node_number, set_seed, convert_dataset +from ....module.nas.estimator import NAS_ESTIMATOR_DICT +from ...utils import LeaderBoard, set_seed from ....datasets import utils from ....utils import get_logger @@ -76,7 +73,6 @@ class AutoHeteroNodeClassifier(BaseClassifier): def __init__( self, - feature_module=None, graph_models=("han", "hgt"), hpo_module="anneal", ensemble_module="voting", @@ -89,7 +85,8 @@ class AutoHeteroNodeClassifier(BaseClassifier): ): super().__init__( - feature_module=feature_module, + # currently we do not support feature engineering + feature_module=None, graph_models=graph_models, # currently we do not support nas for heterogeneous node classifier nas_algorithms=None, @@ -113,97 +110,53 @@ class AutoHeteroNodeClassifier(BaseClassifier): ) -> "AutoHeteroNodeClassifier": # load graph network module self.graph_model_list = [] - if isinstance(graph_models, Iterable): - for model in graph_models: - if isinstance(model, str): - if model in MODEL_DICT: - self.graph_model_list.append( - MODEL_DICT[model]( - dataset=dataset, - num_classes=num_classes, - num_features=num_features, - device=device, - init=False, - ) - ) - else: - raise KeyError("cannot find model %s" % (model)) - elif isinstance(model, type) and issubclass(model, BaseModel): - self.graph_model_list.append( - model( - dataset=dataset, - num_classes=num_classes, - num_features=num_features, - device=device, - init=False, - ) - ) - elif isinstance(model, BaseModel): - # setup the hp of num_classes and num_features - model.set_num_classes(num_classes) - model.set_num_features(num_features) - model.from_dataset(dataset) - self.graph_model_list.append(model.to(device)) - elif isinstance(model, NodeClassificationHetTrainer): - # receive a trainer list, put trainer to list - assert ( - model.get_model() is not None - ), "Passed trainer should contain a model" - model.model.set_num_classes(num_classes) - model.model.set_num_features(num_features) - model.update_parameters( - num_classes=num_classes, - num_features=num_features, - loss=loss, - feval=feval, - device=device, - ) - model.model.from_dataset(dataset) - self.graph_model_list.append(model) + + for i, model in enumerate(graph_models): + # init the trainer + if not isinstance(model, BaseNodeClassificationHetTrainer): + trainer = ( + self._default_trainer if not isinstance(self._default_trainer, (tuple, list)) + else self._default_trainer[i] + ) + if isinstance(trainer, str): + trainer = TRAINER_DICT[trainer]() + if isinstance(model, (tuple, list)): + trainer.encoder = model[0] + trainer.decoder = model[1] else: - raise KeyError("cannot find graph network %s." % (model)) - else: - raise ValueError( - "need graph network to be (list of) str or a BaseModel class/instance, get", - graph_models, - "instead.", - ) + trainer.encoder = model + else: + trainer = model - # wrap all model_cls with specified trainer - for i, model in enumerate(self.graph_model_list): # set model hp space if self._model_hp_spaces is not None: if self._model_hp_spaces[i] is not None: - if isinstance(model, NodeClassificationHetTrainer): - model.model.hyper_parameter_space = self._model_hp_spaces[i] + if isinstance(self._model_hp_spaces[i], dict): + encoder_hp_space = self._model_hp_spaces[i].get('encoder', None) + decoder_hp_space = self._model_hp_spaces[i].get('decoder', None) else: - model.hyper_parameter_space = self._model_hp_spaces[i] - # initialize trainer if needed - if isinstance(model, BaseModel): - # FIXME: seems that currently we only support passing str of trainer? - name = ( - self._default_trainer - if isinstance(self._default_trainer, str) - else self._default_trainer[i] - ) - model = TRAINER_DICT[name]( - model=model, - dataset=dataset, - num_features=num_features, - num_classes=num_classes, - loss=loss, - feval=feval, - device=device, - init=False, - ) + encoder_hp_space = self._model_hp_spaces[i] + decoder_hp_space = None + if encoder_hp_space is not None: + trainer.encoder.hyper_parameter_space = encoder_hp_space + if decoder_hp_space is not None: + trainer.decoder.hyper_parameter_space = decoder_hp_space + # set trainer hp space if self._trainer_hp_space is not None: if isinstance(self._trainer_hp_space[0], list): current_hp_for_trainer = self._trainer_hp_space[i] else: current_hp_for_trainer = self._trainer_hp_space - model.hyper_parameter_space = current_hp_for_trainer - self.graph_model_list[i] = model + trainer.hyper_parameter_space = current_hp_for_trainer + + trainer.num_features = num_features + trainer.num_classes = num_classes + trainer.from_dataset(dataset) + trainer.loss = loss + trainer.feval = feval + trainer.to(device) + self.graph_model_list.append(trainer) return self @@ -212,10 +165,6 @@ class AutoHeteroNodeClassifier(BaseClassifier): self, dataset, time_limit=-1, - inplace=False, - train_split=None, - val_split=None, - balanced=True, evaluation_method="infer", seed=None, ) -> "AutoHeteroNodeClassifier": @@ -235,21 +184,6 @@ class AutoHeteroNodeClassifier(BaseClassifier): Whether we process the given dataset in inplace manner. Default ``False``. Set it to True if you want to save memory by modifying the given dataset directly. - train_split: float or int (Optional) - The train ratio (in ``float``) or number (in ``int``) of dataset. If you want to - use default train/val/test split in dataset, please set this to ``None``. - Default ``None``. - - val_split: float or int (Optional) - The validation ratio (in ``float``) or number (in ``int``) of dataset. If you want - to use default train/val/test split in dataset, please set this to ``None``. - Default ``None``. - - balanced: bool - Wether to create the train/valid/test split in a balanced way. - If set to ``True``, the train/valid will have the same number of different classes. - Default ``True``. - evaluation_method: (list of) str or autogl.module.train.evaluation A (list of) evaluation method for current solver. If ``infer``, will automatically determine. Default ``infer``. @@ -269,8 +203,9 @@ class AutoHeteroNodeClassifier(BaseClassifier): time_limit = 3600 * 24 time_begin = time.time() - graph_data = get_graph_from_dataset(dataset, 0) - all_labels = get_graph_labels(graph_data) + graph_data = dataset[0] + field = dataset.schema["target_node_type"] + all_labels = graph_data.nodes[field].data['label'] num_classes = all_labels.max().item() + 1 # initialize leaderboard @@ -291,47 +226,17 @@ class AutoHeteroNodeClassifier(BaseClassifier): {e.get_eval_name(): e.is_higher_better() for e in evaluator_list}, ) - # set up the dataset - if train_split is not None and val_split is not None: - size = get_graph_node_number(graph_data) - if balanced: - train_split = ( - train_split if train_split > 1 else int(train_split * size) - ) - val_split = val_split if val_split > 1 else int(val_split * size) - # FIXME: CAUTION! May have problems - utils.random_splits_mask_class( - dataset, - num_train_per_class=train_split // num_classes, - num_val_per_class=val_split // num_classes, - seed=seed, - ) - else: - train_split = train_split if train_split < 1 else train_split / size - val_split = val_split if val_split < 1 else val_split / size - # FIXME: CAUTION! May have problems - utils.random_splits_mask( - dataset, train_ratio=train_split, val_ratio=val_split - ) - else: - assert get_graph_masks(graph_data, 'train') is not None and get_graph_masks(graph_data, 'val') is not None, ( - "The dataset has no default train/val split! Please manually pass " - "train and val ratio." - ) - LOGGER.info("Use the default train/val/test ratio in given dataset") - - # feature engineering - if self.feature_module is not None: - dataset = self.feature_module.fit_transform(dataset, inplace=inplace) + assert ("train_mask" in graph_data.nodes[field].data + and "val_mask" in graph_data.nodes[field].data), ("Currently only support" + " Dataset with default train/val/test split") self.dataset = dataset # check whether the dataset has features. # currently we only support hetero graph classification with features. - # FIXME: CAUTION! May have problems - feat = get_graph_node_features(graph_data) + feat = graph_data.nodes[field].data['feat'] assert feat is not None, ( "Does not support fit on non node-feature dataset!" " Please add node features to dataset or specify feature engineers that generate" @@ -340,8 +245,6 @@ class AutoHeteroNodeClassifier(BaseClassifier): num_features = feat.size(-1) - dataset_converted = convert_dataset(self.dataset) - # initialize graph networks self._init_graph_module( self.gml, @@ -350,7 +253,7 @@ class AutoHeteroNodeClassifier(BaseClassifier): feval=evaluator_list, device=self.runtime_device, loss="nll_loss" if not hasattr(dataset, "loss") else self.dataset.loss, - dataset=dataset_converted + dataset=dataset ) # train the models and tune hpo @@ -362,11 +265,11 @@ class AutoHeteroNodeClassifier(BaseClassifier): ) if self.hpo_module is None: model.initialize() - model.train(dataset_converted, True) + model.train(dataset, True) optimized = model else: optimized, _ = self.hpo_module.optimize( - trainer=model, dataset=dataset_converted, time_limit=time_for_each_model + trainer=model, dataset=dataset, time_limit=time_for_each_model ) # to save memory, all the trainer derived will be mapped to cpu optimized.to(torch.device("cpu")) @@ -389,7 +292,7 @@ class AutoHeteroNodeClassifier(BaseClassifier): if self.ensemble_module is not None: performance = self.ensemble_module.fit( result_valid, - all_labels[get_graph_masks(graph_data, 'val')].cpu().numpy(), + all_labels[graph_data.nodes[field].data["val_mask"]].cpu().numpy(), names, evaluator_list, n_classes=num_classes, @@ -405,10 +308,6 @@ class AutoHeteroNodeClassifier(BaseClassifier): self, dataset, time_limit=-1, - inplace=False, - train_split=None, - val_split=None, - balanced=True, evaluation_method="infer", use_ensemble=True, use_best=True, @@ -471,16 +370,10 @@ class AutoHeteroNodeClassifier(BaseClassifier): self.fit( dataset=dataset, time_limit=time_limit, - inplace=inplace, - train_split=train_split, - val_split=val_split, - balanced=balanced, evaluation_method=evaluation_method, ) return self.predict( dataset=dataset, - inplaced=inplace, - inplace=inplace, use_ensemble=use_ensemble, use_best=use_best, name=name, @@ -489,8 +382,6 @@ class AutoHeteroNodeClassifier(BaseClassifier): def predict_proba( self, dataset=None, - inplaced=False, - inplace=False, use_ensemble=True, use_best=True, name=None, @@ -540,8 +431,6 @@ class AutoHeteroNodeClassifier(BaseClassifier): assert dataset is not None, ( "Please execute fit() first before" " predicting on remembered dataset" ) - elif not inplaced and self.feature_module is not None: - dataset = self.feature_module.transform(dataset, inplace=inplace) if use_ensemble: LOGGER.info("Ensemble argument on, will try using ensemble model.") @@ -589,7 +478,7 @@ class AutoHeteroNodeClassifier(BaseClassifier): def _predict_proba_by_name(self, dataset, name, mask="test"): self.trained_models[name].to(self.runtime_device) predicted = ( - self.trained_models[name].predict_proba(convert_dataset(dataset), mask=mask).cpu().numpy() + self.trained_models[name].predict_proba(dataset, mask=mask).cpu().numpy() ) self.trained_models[name].to(torch.device("cpu")) return predicted @@ -597,8 +486,6 @@ class AutoHeteroNodeClassifier(BaseClassifier): def predict( self, dataset=None, - inplaced=False, - inplace=False, use_ensemble=True, use_best=True, name=None, @@ -644,7 +531,7 @@ class AutoHeteroNodeClassifier(BaseClassifier): The prediction on given dataset. """ proba = self.predict_proba( - dataset, inplaced, inplace, use_ensemble, use_best, name, mask + dataset, use_ensemble, use_best, name, mask ) return np.argmax(proba, axis=1) @@ -700,30 +587,25 @@ class AutoHeteroNodeClassifier(BaseClassifier): path_or_dict = deepcopy(path_or_dict) solver = cls(None, [], None, None) - fe_list = path_or_dict.pop("feature", None) - if fe_list is not None: - fe_list_ele = [] - for feature_engineer in fe_list: - name = feature_engineer.pop("name") - if name is not None: - fe_list_ele.append(FEATURE_DICT[name](**feature_engineer)) - if fe_list_ele != []: - solver.set_feature_module(fe_list_ele) - - models = path_or_dict.pop("models", [{"name": "gcn"}, {"name": "gat"}]) + + models = path_or_dict.pop("models", [{"name": "hgt"}, {"name": "han"}]) + # models should be a list of model + # with each element in two cases + # * a dict describing a certain model + # * a dict containing {"encoder": encoder, "decoder": decoder} model_hp_space = [ - _parse_hp_space(model.pop("hp_space", None)) for model in models + _parse_model_hp(model) for model in models ] model_list = [ - _initialize_single_model(model.pop("name"), model) for model in models + _initialize_single_model(model) for model in models ] trainer = path_or_dict.pop("trainer", None) - default_trainer = "NodeClassificationFull" + default_trainer = "NodeClassificationHet" trainer_space = None if isinstance(trainer, dict): # global default - default_trainer = trainer.pop("name", "NodeClassificationFull") + default_trainer = trainer.pop("name", "NodeClassificationHet") trainer_space = _parse_hp_space(trainer.pop("hp_space", None)) default_kwargs = {"num_features": None, "num_classes": None} default_kwargs.update(trainer) @@ -742,7 +624,7 @@ class AutoHeteroNodeClassifier(BaseClassifier): trainer_space = [] for i in range(len(model_list)): train, model = trainer[i], model_list[i] - default_trainer = train.pop("name", "NodeClassificationFull") + default_trainer = train.pop("name", "NodeClassificationHet") trainer_space.append(_parse_hp_space(train.pop("hp_space", None))) default_kwargs = {"num_features": None, "num_classes": None} default_kwargs.update(train) @@ -766,28 +648,4 @@ class AutoHeteroNodeClassifier(BaseClassifier): name = ensemble_dict.pop("name") solver.set_ensemble_module(name, **ensemble_dict) - nas_dict = path_or_dict.pop("nas", None) - if nas_dict is not None: - keys: set = set(nas_dict.keys()) - needed = {"space", "algorithm", "estimator"} - if keys != needed: - LOGGER.error("Key mismatch, we need %s, you give %s", needed, keys) - raise KeyError("Key mismatch, we need %s, you give %s" % (needed, keys)) - - spaces, algorithms, estimators = [], [], [] - - for container, indexer, k in zip( - [spaces, algorithms, estimators], - [NAS_SPACE_DICT, NAS_ALGO_DICT, NAS_ESTIMATOR_DICT], - ["space", "algorithm", "estimator"], - ): - configs = nas_dict[k] - if isinstance(configs, list): - for item in configs: - container.append(indexer[item.pop("name")](**item)) - else: - container.append(indexer[configs.pop("name")](**configs)) - - solver.set_nas_module(algorithms, spaces, estimators) - return solver diff --git a/examples/hetero_node_classification.py b/examples/hetero_node_classification.py new file mode 100644 index 0000000..89ff6b5 --- /dev/null +++ b/examples/hetero_node_classification.py @@ -0,0 +1,8 @@ +from autogl.datasets import build_dataset_from_name +from autogl.solver import AutoHeteroNodeClassifier + +if __name__ == '__main__': + acm = build_dataset_from_name("hetero-acm-han") + solver = AutoHeteroNodeClassifier(max_evals=10) + solver.fit(acm) + res = solver.predict_proba() diff --git a/test/performance/heterogeneous/dgl/solver.py b/test/performance/heterogeneous/dgl/solver.py new file mode 100644 index 0000000..bdd7c59 --- /dev/null +++ b/test/performance/heterogeneous/dgl/solver.py @@ -0,0 +1,25 @@ +from autogl.datasets import build_dataset_from_name +from autogl.solver import AutoHeteroNodeClassifier + +if __name__ == '__main__': + import argparse + parser = argparse.ArgumentParser() + parser.add_argument("--model", type=str, choices=["han", "hgt", "heteroRGCN"], default="hgt") + parser.add_argument("--epoch", type=int, default=200) + parser.add_argument("--lr", type=float, default=1e-3) + parser.add_argument("--weight_decay", type=float, default=1e-2) + parser.add_argument("--device", type=str, default="cuda") + parser.add_argument("--repeat", type=int, default=10) + + args = parser.parse_args() + + dataset = { + "han": "hetero-acm-han", + "hgt": "hetero-acm-hgt", + "heteroRGCN": "hetero-acm-hgt" + } + + dataset = build_dataset_from_name(dataset[args.model]) + + for rep in range(args.repeat): + pass From c2ce43a3e29ff142555bc3768a209e663a99dbeb Mon Sep 17 00:00:00 2001 From: Frozenmad Date: Mon, 20 Dec 2021 10:23:29 +0800 Subject: [PATCH 5/7] add solver test file --- .../_dgl_heterogeneous_datasets.py | 2 +- .../module/train/node_classification_het.py | 2 +- autogl/solver/classifier/__init__.py | 6 ++- .../classifier/hetero/node_classifier.py | 23 +++++++++++ .../performance/heterogeneous/dgl/han_main.py | 6 +-- test/performance/heterogeneous/dgl/helper.py | 28 +++++++++++++ test/performance/heterogeneous/dgl/solver.py | 40 +++++++++++++++++-- .../heterogeneous/dgl/train_hgt.py | 2 +- 8 files changed, 99 insertions(+), 10 deletions(-) diff --git a/autogl/datasets/_heterogeneous_datasets/_dgl_heterogeneous_datasets.py b/autogl/datasets/_heterogeneous_datasets/_dgl_heterogeneous_datasets.py index 4c314a1..133f75b 100644 --- a/autogl/datasets/_heterogeneous_datasets/_dgl_heterogeneous_datasets.py +++ b/autogl/datasets/_heterogeneous_datasets/_dgl_heterogeneous_datasets.py @@ -10,7 +10,7 @@ from .. import _dataset_registry def get_binary_mask(total_size, indices): mask = torch.zeros(total_size) mask[indices] = 1 - return mask.byte() + return mask.bool() @_dataset_registry.DatasetUniversalRegistry.register_dataset("hetero-acm-han") diff --git a/autogl/module/train/node_classification_het.py b/autogl/module/train/node_classification_het.py index 33f0e3a..6c5b035 100644 --- a/autogl/module/train/node_classification_het.py +++ b/autogl/module/train/node_classification_het.py @@ -343,7 +343,7 @@ class NodeClassificationHetTrainer(BaseNodeClassificationHetTrainer): def _get_mask(self, dataset, mask): if mask in ["train", "val", "test"]: - return dataset[0].nodes[dataset.schema["target_node_type"]].data[f"{mask}_mask"].bool() + return dataset[0].nodes[dataset.schema["target_node_type"]].data[f"{mask}_mask"] return mask def evaluate(self, dataset, mask='val', feval = None): diff --git a/autogl/solver/classifier/__init__.py b/autogl/solver/classifier/__init__.py index 5c7137f..c82dea5 100644 --- a/autogl/solver/classifier/__init__.py +++ b/autogl/solver/classifier/__init__.py @@ -6,7 +6,11 @@ from .base import BaseClassifier from .graph_classifier import AutoGraphClassifier from .node_classifier import AutoNodeClassifier from .link_predictor import AutoLinkPredictor -from .hetero import AutoHeteroNodeClassifier +from autogl.backend import DependentBackend +if DependentBackend.is_dgl(): + from .hetero import AutoHeteroNodeClassifier +else: + AutoHeteroNodeClassifier = None __all__ = [ "BaseClassifier", diff --git a/autogl/solver/classifier/hetero/node_classifier.py b/autogl/solver/classifier/hetero/node_classifier.py index b719498..9f539b3 100644 --- a/autogl/solver/classifier/hetero/node_classifier.py +++ b/autogl/solver/classifier/hetero/node_classifier.py @@ -5,6 +5,7 @@ import time import json from copy import deepcopy +from typing import Sequence import torch import numpy as np @@ -535,6 +536,28 @@ class AutoHeteroNodeClassifier(BaseClassifier): ) return np.argmax(proba, axis=1) + + def evaluate(self, dataset=None, + use_ensemble=True, + use_best=True, + name=None, + mask="test", + label=None, + metric="acc" + ): + predicted = self.predict_proba(dataset, use_ensemble, use_best, name, mask) + if dataset is None: + dataset = self.dataset + if label is None: + graph_nodes = dataset[0].nodes[dataset.schema["target_node_type"]].data + if mask in ["train", "val", "test"]: + mask = graph_nodes[f"{mask}_mask"] + label = graph_nodes["label"][mask].cpu().numpy() + evaluator = get_feval(metric) + if isinstance(evaluator, Sequence): + return [evals.evaluate(predicted, label) for evals in evaluator] + return evaluator.evaluate(predicted, label) + @classmethod def from_config(cls, path_or_dict, filetype="auto") -> "AutoHeteroNodeClassifier": """ diff --git a/test/performance/heterogeneous/dgl/han_main.py b/test/performance/heterogeneous/dgl/han_main.py index 9792af7..e6d70eb 100644 --- a/test/performance/heterogeneous/dgl/han_main.py +++ b/test/performance/heterogeneous/dgl/han_main.py @@ -64,9 +64,9 @@ def main(args): num_classes = labels.max().item() + 1 labels = labels.to(args['device']) - train_mask = g.nodes[node_type].data['train_mask'].to(args['device']).bool() - val_mask = g.nodes[node_type].data['val_mask'].to(args['device']).bool() - test_mask = g.nodes[node_type].data['test_mask'].to(args['device']).bool() + train_mask = g.nodes[node_type].data['train_mask'].to(args['device']) + val_mask = g.nodes[node_type].data['val_mask'].to(args['device']) + test_mask = g.nodes[node_type].data['test_mask'].to(args['device']) model = AutoHAN( dataset=dataset, diff --git a/test/performance/heterogeneous/dgl/helper.py b/test/performance/heterogeneous/dgl/helper.py index d727943..e798b77 100644 --- a/test/performance/heterogeneous/dgl/helper.py +++ b/test/performance/heterogeneous/dgl/helper.py @@ -35,3 +35,31 @@ class EarlyStopping(object): def load_checkpoint(self, model): """Load the latest checkpoint.""" model.load_state_dict(pickle.loads(self.model)) + +def get_encoder_decoder_hp(model='han'): + if model == "han": + return { + "num_layers": 2, + "hidden": [256], ## + "heads": [8], ## + "dropout": 0.2, + "act": "gelu", + }, None + if model == "hgt": + return { + "num_layers": 2, + "hidden": [256,256,256], + "heads": 4, + "dropout": 0.2, + "act": "gelu", + "use_norm": True, + }, None + if model == "HeteroRGCN": + return { + "num_layers": 2, + "hidden": [256], + "heads": 4, + "dropout": 0.2, + "act": "leaky_relu", + }, None + return {}, None \ No newline at end of file diff --git a/test/performance/heterogeneous/dgl/solver.py b/test/performance/heterogeneous/dgl/solver.py index bdd7c59..02e6868 100644 --- a/test/performance/heterogeneous/dgl/solver.py +++ b/test/performance/heterogeneous/dgl/solver.py @@ -1,10 +1,20 @@ +import numpy as np from autogl.datasets import build_dataset_from_name from autogl.solver import AutoHeteroNodeClassifier +from helper import get_encoder_decoder_hp +from tqdm import tqdm + +def fixed(**kwargs): + return [{ + 'parameterName': k, + "type": "FIXED", + "value": v + } for k, v in kwargs.items()] if __name__ == '__main__': import argparse parser = argparse.ArgumentParser() - parser.add_argument("--model", type=str, choices=["han", "hgt", "heteroRGCN"], default="hgt") + parser.add_argument("--model", type=str, choices=["han", "hgt", "HeteroRGCN"], default="hgt") parser.add_argument("--epoch", type=int, default=200) parser.add_argument("--lr", type=float, default=1e-3) parser.add_argument("--weight_decay", type=float, default=1e-2) @@ -16,10 +26,34 @@ if __name__ == '__main__': dataset = { "han": "hetero-acm-han", "hgt": "hetero-acm-hgt", - "heteroRGCN": "hetero-acm-hgt" + "HeteroRGCN": "hetero-acm-hgt" } dataset = build_dataset_from_name(dataset[args.model]) + model_hp, _ = get_encoder_decoder_hp(args.model) + + accs = [] + process = tqdm(total=args.repeat) for rep in range(args.repeat): - pass + solver = AutoHeteroNodeClassifier( + graph_models=[args.model], + hpo_module="random", + ensemble_module=None, + max_evals=1, + device=args.device, + trainer_hp_space=fixed( + max_epoch=args.epoch, + early_stopping_round=args.epoch + 1, + lr=args.lr, + weight_decay=args.weight_decay + ), + model_hp_spaces=[fixed(**model_hp)] + ) + solver.fit(dataset) + acc = solver.evaluate() + accs.append(acc) + process.update(1) + process.set_postfix(mean=np.mean(accs), std=np.std(accs)) + process.close() + print("mean: {:.4f} ~ std: {:.4f}".format(np.mean(accs), np.std(accs))) diff --git a/test/performance/heterogeneous/dgl/train_hgt.py b/test/performance/heterogeneous/dgl/train_hgt.py index d35ed78..6a4d37a 100644 --- a/test/performance/heterogeneous/dgl/train_hgt.py +++ b/test/performance/heterogeneous/dgl/train_hgt.py @@ -30,7 +30,7 @@ def main(args): num_features = 256 labels = G.nodes[field].data['label'].to(args.device) num_classes = labels.max().item()+1 - test_mask = G.nodes[field].data['test_mask'].bool().to(args.device) + test_mask = G.nodes[field].data['test_mask'].to(args.device) accs = [] for seed in tqdm(range(args.repeat)): From 3cbc8ec8eac7c5cf2555a7902718f0f25b81b1e1 Mon Sep 17 00:00:00 2001 From: Frozenmad Date: Mon, 20 Dec 2021 10:25:49 +0800 Subject: [PATCH 6/7] remove uncessary import --- autogl/solver/classifier/hetero/node_classifier.py | 5 ----- autogl/solver/classifier/link_predictor.py | 1 - autogl/solver/classifier/node_classifier.py | 5 +---- 3 files changed, 1 insertion(+), 10 deletions(-) diff --git a/autogl/solver/classifier/hetero/node_classifier.py b/autogl/solver/classifier/hetero/node_classifier.py index 9f539b3..9f944b2 100644 --- a/autogl/solver/classifier/hetero/node_classifier.py +++ b/autogl/solver/classifier/hetero/node_classifier.py @@ -13,14 +13,9 @@ import yaml from ..base import BaseClassifier from ...base import _parse_hp_space, _initialize_single_model, _parse_model_hp -from ....module.feature import FEATURE_DICT from ....module.train import TRAINER_DICT, BaseNodeClassificationHetTrainer from ....module.train import get_feval -from ....module.nas.space import NAS_SPACE_DICT -from ....module.nas.algorithm import NAS_ALGO_DICT -from ....module.nas.estimator import NAS_ESTIMATOR_DICT from ...utils import LeaderBoard, set_seed -from ....datasets import utils from ....utils import get_logger LOGGER = get_logger("HeteroNodeClassifier") diff --git a/autogl/solver/classifier/link_predictor.py b/autogl/solver/classifier/link_predictor.py index d6b6c47..e4ca2a0 100644 --- a/autogl/solver/classifier/link_predictor.py +++ b/autogl/solver/classifier/link_predictor.py @@ -12,7 +12,6 @@ import torch import numpy as np import yaml -from ...data import Data from .base import BaseClassifier from ..base import _parse_hp_space, _initialize_single_model, _parse_model_hp from ...module.feature import FEATURE_DICT diff --git a/autogl/solver/classifier/node_classifier.py b/autogl/solver/classifier/node_classifier.py index 5afd174..cd04dd6 100644 --- a/autogl/solver/classifier/node_classifier.py +++ b/autogl/solver/classifier/node_classifier.py @@ -5,18 +5,15 @@ import time import json from copy import deepcopy -from typing import Union, Sequence, Tuple +from typing import Sequence import torch import numpy as np import yaml -from typing import Iterable - from .base import BaseClassifier from ..base import _parse_hp_space, _initialize_single_model, _parse_model_hp from ...module.feature import FEATURE_DICT -from ...module.model import BaseEncoderMaintainer, BaseDecoderMaintainer, BaseAutoModel from ...module.train import TRAINER_DICT, BaseNodeClassificationTrainer from ...module.train import get_feval from ...module.nas.space import NAS_SPACE_DICT From fc38edea94e2c01b8caea93779751c8976bdeb89 Mon Sep 17 00:00:00 2001 From: Frozenmad Date: Mon, 20 Dec 2021 10:29:08 +0800 Subject: [PATCH 7/7] finish the solver setting --- examples/hetero_node_classification.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/examples/hetero_node_classification.py b/examples/hetero_node_classification.py index 89ff6b5..47b0bb7 100644 --- a/examples/hetero_node_classification.py +++ b/examples/hetero_node_classification.py @@ -1,3 +1,6 @@ +import os +os.environ["AUTOGL_BACKEND"] = 'dgl' + from autogl.datasets import build_dataset_from_name from autogl.solver import AutoHeteroNodeClassifier @@ -5,4 +8,6 @@ if __name__ == '__main__': acm = build_dataset_from_name("hetero-acm-han") solver = AutoHeteroNodeClassifier(max_evals=10) solver.fit(acm) - res = solver.predict_proba() + acc = solver.evaluate(metric='acc') + + print("acc: ", acc)