PR [#39] nas_example -> dev

Fix bugs in nas, revise example.

autogl/module/nas/estimator/train_scratch.py

@@ -16,6 +16,7 @@ class TrainEstimator(BaseEstimator):
     """
     def __init__(self, loss_f = "nll_loss", evaluation = [Acc()]):
         super().__init__(loss_f, evaluation)
+        self.evaluation = evaluation
         self.estimator=OneShotEstimator(self.loss_f, self.evaluation)
 
     def infer(self, model: BaseSpace, dataset, mask="train"):
@@ -34,6 +35,14 @@ class TrainEstimator(BaseEstimator):
                     feval=self.evaluation,
                     loss=self.loss_f,
                     lr_scheduler_type=None)
-        self.trainer.train(dataset)
-        with torch.no_grad():
-            return self.estimator.infer(boxmodel.model, dataset, mask)
+        try:
+            self.trainer.train(dataset)
+            with torch.no_grad():
+                return self.estimator.infer(boxmodel.model, dataset, mask)
+        except RuntimeError as e:
+            if "cuda" in str(e) or "CUDA" in str(e):
+                INF = 100
+                fin = [-INF if eva.is_higher_better else INF for eva in self.evaluation] 
+                return fin, 0
+            else:
+                raise e

autogl/module/nas/space/base.py

@@ -89,6 +89,7 @@ class BoxModel(BaseModel):
         self.num_classes = self._model.output_dim
         self.params = {"num_class": self.num_classes, "features_num": self.num_features}
         self.device = device
+        self.selection = None
 
     def fix(self, selection):
         """
@@ -119,7 +120,8 @@ class BoxModel(BaseModel):
 
         ret_self = deepcopy(self)
         ret_self._model.instantiate()
-        apply_fixed_architecture(ret_self._model, ret_self.selection, verbose=False)
+        if ret_self.selection:
+            apply_fixed_architecture(ret_self._model, ret_self.selection, verbose=False)
         ret_self.to(self.device)
         return ret_self
 

configs/nodeclf_nas_benchmark.yml

@@ -11,7 +11,7 @@ nas:
     hidden_dim: 64
     layer_number: 4
   algorithm:
-    name: rl
+    name: graphnas
     num_epochs: 200
   estimator:
     name: scratch

configs/nodeclf_nas_darts_benchmark.yml

@@ -0,0 +1,42 @@
+ensemble:
+  name: null
+feature:
+- name: PYGNormalizeFeatures
+hpo:
+  max_evals: 10
+  name: random
+nas:
+  space:
+    name: singlepath
+    hidden_dim: 64
+    layer_number: 2
+    dropout: 0.8
+    ops: ['gcn', 'gat', 'linear'] 
+  algorithm:
+    name: darts
+    num_epochs: 200
+  estimator:
+    name: oneshot
+models: []
+trainer:
+  hp_space:
+  - maxValue: 300
+    minValue: 100
+    parameterName: max_epoch
+    scalingType: LINEAR
+    type: INTEGER
+  - maxValue: 30
+    minValue: 10
+    parameterName: early_stopping_round
+    scalingType: LINEAR
+    type: INTEGER
+  - maxValue: 0.05
+    minValue: 0.01
+    parameterName: lr
+    scalingType: LOG
+    type: DOUBLE
+  - maxValue: 0.0005
+    minValue: 5.0e-05
+    parameterName: weight_decay
+    scalingType: LOG
+    type: DOUBLE

configs/nodeclf_nas_enas_benchmark.yml

@@ -0,0 +1,42 @@
+ensemble:
+  name: null
+feature:
+- name: PYGNormalizeFeatures
+hpo:
+  max_evals: 10
+  name: random
+nas:
+  space:
+    name: singlepath
+    hidden_dim: 64
+    layer_number: 2
+    dropout: 0.8
+    ops: ['gcn', 'gat', 'linear'] 
+  algorithm:
+    name: enas
+    num_epochs: 200
+  estimator:
+    name: oneshot
+models: []
+trainer:
+  hp_space:
+  - maxValue: 300
+    minValue: 100
+    parameterName: max_epoch
+    scalingType: LINEAR
+    type: INTEGER
+  - maxValue: 30
+    minValue: 10
+    parameterName: early_stopping_round
+    scalingType: LINEAR
+    type: INTEGER
+  - maxValue: 0.05
+    minValue: 0.01
+    parameterName: lr
+    scalingType: LOG
+    type: DOUBLE
+  - maxValue: 0.0005
+    minValue: 5.0e-05
+    parameterName: weight_decay
+    scalingType: LOG
+    type: DOUBLE

configs/nodeclf_nas_macro_benchmark.yml

@@ -0,0 +1,40 @@
+ensemble:
+  name: null
+feature:
+- name: PYGNormalizeFeatures
+hpo:
+  max_evals: 10
+  name: random
+nas:
+  space:
+    name: graphnasmacro
+    hidden_dim: 64
+    layer_number: 2
+  algorithm:
+    name: graphnas
+    num_epochs: 200
+  estimator:
+    name: scratch
+models: []
+trainer:
+  hp_space:
+  - maxValue: 300
+    minValue: 100
+    parameterName: max_epoch
+    scalingType: LINEAR
+    type: INTEGER
+  - maxValue: 30
+    minValue: 10
+    parameterName: early_stopping_round
+    scalingType: LINEAR
+    type: INTEGER
+  - maxValue: 0.05
+    minValue: 0.01
+    parameterName: lr
+    scalingType: LOG
+    type: DOUBLE
+  - maxValue: 0.0005
+    minValue: 5.0e-05
+    parameterName: weight_decay
+    scalingType: LOG
+    type: DOUBLE

examples/test_nas.py

@@ -0,0 +1,22 @@
+import sys
+sys.path.append('../')
+from autogl.datasets import build_dataset_from_name
+from autogl.solver import AutoNodeClassifier
+from autogl.module.train import Acc
+from autogl.solver.utils import set_seed
+import argparse
+
+if __name__ == '__main__':
+    set_seed(202106)
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--config', type=str, default='../configs/nodeclf_nas_macro_benchmark.yml')
+    parser.add_argument('--dataset', choices=['cora', 'citeseer', 'pubmed'], default='cora', type=str)
+
+    args = parser.parse_args()
+
+    dataset = build_dataset_from_name('cora')
+    solver = AutoNodeClassifier.from_config(args.config)
+    solver.fit(dataset)
+    solver.get_leaderboard().show()
+    out = solver.predict_proba()
+    print('acc on dataset', Acc.evaluate(out, dataset[0].y[dataset[0].test_mask].detach().numpy()))