Merge branch 'Dev' of https://github.com/AbductiveLearning/ABL-Package into Dev

abl/bridge/base_bridge.py

@@ -5,8 +5,6 @@ from ..learning import ABLModel
 from ..reasoning import Reasoner
 from ..structures import ListData
 
-DataSet = Tuple[List[List[Any]], Optional[List[List[Any]]], List[List[Any]]]
-
 
 class BaseBridge(metaclass=ABCMeta):
     def __init__(self, model: ABLModel, reasoner: Reasoner) -> None:
@@ -24,19 +22,19 @@ class BaseBridge(metaclass=ABCMeta):
 
     @abstractmethod
     def predict(self, data_samples: ListData) -> Tuple[List[List[Any]], List[List[Any]]]:
-        """Placeholder for predict labels from input."""
+        """Placeholder for predicting labels from input."""
 
     @abstractmethod
     def abduce_pseudo_label(self, data_samples: ListData) -> List[List[Any]]:
-        """Placeholder for abduce pseudo labels."""
+        """Placeholder for abducing pseudo labels."""
 
     @abstractmethod
     def idx_to_pseudo_label(self, data_samples: ListData) -> List[List[Any]]:
-        """Placeholder for map label space to symbol space."""
+        """Placeholder for mapping indexes to pseudo labels."""
 
     @abstractmethod
     def pseudo_label_to_idx(self, data_samples: ListData) -> List[List[Any]]:
-        """Placeholder for map symbol space to label space."""
+        """Placeholder for mapping pseudo labels to indexes."""
 
     def filter_pseudo_label(self, data_samples: ListData) -> List[List[Any]]:
         '''Default filter function for pseudo label.'''
@@ -48,13 +46,22 @@ class BaseBridge(metaclass=ABCMeta):
         return data_samples
 
     @abstractmethod
-    def train(self, train_data: Union[ListData, DataSet]):
-        """Placeholder for train loop of ABductive Learning."""
+    def train(
+        self,
+        train_data: Union[ListData, Tuple[List[List[Any]], Optional[List[List[Any]]], List[Any]]],
+    ):
+        """Placeholder for training loop of ABductive Learning."""
 
     @abstractmethod
-    def valid(self, valid_data: Union[ListData, DataSet]) -> None:
+    def valid(
+        self,
+        valid_data: Union[ListData, Tuple[List[List[Any]], Optional[List[List[Any]]], List[Any]]],
+    ) -> None:
         """Placeholder for model test."""
 
     @abstractmethod
-    def test(self, test_data: Union[ListData, DataSet]) -> None:
+    def test(
+        self,
+        test_data: Union[ListData, Tuple[List[List[Any]], Optional[List[List[Any]]], List[Any]]],
+    ) -> None:
         """Placeholder for model validation."""

abl/bridge/simple_bridge.py

@@ -8,7 +8,7 @@ from ..learning import ABLModel
 from ..reasoning import Reasoner
 from ..structures import ListData
 from ..utils import print_log
-from .base_bridge import BaseBridge, DataSet
+from .base_bridge import BaseBridge
 
 
 class SimpleBridge(BaseBridge):
@@ -55,8 +55,10 @@ class SimpleBridge(BaseBridge):
 
     def train(
         self,
-        train_data: Union[ListData, DataSet],
-        val_data: Optional[Union[ListData, DataSet]] = None,
+        train_data: Union[ListData, Tuple[List[List[Any]], Optional[List[List[Any]]], List[Any]]],
+        val_data: Optional[
+            Union[ListData, Tuple[List[List[Any]], Optional[List[List[Any]]], List[Any]]]
+        ] = None,
         loops: int = 50,
         segment_size: Union[int, float] = -1,
         eval_interval: int = 1,
@@ -81,7 +83,8 @@ class SimpleBridge(BaseBridge):
                 self.model.train(sub_data_samples)
 
                 print_log(
-                    f"loop(train) [{loop + 1}/{loops}] segment(train) [{(seg_idx + 1)}/{(len(data_samples) - 1) // segment_size + 1}]",
+                    f"loop(train) [{loop + 1}/{loops}] segment(train) "
+                    f"[{(seg_idx + 1)}/{(len(data_samples) - 1) // segment_size + 1}] ",
                     logger="current",
                 )
 
@@ -113,12 +116,18 @@ class SimpleBridge(BaseBridge):
             msg += k + f": {v:.3f} "
         print_log(msg, logger="current")
 
-    def valid(self, valid_data: Union[ListData, DataSet]) -> None:
+    def valid(
+        self,
+        valid_data: Union[ListData, Tuple[List[List[Any]], Optional[List[List[Any]]], List[Any]]],
+    ) -> None:
         if not isinstance(valid_data, ListData):
             data_samples = self.data_preprocess(*valid_data)
         else:
             data_samples = valid_data
         self._valid(data_samples)
 
-    def test(self, test_data: Union[ListData, DataSet]) -> None:
+    def test(
+        self,
+        test_data: Union[ListData, Tuple[List[List[Any]], Optional[List[List[Any]]], List[Any]]],
+    ) -> None:
         self.valid(test_data)

abl/dataset/classification_dataset.py

@@ -15,7 +15,7 @@ class ClassificationDataset(Dataset):
     Y : List[int]
         The target data.
     transform : Callable[..., Any], optional
-        A function/transform that takes in an object and returns a transformed version.
+        A function/transform that takes an object and returns a transformed version.
         Defaults to None.
     """
 

abl/dataset/prediction_dataset.py

@@ -13,7 +13,7 @@ class PredictionDataset(Dataset):
     X : List[Any]
         The input data.
     transform : Callable[..., Any], optional
-        A function/transform that takes in an object and returns a transformed version.
+        A function/transform that takes an object and returns a transformed version.
         Defaults to None.
     """
 

abl/evaluation/base_metric.py

@@ -1,7 +1,8 @@
 import logging
 from abc import ABCMeta, abstractmethod
-from typing import Any, List, Optional, Sequence
+from typing import Any, List, Optional
 
+from ..structures import ListData
 from ..utils import print_log
 
 
@@ -28,23 +29,20 @@ class BaseMetric(metaclass=ABCMeta):
         self.prefix = prefix or self.default_prefix
 
     @abstractmethod
-    def process(self, data_samples: Sequence[dict]) -> None:
+    def process(self, data_samples: ListData) -> None:
         """Process one batch of data samples and predictions. The processed
         results should be stored in ``self.results``, which will be used to
         compute the metrics when all batches have been processed.
 
         Args:
-            data_samples (Sequence[dict]): A batch of outputs from
+            data_samples (ListData): A batch of outputs from
                 the model.
         """
 
     @abstractmethod
-    def compute_metrics(self, results: list) -> dict:
+    def compute_metrics(self) -> dict:
         """Compute the metrics from processed results.
 
-        Args:
-            results (list): The processed results of each batch.
-
         Returns:
             dict: The computed metrics. The keys are the names of the metrics,
             and the values are corresponding results.
@@ -54,13 +52,6 @@ class BaseMetric(metaclass=ABCMeta):
         """Evaluate the model performance of the whole dataset after processing
         all batches.
 
-        Args:
-            size (int): Length of the entire validation dataset. When batch
-                size > 1, the dataloader may pad some data samples to make
-                sure all ranks have the same length of dataset slice. The
-                ``collect_results`` function will drop the padded data based on
-                this size.
-
         Returns:
             dict: Evaluation metrics dict on the val dataset. The keys are the
             names of the metrics, and the values are corresponding results.
@@ -74,7 +65,7 @@ class BaseMetric(metaclass=ABCMeta):
                 level=logging.WARNING,
             )
 
-        metrics = self.compute_metrics(self.results)
+        metrics = self.compute_metrics()
         # Add prefix to metric names
         if self.prefix:
             metrics = {"/".join((self.prefix, k)): v for k, v in metrics.items()}

abl/evaluation/semantics_metric.py

@@ -1,6 +1,7 @@
-from typing import Optional, Sequence
+from typing import Optional
 
 from ..reasoning import KBBase
+from ..structures import ListData
 from .base_metric import BaseMetric
 
 
@@ -9,7 +10,7 @@ class SemanticsMetric(BaseMetric):
         super().__init__(prefix)
         self.kb = kb
 
-    def process(self, data_samples: Sequence[dict]) -> None:
+    def process(self, data_samples: ListData) -> None:
         pred_pseudo_label_list = data_samples.pred_pseudo_label
         y_list = data_samples.Y
         x_list = data_samples.X
@@ -19,7 +20,8 @@ class SemanticsMetric(BaseMetric):
             else:
                 self.results.append(0)
 
-    def compute_metrics(self, results: list) -> dict:
+    def compute_metrics(self) -> dict:
+        results = self.results
         metrics = dict()
         metrics["semantics_accuracy"] = sum(results) / len(results)
         return metrics

abl/evaluation/symbol_metric.py

@@ -1,5 +1,6 @@
-from typing import Optional, Sequence
+from typing import Optional
 
+from ..structures import ListData
 from .base_metric import BaseMetric
 
 
@@ -7,22 +8,21 @@ class SymbolMetric(BaseMetric):
     def __init__(self, prefix: Optional[str] = None) -> None:
         super().__init__(prefix)
 
-    def process(self, data_samples: Sequence[dict]) -> None:
-        pred_pseudo_label = data_samples.pred_pseudo_label
+    def process(self, data_samples: ListData) -> None:
+        pred_pseudo_label_list = data_samples.flatten("pred_pseudo_label")
+        gt_pseudo_label_list = data_samples.flatten("gt_pseudo_label")
 
-        gt_pseudo_label = data_samples.gt_pseudo_label
-
-        if not len(pred_pseudo_label) == len(gt_pseudo_label):
+        if not len(pred_pseudo_label_list) == len(gt_pseudo_label_list):
             raise ValueError("lengthes of pred_pseudo_label and gt_pseudo_label should be equal")
 
-        for pred_z, z in zip(pred_pseudo_label, gt_pseudo_label):
-            correct_num = 0
-            for pred_symbol, symbol in zip(pred_z, z):
-                if pred_symbol == symbol:
-                    correct_num += 1
-            self.results.append(correct_num / len(z))
+        correct_num = 0
+        for pred_pseudo_label, gt_pseudo_label in zip(pred_pseudo_label_list, gt_pseudo_label_list):
+            if pred_pseudo_label == gt_pseudo_label:
+                correct_num += 1
+        self.results.append((correct_num, len(pred_pseudo_label_list)))
 
-    def compute_metrics(self, results: list) -> dict:
+    def compute_metrics(self) -> dict:
+        results = self.results
         metrics = dict()
-        metrics["character_accuracy"] = sum(results) / len(results)
+        metrics["character_accuracy"] = sum(t[0] for t in results) / sum(t[1] for t in results)
         return metrics

abl/learning/basic_nn.py

@@ -38,10 +38,10 @@ class BasicNN:
     save_dir : Optional[str], optional
         The directory in which to save the model during training, by default None.
     train_transform : Callable[..., Any], optional
-        A function/transform that takes in an object and returns a transformed version used
+        A function/transform that takes an object and returns a transformed version used
         in the `fit` and `train_epoch` methods, by default None.
     test_transform : Callable[..., Any], optional
-        A function/transform that takes in an object and returns a transformed version in the
+        A function/transform that takes an object and returns a transformed version in the
         `predict`, `predict_proba` and `score` methods, , by default None.
     collate_fn : Callable[[List[T]], Any], optional
         The function used to collate data, by default None.

docs/Intro/Bridge.rst

@@ -35,6 +35,8 @@ Bridging machine learning and reasoning to train the model is the fundamental id
 | test(test_data)                   | Test the model.                                                                      |
 +-----------------------------------+--------------------------------------------------------------------------------------+
 
+where ``train_data`` and ``test_data`` are both in the form of ``(X, gt_pseudo_label, Y)``. They will be used to construct ``ListData`` instances which are referred to as ``data_samples`` in the ``train`` and ``test`` methods respectively. More details can be found in `preparing datasets <Datasets.html>`_.
+
 
 ``SimpleBridge`` inherits from ``BaseBridge`` and provides a basic implementation. Besides the ``model`` and ``reasoner``, ``SimpleBridge`` has an extra initialization arguments, ``metric_list``, which will be used to evaluate model performance. Its training process involves several Abductive Learning loops and each loop consists of the following five steps:
 

docs/Intro/Evaluation.rst

@@ -10,8 +10,14 @@
 Evaluation Metrics
 ==================
 
-ABL-Package seperates the evaluation process as an independent class from the ``BaseBridge`` which accounts for training and testing. To customize our own metrics, we need to inherit from ``BaseMetric`` and implement the ``process`` and ``compute_metrics`` methods. The ``process`` method accepts a batch of model prediction. After processing this batch, we save the information to ``self.results`` property. The input results of ``compute_metrics`` is all the information saved in ``process`` and it uses these information to calculate and return a dict that holds the evaluation results. 
+ABL-Package seperates the evaluation process from model training and testing as an independent class, ``BaseMetric``. The training and testing processes are implemented in the ``BaseBridge`` class, so metrics are used by this class and its sub-classes. After building a ``bridge`` with a list of ``BaseMetric`` instances, these metrics will be used by the ``bridge.valid`` method to evaluate the model performance during training and testing. 
 
+To customize our own metrics, we need to inherit from ``BaseMetric`` and implement the ``process`` and ``compute_metrics`` methods. 
+
+- The ``process`` method accepts a batch of model prediction and saves the information to ``self.results`` property after processing this batch. 
+- The ``compute_metrics`` method uses all the information saved in ``self.results`` to calculate and return a dict that holds the evaluation results. 
+
+Besides, we can assign a ``str`` to the ``prefix`` argument of the ``__init__`` method. This string is automatically prefixed to the output metric names. For example, if we set ``prefix="mnist_add"``, the output metric name will be ``character_accuracy``.
 We provide two basic metrics, namely ``SymbolMetric`` and ``SemanticsMetric``, which are used to evaluate the accuracy of the machine learning model's predictions and the accuracy of the ``logic_forward`` results, respectively. Using ``SymbolMetric`` as an example, the following code shows how to implement a custom metrics.
 
 .. code:: python

docs/Intro/Quick-Start.rst

@@ -15,7 +15,7 @@ Working with Data
 -----------------
 
 ABL-Package assumes data to be in the form of ``(X, gt_pseudo_label, Y)``  where ``X`` is the input of the machine learning model, 
-``gt_pseudo_label`` is the ground truth label of each element in ``X`` and ``Y`` is the ground truth reasoning result of each instance in ``X``. 
+``gt_pseudo_label`` is the ground truth label of each element in ``X`` and ``Y`` is the ground truth reasoning result of each instance in ``X``. Note that ``gt_pseudo_label`` is only used to evaluate the performance of the machine learning part but not to train the model. If elements in ``X`` are unlabeled, ``gt_pseudo_label`` can be ``None``.
 
 In the MNIST Addition task, the data loading looks like
 

examples/hed/hed_bridge.py

@@ -7,7 +7,7 @@ from abl.bridge import SimpleBridge
 from abl.dataset import RegressionDataset
 from abl.evaluation import BaseMetric
 from abl.learning import ABLModel, BasicNN
-from abl.reasoning import ReasonerBase
+from abl.reasoning import Reasoner
 from abl.structures import ListData
 from abl.utils import print_log
 from examples.hed.datasets.get_hed import get_pretrain_data
@@ -19,7 +19,7 @@ class HEDBridge(SimpleBridge):
     def __init__(
         self,
         model: ABLModel,
-        reasoner: ReasonerBase,
+        reasoner: Reasoner,
         metric_list: BaseMetric,
     ) -> None:
         super().__init__(model, reasoner, metric_list)
@@ -92,11 +92,11 @@ class HEDBridge(SimpleBridge):
     def check_training_impact(self, filtered_data_samples, data_samples):
         character_accuracy = self.model.valid(filtered_data_samples)
         revisible_ratio = len(filtered_data_samples.X) / len(data_samples.X)
-        print_log(
-            f"Revisible ratio is {revisible_ratio:.3f}, Character \
-                accuracy is {character_accuracy:.3f}",
-            logger="current",
+        log_string = (
+            f"Revisible ratio is {revisible_ratio:.3f}, Character "
+            f"accuracy is {character_accuracy:.3f}"
         )
+        print_log(log_string, logger="current")
 
         if character_accuracy >= 0.9 and revisible_ratio >= 0.9:
             return True
@@ -109,11 +109,11 @@ class HEDBridge(SimpleBridge):
         true_ratio = self.calc_consistent_ratio(val_X_true, rule)
         false_ratio = self.calc_consistent_ratio(val_X_false, rule)
 
-        print_log(
-            f"True consistent ratio is {true_ratio:.3f}, False inconsistent ratio \
-                is {1 - false_ratio:.3f}",
-            logger="current",
+        log_string = (
+            f"True consistent ratio is {true_ratio:.3f}, False inconsistent ratio "
+            f"is {1 - false_ratio:.3f}"
         )
+        print_log(log_string, logger="current")
 
         if true_ratio > 0.95 and false_ratio < 0.1:
             return True

examples/hed/hed_example.ipynb

@@ -14,12 +14,11 @@
     "\n",
     "from abl.evaluation import SemanticsMetric, SymbolMetric\n",
     "from abl.learning import ABLModel, BasicNN\n",
-    "from abl.reasoning import PrologKB, ReasonerBase\n",
+    "from abl.reasoning import PrologKB, Reasoner\n",
     "from abl.utils import ABLLogger, print_log, reform_list\n",
     "from examples.hed.datasets.get_hed import get_hed, split_equation\n",
     "from examples.hed.hed_bridge import HEDBridge\n",
-    "from examples.models.nn import SymbolNet\n",
-    "from zoopt import Dimension, Objective, Parameter, Opt"
+    "from examples.models.nn import SymbolNet"
    ]
   },
   {
@@ -68,7 +67,7 @@
     "        return rules\n",
     "\n",
     "\n",
-    "class HedReasoner(ReasonerBase):\n",
+    "class HedReasoner(Reasoner):\n",
     "    def revise_at_idx(self, data_sample):\n",
     "        revision_idx = np.where(np.array(data_sample.flatten(\"revision_flag\")) != 0)[0]\n",
     "        candidate = self.kb.revise_at_idx(\n",

examples/hwf/hwf_example.ipynb

@@ -11,7 +11,7 @@
     "import torch.nn as nn\n",
     "import os.path as osp\n",
     "\n",
-    "from abl.reasoning import ReasonerBase, KBBase\n",
+    "from abl.reasoning import Reasoner, KBBase\n",
     "from abl.learning import BasicNN, ABLModel\n",
     "from abl.bridge import SimpleBridge\n",
     "from abl.evaluation import SymbolMetric, SemanticsMetric\n",
@@ -75,7 +75,7 @@
     "    max_err=1e-10,\n",
     "    use_cache=False,\n",
     ")\n",
-    "reasoner = ReasonerBase(kb, dist_func=\"confidence\")"
+    "reasoner = Reasoner(kb, dist_func=\"confidence\")"
    ]
   },
   {
@@ -220,7 +220,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.16"
+   "version": "3.8.18"
   },
   "orig_nbformat": 4,
   "vscode": {

examples/mnist_add/mnist_add_example.ipynb

@@ -14,7 +14,7 @@
     "from abl.bridge import SimpleBridge\n",
     "from abl.evaluation import SemanticsMetric, SymbolMetric\n",
     "from abl.learning import ABLModel, BasicNN\n",
-    "from abl.reasoning import KBBase, ReasonerBase\n",
+    "from abl.reasoning import KBBase, Reasoner\n",
     "from abl.utils import ABLLogger, print_log\n",
     "from examples.mnist_add.datasets.get_mnist_add import get_mnist_add\n",
     "from examples.models.nn import LeNet5"
@@ -109,7 +109,7 @@
     "\n",
     "\n",
     "kb = AddKB(pseudo_label_list=list(range(10)))\n",
-    "reasoner = ReasonerBase(kb, dist_func=\"confidence\")"
+    "reasoner = Reasoner(kb, dist_func=\"confidence\")"
    ]
   },
   {