From 72289caccd8eeae8b34ee8d4fdc40839c403426c Mon Sep 17 00:00:00 2001 From: Gao Enhao Date: Sun, 10 Dec 2023 01:10:06 +0800 Subject: [PATCH 1/5] [MNT] flatten DataSet type --- abl/bridge/base_bridge.py | 27 +++++++++++++++++---------- abl/bridge/simple_bridge.py | 28 ++++++++++++++++++---------- 2 files changed, 35 insertions(+), 20 deletions(-) diff --git a/abl/bridge/base_bridge.py b/abl/bridge/base_bridge.py index ba1a663..c9261fb 100644 --- a/abl/bridge/base_bridge.py +++ b/abl/bridge/base_bridge.py @@ -5,8 +5,6 @@ from ..learning import ABLModel from ..reasoning import Reasoner from ..structures import ListData -DataSet = Tuple[List[List[Any]], Optional[List[List[Any]]], List[List[Any]]] - class BaseBridge(metaclass=ABCMeta): def __init__(self, model: ABLModel, reasoner: Reasoner) -> None: @@ -24,28 +22,37 @@ class BaseBridge(metaclass=ABCMeta): @abstractmethod def predict(self, data_samples: ListData) -> Tuple[List[List[Any]], List[List[Any]]]: - """Placeholder for predict labels from input.""" + """Placeholder for predicting labels from input.""" @abstractmethod def abduce_pseudo_label(self, data_samples: ListData) -> List[List[Any]]: - """Placeholder for abduce pseudo labels.""" + """Placeholder for abducing pseudo labels.""" @abstractmethod def idx_to_pseudo_label(self, data_samples: ListData) -> List[List[Any]]: - """Placeholder for map label space to symbol space.""" + """Placeholder for mapping indexes to pseudo labels.""" @abstractmethod def pseudo_label_to_idx(self, data_samples: ListData) -> List[List[Any]]: - """Placeholder for map symbol space to label space.""" + """Placeholder for mapping pseudo labels to indexes.""" @abstractmethod - def train(self, train_data: Union[ListData, DataSet]): - """Placeholder for train loop of ABductive Learning.""" + def train( + self, + train_data: Union[ListData, Tuple[List[List[Any]], Optional[List[List[Any]]], List[Any]]], + ): + """Placeholder for training loop of ABductive Learning.""" @abstractmethod - def valid(self, valid_data: Union[ListData, DataSet]) -> None: + def valid( + self, + valid_data: Union[ListData, Tuple[List[List[Any]], Optional[List[List[Any]]], List[Any]]], + ) -> None: """Placeholder for model test.""" @abstractmethod - def test(self, test_data: Union[ListData, DataSet]) -> None: + def test( + self, + test_data: Union[ListData, Tuple[List[List[Any]], Optional[List[List[Any]]], List[Any]]], + ) -> None: """Placeholder for model validation.""" diff --git a/abl/bridge/simple_bridge.py b/abl/bridge/simple_bridge.py index 508c106..df4accd 100644 --- a/abl/bridge/simple_bridge.py +++ b/abl/bridge/simple_bridge.py @@ -8,7 +8,7 @@ from ..learning import ABLModel from ..reasoning import Reasoner from ..structures import ListData from ..utils import print_log -from .base_bridge import BaseBridge, DataSet +from .base_bridge import BaseBridge class SimpleBridge(BaseBridge): @@ -55,8 +55,10 @@ class SimpleBridge(BaseBridge): def train( self, - train_data: Union[ListData, DataSet], - val_data: Optional[Union[ListData, DataSet]] = None, + train_data: Union[ListData, Tuple[List[List[Any]], Optional[List[List[Any]]], List[Any]]], + val_data: Optional[ + Union[ListData, Tuple[List[List[Any]], Optional[List[List[Any]]], List[Any]]] + ] = None, loops: int = 50, segment_size: Union[int, float] = -1, eval_interval: int = 1, @@ -79,12 +81,12 @@ class SimpleBridge(BaseBridge): self.pseudo_label_to_idx(sub_data_samples) loss = self.model.train(sub_data_samples) - print_log( - f"loop(train) [{loop + 1}/{loops}] segment(train) \ - [{(seg_idx + 1)}/{(len(data_samples) - 1) // segment_size + 1}] \ - model loss is {loss:.5f}", - logger="current", + log_string = ( + f"loop(train) [{loop + 1}/{loops}] segment(train) " + f"[{(seg_idx + 1)}/{(len(data_samples) - 1) // segment_size + 1}] " + f"model loss is {loss:.5f}" ) + print_log(log_string, logger="current") if (loop + 1) % eval_interval == 0 or loop == loops - 1: print_log(f"Evaluation start: loop(val) [{loop + 1}]", logger="current") @@ -114,12 +116,18 @@ class SimpleBridge(BaseBridge): msg += k + f": {v:.3f} " print_log(msg, logger="current") - def valid(self, valid_data: Union[ListData, DataSet]) -> None: + def valid( + self, + valid_data: Union[ListData, Tuple[List[List[Any]], Optional[List[List[Any]]], List[Any]]], + ) -> None: if not isinstance(valid_data, ListData): data_samples = self.data_preprocess(*valid_data) else: data_samples = valid_data self._valid(data_samples) - def test(self, test_data: Union[ListData, DataSet]) -> None: + def test( + self, + test_data: Union[ListData, Tuple[List[List[Any]], Optional[List[List[Any]]], List[Any]]], + ) -> None: self.valid(test_data) From 87348bf52d2b0c6f3555578844dd499a56210e37 Mon Sep 17 00:00:00 2001 From: Gao Enhao Date: Sun, 10 Dec 2023 01:10:58 +0800 Subject: [PATCH 2/5] [MNT] delete 'in' after 'takes' --- abl/dataset/classification_dataset.py | 2 +- abl/dataset/prediction_dataset.py | 2 +- abl/learning/basic_nn.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/abl/dataset/classification_dataset.py b/abl/dataset/classification_dataset.py index 091c3b5..2efb8f6 100644 --- a/abl/dataset/classification_dataset.py +++ b/abl/dataset/classification_dataset.py @@ -15,7 +15,7 @@ class ClassificationDataset(Dataset): Y : List[int] The target data. transform : Callable[..., Any], optional - A function/transform that takes in an object and returns a transformed version. + A function/transform that takes an object and returns a transformed version. Defaults to None. """ diff --git a/abl/dataset/prediction_dataset.py b/abl/dataset/prediction_dataset.py index 7674993..6776988 100644 --- a/abl/dataset/prediction_dataset.py +++ b/abl/dataset/prediction_dataset.py @@ -13,7 +13,7 @@ class PredictionDataset(Dataset): X : List[Any] The input data. transform : Callable[..., Any], optional - A function/transform that takes in an object and returns a transformed version. + A function/transform that takes an object and returns a transformed version. Defaults to None. """ diff --git a/abl/learning/basic_nn.py b/abl/learning/basic_nn.py index b2767b7..40effee 100644 --- a/abl/learning/basic_nn.py +++ b/abl/learning/basic_nn.py @@ -38,10 +38,10 @@ class BasicNN: save_dir : Optional[str], optional The directory in which to save the model during training, by default None. train_transform : Callable[..., Any], optional - A function/transform that takes in an object and returns a transformed version used + A function/transform that takes an object and returns a transformed version used in the `fit` and `train_epoch` methods, by default None. test_transform : Callable[..., Any], optional - A function/transform that takes in an object and returns a transformed version in the + A function/transform that takes an object and returns a transformed version in the `predict`, `predict_proba` and `score` methods, , by default None. collate_fn : Callable[[List[T]], Any], optional The function used to collate data, by default None. From f398d98cd2357382e91d11460a93a766ebcf4dc8 Mon Sep 17 00:00:00 2001 From: Gao Enhao Date: Sun, 10 Dec 2023 01:11:32 +0800 Subject: [PATCH 3/5] [MNT] resolve comments of metrics --- abl/evaluation/base_metric.py | 21 ++++++--------------- abl/evaluation/semantics_metric.py | 8 +++++--- abl/evaluation/symbol_metric.py | 28 ++++++++++++++-------------- 3 files changed, 25 insertions(+), 32 deletions(-) diff --git a/abl/evaluation/base_metric.py b/abl/evaluation/base_metric.py index 03b1997..1fbcf0f 100644 --- a/abl/evaluation/base_metric.py +++ b/abl/evaluation/base_metric.py @@ -1,7 +1,8 @@ import logging from abc import ABCMeta, abstractmethod -from typing import Any, List, Optional, Sequence +from typing import Any, List, Optional +from ..structures import ListData from ..utils import print_log @@ -28,23 +29,20 @@ class BaseMetric(metaclass=ABCMeta): self.prefix = prefix or self.default_prefix @abstractmethod - def process(self, data_samples: Sequence[dict]) -> None: + def process(self, data_samples: ListData) -> None: """Process one batch of data samples and predictions. The processed results should be stored in ``self.results``, which will be used to compute the metrics when all batches have been processed. Args: - data_samples (Sequence[dict]): A batch of outputs from + data_samples (ListData): A batch of outputs from the model. """ @abstractmethod - def compute_metrics(self, results: list) -> dict: + def compute_metrics(self) -> dict: """Compute the metrics from processed results. - Args: - results (list): The processed results of each batch. - Returns: dict: The computed metrics. The keys are the names of the metrics, and the values are corresponding results. @@ -54,13 +52,6 @@ class BaseMetric(metaclass=ABCMeta): """Evaluate the model performance of the whole dataset after processing all batches. - Args: - size (int): Length of the entire validation dataset. When batch - size > 1, the dataloader may pad some data samples to make - sure all ranks have the same length of dataset slice. The - ``collect_results`` function will drop the padded data based on - this size. - Returns: dict: Evaluation metrics dict on the val dataset. The keys are the names of the metrics, and the values are corresponding results. @@ -74,7 +65,7 @@ class BaseMetric(metaclass=ABCMeta): level=logging.WARNING, ) - metrics = self.compute_metrics(self.results) + metrics = self.compute_metrics() # Add prefix to metric names if self.prefix: metrics = {"/".join((self.prefix, k)): v for k, v in metrics.items()} diff --git a/abl/evaluation/semantics_metric.py b/abl/evaluation/semantics_metric.py index 14c4f46..7254f34 100644 --- a/abl/evaluation/semantics_metric.py +++ b/abl/evaluation/semantics_metric.py @@ -1,6 +1,7 @@ -from typing import Optional, Sequence +from typing import Optional from ..reasoning import KBBase +from ..structures import ListData from .base_metric import BaseMetric @@ -9,7 +10,7 @@ class SemanticsMetric(BaseMetric): super().__init__(prefix) self.kb = kb - def process(self, data_samples: Sequence[dict]) -> None: + def process(self, data_samples: ListData) -> None: pred_pseudo_label_list = data_samples.pred_pseudo_label y_list = data_samples.Y for pred_pseudo_label, y in zip(pred_pseudo_label_list, y_list): @@ -18,7 +19,8 @@ class SemanticsMetric(BaseMetric): else: self.results.append(0) - def compute_metrics(self, results: list) -> dict: + def compute_metrics(self) -> dict: + results = self.results metrics = dict() metrics["semantics_accuracy"] = sum(results) / len(results) return metrics diff --git a/abl/evaluation/symbol_metric.py b/abl/evaluation/symbol_metric.py index 112dc8b..46b0a70 100644 --- a/abl/evaluation/symbol_metric.py +++ b/abl/evaluation/symbol_metric.py @@ -1,5 +1,6 @@ -from typing import Optional, Sequence +from typing import Optional +from ..structures import ListData from .base_metric import BaseMetric @@ -7,22 +8,21 @@ class SymbolMetric(BaseMetric): def __init__(self, prefix: Optional[str] = None) -> None: super().__init__(prefix) - def process(self, data_samples: Sequence[dict]) -> None: - pred_pseudo_label = data_samples.pred_pseudo_label + def process(self, data_samples: ListData) -> None: + pred_pseudo_label_list = data_samples.flatten("pred_pseudo_label") + gt_pseudo_label_list = data_samples.flatten("gt_pseudo_label") - gt_pseudo_label = data_samples.gt_pseudo_label - - if not len(pred_pseudo_label) == len(gt_pseudo_label): + if not len(pred_pseudo_label_list) == len(gt_pseudo_label_list): raise ValueError("lengthes of pred_pseudo_label and gt_pseudo_label should be equal") - for pred_z, z in zip(pred_pseudo_label, gt_pseudo_label): - correct_num = 0 - for pred_symbol, symbol in zip(pred_z, z): - if pred_symbol == symbol: - correct_num += 1 - self.results.append(correct_num / len(z)) + correct_num = 0 + for pred_pseudo_label, gt_pseudo_label in zip(pred_pseudo_label_list, gt_pseudo_label_list): + if pred_pseudo_label == gt_pseudo_label: + correct_num += 1 + self.results.append((correct_num, len(pred_pseudo_label_list))) - def compute_metrics(self, results: list) -> dict: + def compute_metrics(self) -> dict: + results = self.results metrics = dict() - metrics["character_accuracy"] = sum(results) / len(results) + metrics["character_accuracy"] = sum(t[0] for t in results) / sum(t[1] for t in results) return metrics From d7fba3bcf5b85b0b106ee8c8a8c82ab25ffa3eb6 Mon Sep 17 00:00:00 2001 From: Gao Enhao Date: Sun, 10 Dec 2023 01:12:14 +0800 Subject: [PATCH 4/5] [DOC] resolve comments in Bri, Eva, Quick --- docs/Intro/Bridge.rst | 2 ++ docs/Intro/Evaluation.rst | 8 +++++++- docs/Intro/Quick-Start.rst | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/Intro/Bridge.rst b/docs/Intro/Bridge.rst index 241f7b8..c645cbf 100644 --- a/docs/Intro/Bridge.rst +++ b/docs/Intro/Bridge.rst @@ -35,6 +35,8 @@ Bridging machine learning and reasoning to train the model is the fundamental id | test(test_data) | Test the model. | +-----------------------------------+--------------------------------------------------------------------------------------+ +where ``train_data`` and ``test_data`` are both in the form of ``(X, gt_pseudo_label, Y)``. They will be used to construct ``ListData`` instances which are referred to as ``data_samples`` in the ``train`` and ``test`` methods respectively. More details can be found in `preparing datasets `_. + ``SimpleBridge`` inherits from ``BaseBridge`` and provides a basic implementation. Besides the ``model`` and ``reasoner``, ``SimpleBridge`` has an extra initialization arguments, ``metric_list``, which will be used to evaluate model performance. Its training process involves several Abductive Learning loops and each loop consists of the following five steps: diff --git a/docs/Intro/Evaluation.rst b/docs/Intro/Evaluation.rst index 42f21e5..bfbdd20 100644 --- a/docs/Intro/Evaluation.rst +++ b/docs/Intro/Evaluation.rst @@ -10,8 +10,14 @@ Evaluation Metrics ================== -ABL-Package seperates the evaluation process as an independent class from the ``BaseBridge`` which accounts for training and testing. To customize our own metrics, we need to inherit from ``BaseMetric`` and implement the ``process`` and ``compute_metrics`` methods. The ``process`` method accepts a batch of model prediction. After processing this batch, we save the information to ``self.results`` property. The input results of ``compute_metrics`` is all the information saved in ``process`` and it uses these information to calculate and return a dict that holds the evaluation results. +ABL-Package seperates the evaluation process from model training and testing as an independent class, ``BaseMetric``. The training and testing processes are implemented in the ``BaseBridge`` class, so metrics are used by this class and its sub-classes. After building a ``bridge`` with a list of ``BaseMetric`` instances, these metrics will be used by the ``bridge.valid`` method to evaluate the model performance during training and testing. +To customize our own metrics, we need to inherit from ``BaseMetric`` and implement the ``process`` and ``compute_metrics`` methods. + +- The ``process`` method accepts a batch of model prediction and saves the information to ``self.results`` property after processing this batch. +- The ``compute_metrics`` method uses all the information saved in ``self.results`` to calculate and return a dict that holds the evaluation results. + +Besides, we can assign a ``str`` to the ``prefix`` argument of the ``__init__`` method. This string is automatically prefixed to the output metric names. For example, if we set ``prefix="mnist_add"``, the output metric name will be ``character_accuracy``. We provide two basic metrics, namely ``SymbolMetric`` and ``SemanticsMetric``, which are used to evaluate the accuracy of the machine learning model's predictions and the accuracy of the ``logic_forward`` results, respectively. Using ``SymbolMetric`` as an example, the following code shows how to implement a custom metrics. .. code:: python diff --git a/docs/Intro/Quick-Start.rst b/docs/Intro/Quick-Start.rst index 48d0777..9f7f9f1 100644 --- a/docs/Intro/Quick-Start.rst +++ b/docs/Intro/Quick-Start.rst @@ -15,7 +15,7 @@ Working with Data ----------------- ABL-Package assumes data to be in the form of ``(X, gt_pseudo_label, Y)`` where ``X`` is the input of the machine learning model, -``gt_pseudo_label`` is the ground truth label of each element in ``X`` and ``Y`` is the ground truth reasoning result of each instance in ``X``. +``gt_pseudo_label`` is the ground truth label of each element in ``X`` and ``Y`` is the ground truth reasoning result of each instance in ``X``. Note that ``gt_pseudo_label`` is only used to evaluate the performance of the machine learning part but not to train the model. If elements in ``X`` are unlabeled, ``gt_pseudo_label`` can be ``None``. In the MNIST Addition task, the data loading looks like From 70678bf968d46c9150e95b6b2b43ee7ce81b3ede Mon Sep 17 00:00:00 2001 From: Gao Enhao Date: Sun, 10 Dec 2023 01:12:56 +0800 Subject: [PATCH 5/5] [MNT] change 'ReasonerBase' to 'Reasoner' --- examples/hed/hed_bridge.py | 20 ++++++++++---------- examples/hed/hed_example.ipynb | 7 +++---- examples/hwf/hwf_example.ipynb | 6 +++--- examples/mnist_add/mnist_add_example.ipynb | 4 ++-- 4 files changed, 18 insertions(+), 19 deletions(-) diff --git a/examples/hed/hed_bridge.py b/examples/hed/hed_bridge.py index 13f5946..0b08ec8 100644 --- a/examples/hed/hed_bridge.py +++ b/examples/hed/hed_bridge.py @@ -7,7 +7,7 @@ from abl.bridge import SimpleBridge from abl.dataset import RegressionDataset from abl.evaluation import BaseMetric from abl.learning import ABLModel, BasicNN -from abl.reasoning import ReasonerBase +from abl.reasoning import Reasoner from abl.structures import ListData from abl.utils import print_log from examples.hed.datasets.get_hed import get_pretrain_data @@ -19,7 +19,7 @@ class HEDBridge(SimpleBridge): def __init__( self, model: ABLModel, - reasoner: ReasonerBase, + reasoner: Reasoner, metric_list: BaseMetric, ) -> None: super().__init__(model, reasoner, metric_list) @@ -92,11 +92,11 @@ class HEDBridge(SimpleBridge): def check_training_impact(self, filtered_data_samples, data_samples): character_accuracy = self.model.valid(filtered_data_samples) revisible_ratio = len(filtered_data_samples.X) / len(data_samples.X) - print_log( - f"Revisible ratio is {revisible_ratio:.3f}, Character \ - accuracy is {character_accuracy:.3f}", - logger="current", + log_string = ( + f"Revisible ratio is {revisible_ratio:.3f}, Character " + f"accuracy is {character_accuracy:.3f}" ) + print_log(log_string, logger="current") if character_accuracy >= 0.9 and revisible_ratio >= 0.9: return True @@ -109,11 +109,11 @@ class HEDBridge(SimpleBridge): true_ratio = self.calc_consistent_ratio(val_X_true, rule) false_ratio = self.calc_consistent_ratio(val_X_false, rule) - print_log( - f"True consistent ratio is {true_ratio:.3f}, False inconsistent ratio \ - is {1 - false_ratio:.3f}", - logger="current", + log_string = ( + f"True consistent ratio is {true_ratio:.3f}, False inconsistent ratio " + f"is {1 - false_ratio:.3f}" ) + print_log(log_string, logger="current") if true_ratio > 0.95 and false_ratio < 0.1: return True diff --git a/examples/hed/hed_example.ipynb b/examples/hed/hed_example.ipynb index e479b52..0e9c9d4 100644 --- a/examples/hed/hed_example.ipynb +++ b/examples/hed/hed_example.ipynb @@ -14,12 +14,11 @@ "\n", "from abl.evaluation import SemanticsMetric, SymbolMetric\n", "from abl.learning import ABLModel, BasicNN\n", - "from abl.reasoning import PrologKB, ReasonerBase\n", + "from abl.reasoning import PrologKB, Reasoner\n", "from abl.utils import ABLLogger, print_log, reform_list\n", "from examples.hed.datasets.get_hed import get_hed, split_equation\n", "from examples.hed.hed_bridge import HEDBridge\n", - "from examples.models.nn import SymbolNet\n", - "from zoopt import Dimension, Objective, Parameter, Opt" + "from examples.models.nn import SymbolNet" ] }, { @@ -68,7 +67,7 @@ " return rules\n", "\n", "\n", - "class HedReasoner(ReasonerBase):\n", + "class HedReasoner(Reasoner):\n", " def revise_at_idx(self, data_sample):\n", " revision_idx = np.where(np.array(data_sample.flatten(\"revision_flag\")) != 0)[0]\n", " candidate = self.kb.revise_at_idx(\n", diff --git a/examples/hwf/hwf_example.ipynb b/examples/hwf/hwf_example.ipynb index 485087c..bae8d0d 100644 --- a/examples/hwf/hwf_example.ipynb +++ b/examples/hwf/hwf_example.ipynb @@ -11,7 +11,7 @@ "import torch.nn as nn\n", "import os.path as osp\n", "\n", - "from abl.reasoning import ReasonerBase, KBBase\n", + "from abl.reasoning import Reasoner, KBBase\n", "from abl.learning import BasicNN, ABLModel\n", "from abl.bridge import SimpleBridge\n", "from abl.evaluation import SymbolMetric, SemanticsMetric\n", @@ -75,7 +75,7 @@ " max_err=1e-10,\n", " use_cache=False,\n", ")\n", - "reasoner = ReasonerBase(kb, dist_func=\"confidence\")" + "reasoner = Reasoner(kb, dist_func=\"confidence\")" ] }, { @@ -220,7 +220,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.16" + "version": "3.8.18" }, "orig_nbformat": 4, "vscode": { diff --git a/examples/mnist_add/mnist_add_example.ipynb b/examples/mnist_add/mnist_add_example.ipynb index eb4e4d3..980094f 100644 --- a/examples/mnist_add/mnist_add_example.ipynb +++ b/examples/mnist_add/mnist_add_example.ipynb @@ -14,7 +14,7 @@ "from abl.bridge import SimpleBridge\n", "from abl.evaluation import SemanticsMetric, SymbolMetric\n", "from abl.learning import ABLModel, BasicNN\n", - "from abl.reasoning import KBBase, ReasonerBase\n", + "from abl.reasoning import KBBase, Reasoner\n", "from abl.utils import ABLLogger, print_log\n", "from examples.mnist_add.datasets.get_mnist_add import get_mnist_add\n", "from examples.models.nn import LeNet5" @@ -109,7 +109,7 @@ "\n", "\n", "kb = AddKB(pseudo_label_list=list(range(10)))\n", - "reasoner = ReasonerBase(kb, dist_func=\"confidence\")" + "reasoner = Reasoner(kb, dist_func=\"confidence\")" ] }, {