From 48f34e5d95db3b43aa00bcb3735e78571a08d1fb Mon Sep 17 00:00:00 2001 From: troyyyyy Date: Wed, 17 Jan 2024 17:40:51 +0800 Subject: [PATCH 1/8] [MNT] update results for HWF --- examples/hwf/README.md | 7 +++++-- examples/hwf/hwf.ipynb | 9 +++++++-- examples/hwf/main.py | 23 +++++++++++------------ examples/mnist_add/README.md | 5 ++++- examples/mnist_add/main.py | 13 ++++++++----- 5 files changed, 35 insertions(+), 22 deletions(-) diff --git a/examples/hwf/README.md b/examples/hwf/README.md index b1a591d..7412ce0 100644 --- a/examples/hwf/README.md +++ b/examples/hwf/README.md @@ -12,7 +12,8 @@ python main.py ## Usage ```bash -usage: main.py [-h] [--no-cuda] [--epochs EPOCHS] [--lr LR] +usage: main.py [-h] [--no-cuda] [--epochs EPOCHS] + [--label_smoothing LABEL_SMOOTHING] [--lr LR] [--batch-size BATCH_SIZE] [--loops LOOPS] [--segment_size SEGMENT_SIZE] [--save_interval SAVE_INTERVAL] [--max-revision MAX_REVISION] @@ -26,6 +27,8 @@ optional arguments: --no-cuda disables CUDA training --epochs EPOCHS number of epochs in each learning loop iteration (default : 1) + --label_smoothing LABEL_SMOOTHING + label smoothing in cross entropy loss (default : 0.2) --lr LR base model learning rate (default : 0.001) --batch-size BATCH_SIZE base model batch size (default : 32) @@ -100,7 +103,7 @@ We present the results of ABL as follows, which include the reasoning accuracy ( 89.7 96.5 97.2 - 98.6 + 99.2 77.3 diff --git a/examples/hwf/hwf.ipynb b/examples/hwf/hwf.ipynb index baf161d..ab39b10 100644 --- a/examples/hwf/hwf.ipynb +++ b/examples/hwf/hwf.ipynb @@ -166,7 +166,7 @@ "source": [ "# class of symbol may be one of ['1', ..., '9', '+', '-', '*', '/'], total of 13 classes\n", "cls = SymbolNet(num_classes=13, image_size=(45, 45, 1))\n", - "loss_fn = nn.CrossEntropyLoss()\n", + "loss_fn = nn.CrossEntropyLoss(label_smoothing=0.2)\n", "optimizer = torch.optim.Adam(cls.parameters(), lr=0.001)\n", "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n", "\n", @@ -503,13 +503,18 @@ " 89.7\n", " 96.5\n", " 97.2\n", - " 98.6\n", + " 99.2\n", " 77.3\n", " \n", "\n", "\n", "

* timeout: need more than 1 hour to execute

" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] } ], "metadata": { diff --git a/examples/hwf/main.py b/examples/hwf/main.py index e1e297f..fbcd53a 100644 --- a/examples/hwf/main.py +++ b/examples/hwf/main.py @@ -77,6 +77,12 @@ def main(): default=3, help="number of epochs in each learning loop iteration (default : 3)", ) + parser.add_argument( + "--label-smoothing", + type=float, + default=0.2, + help="label smoothing in cross entropy loss (default : 0.2)" + ) parser.add_argument( "--lr", type=float, default=1e-3, help="base model learning rate (default : 0.001)" ) @@ -84,17 +90,14 @@ def main(): "--batch-size", type=int, default=128, help="base model batch size (default : 128)" ) parser.add_argument( - "--loops", type=int, default=5, help="number of loop iterations (default : 5)" + "--loops", type=int, default=3, help="number of loop iterations (default : 3)" ) parser.add_argument( "--segment_size", type=int, default=1000, help="segment size (default : 1000)" ) parser.add_argument("--save_interval", type=int, default=1, help="save interval (default : 1)") parser.add_argument( - "--max-revision", - type=int, - default=-1, - help="maximum revision in reasoner (default : -1)", + "--max-revision", type=int, default=-1, help="maximum revision in reasoner (default : -1)" ) parser.add_argument( "--require-more-revision", @@ -128,19 +131,14 @@ def main(): # Build necessary components for BasicNN cls = SymbolNet(num_classes=13, image_size=(45, 45, 1)) - loss_fn = nn.CrossEntropyLoss() + loss_fn = nn.CrossEntropyLoss(label_smoothing=args.label_smoothing) optimizer = torch.optim.Adam(cls.parameters(), lr=args.lr) use_cuda = not args.no_cuda and torch.cuda.is_available() device = torch.device("cuda" if use_cuda else "cpu") # Build BasicNN base_model = BasicNN( - cls, - loss_fn, - optimizer, - device=device, - batch_size=args.batch_size, - num_epochs=args.epochs, + cls, loss_fn, optimizer, device=device, batch_size=args.batch_size, num_epochs=args.epochs, ) # Build ABLModel @@ -175,6 +173,7 @@ def main(): # Train and Test bridge.train( train_data, + val_data=test_data, loops=args.loops, segment_size=args.segment_size, save_interval=args.save_interval, diff --git a/examples/mnist_add/README.md b/examples/mnist_add/README.md index c9f24c6..390c265 100644 --- a/examples/mnist_add/README.md +++ b/examples/mnist_add/README.md @@ -12,7 +12,8 @@ python main.py ## Usage ```bash -usage: main.py [-h] [--no-cuda] [--epochs EPOCHS] [--lr LR] +usage: main.py [-h] [--no-cuda] [--epochs EPOCHS] + [--label_smoothing LABEL_SMOOTHING] [--lr LR] [--alpha ALPHA] [--batch-size BATCH_SIZE] [--loops LOOPS] [--segment_size SEGMENT_SIZE] [--save_interval SAVE_INTERVAL] [--max-revision MAX_REVISION] @@ -26,6 +27,8 @@ optional arguments: --no-cuda disables CUDA training --epochs EPOCHS number of epochs in each learning loop iteration (default : 1) + --label_smoothing LABEL_SMOOTHING + label smoothing in cross entropy loss (default : 0.2) --lr LR base model learning rate (default : 0.001) --alpha ALPHA alpha in RMSprop (default : 0.9) --batch-size BATCH_SIZE diff --git a/examples/mnist_add/main.py b/examples/mnist_add/main.py index b6b3173..c992c4e 100644 --- a/examples/mnist_add/main.py +++ b/examples/mnist_add/main.py @@ -42,6 +42,12 @@ def main(): default=1, help="number of epochs in each learning loop iteration (default : 1)", ) + parser.add_argument( + "--label-smoothing", + type=float, + default=0.2, + help="label smoothing in cross entropy loss (default : 0.2)", + ) parser.add_argument( "--lr", type=float, default=3e-4, help="base model learning rate (default : 0.0003)" ) @@ -57,10 +63,7 @@ def main(): ) parser.add_argument("--save_interval", type=int, default=1, help="save interval (default : 1)") parser.add_argument( - "--max-revision", - type=int, - default=-1, - help="maximum revision in reasoner (default : -1)", + "--max-revision", type=int, default=-1, help="maximum revision in reasoner (default : -1)" ) parser.add_argument( "--require-more-revision", @@ -91,7 +94,7 @@ def main(): # Build necessary components for BasicNN cls = LeNet5(num_classes=10) - loss_fn = nn.CrossEntropyLoss(label_smoothing=0.2) + loss_fn = nn.CrossEntropyLoss(label_smoothing=args.label_smoothing) optimizer = RMSprop(cls.parameters(), lr=args.lr, alpha=args.alpha) use_cuda = not args.no_cuda and torch.cuda.is_available() device = torch.device("cuda" if use_cuda else "cpu") From 5d600b9ced86d377293ba36b489fab3aff4f4026 Mon Sep 17 00:00:00 2001 From: troyyyyy Date: Wed, 17 Jan 2024 18:15:34 +0800 Subject: [PATCH 2/8] [MNT] remove lambdalearn converter --- ablkit/data/data_converter.py | 141 ------------ ablkit/learning/model_converter.py | 211 ------------------ .../mnist_add/main_with_model_converter.py | 160 ------------- 3 files changed, 512 deletions(-) delete mode 100644 ablkit/data/data_converter.py delete mode 100644 ablkit/learning/model_converter.py delete mode 100644 examples/mnist_add/main_with_model_converter.py diff --git a/ablkit/data/data_converter.py b/ablkit/data/data_converter.py deleted file mode 100644 index 6841ffe..0000000 --- a/ablkit/data/data_converter.py +++ /dev/null @@ -1,141 +0,0 @@ -from typing import Any, Tuple - -from ablkit.utils import tab_data_to_tuple -from .structures.list_data import ListData -from lambdaLearn.Base.TabularMixin import TabularMixin - - -class DataConverter: - """ - This class provides functionality to convert LambdaLearn data to ABLkit data. - """ - - def __init__(self) -> None: - pass - - def convert_lambdalearn_to_tuple( - self, dataset: TabularMixin, reasoning_result: Any - ) -> Tuple[Tuple, Tuple, Tuple, Tuple]: - """ - Convert a lambdalearn dataset to a tuple of tuples (label_data, train_data, valid_data, test_data), # noqa: E501 - each containing (data, label, reasoning_result). - - Parameters - ---------- - dataset : TabularMixin - The LambdaLearn dataset to be converted. - reasoning_result : Any - The reasoning result of the dataset. - Returns - ------- - Tuple[Tuple, Tuple, Tuple, Tuple] - A tuple of (label_data, train_data, valid_data, test_data), where each element is - a tuple of (data, label, reasoning_result). - """ - - if not isinstance(dataset, TabularMixin): - raise NotImplementedError( - "Only support converting the datasets that are instances of TabularMixin. " - + "Please refer to the documentation and manually convert the dataset into a tuple." - ) - - label_data = tab_data_to_tuple( - dataset.labeled_X, dataset.labeled_y, reasoning_result=reasoning_result - ) - train_data = tab_data_to_tuple( - dataset.unlabeled_X, dataset.unlabeled_y, reasoning_result=reasoning_result - ) - valid_data = tab_data_to_tuple( - dataset.valid_X, dataset.valid_y, reasoning_result=reasoning_result - ) - test_data = tab_data_to_tuple( - dataset.test_X, dataset.test_y, reasoning_result=reasoning_result - ) - - return label_data, train_data, valid_data, test_data - - def convert_lambdalearn_to_listdata( - self, dataset: TabularMixin, reasoning_result: Any - ) -> Tuple[ListData, ListData, ListData, ListData]: - """ - Convert a lambdalearn dataset to a tuple of ListData - (label_data_examples, train_data_examples, valid_data_examples, test_data_examples). - - Parameters - ---------- - dataset : TabularMixin - The LambdaLearn dataset to be converted. - reasoning_result : Any - The reasoning result of the dataset. - Returns - ------- - Tuple[ListData, ListData, ListData, ListData] - A tuple of ListData (label_data_examples, train_data_examples, valid_data_examples, test_data_examples) # noqa: E501 - """ - - if not isinstance(dataset, TabularMixin): - raise NotImplementedError( - "Only support converting the datasets that are instances of TabularMixin. " - + "Please refer to the documentation and manually convert the dataset " - + "into a ListData." - ) - - label_data, train_data, valid_data, test_data = self.convert_lambdalearn_to_tuple( - dataset, reasoning_result - ) - - if label_data is not None: - X, gt_pseudo_label, Y = label_data - label_data_examples = ListData(X=X, gt_pseudo_label=gt_pseudo_label, Y=Y) - if train_data is not None: - X, gt_pseudo_label, Y = train_data - train_data_examples = ListData(X=X, gt_pseudo_label=gt_pseudo_label, Y=Y) - if valid_data is not None: - X, gt_pseudo_label, Y = valid_data - valid_data_examples = ListData(X=X, gt_pseudo_label=gt_pseudo_label, Y=Y) - if test_data is not None: - X, gt_pseudo_label, Y = test_data - test_data_examples = ListData(X=X, gt_pseudo_label=gt_pseudo_label, Y=Y) - - return label_data_examples, train_data_examples, valid_data_examples, test_data_examples - - -if __name__ == "__main__": - from lambdaLearn.Dataset.Tabular.BreastCancer import BreastCancer - - breast_dataset = BreastCancer(labeled_size=0.1, stratified=True, shuffle=True) - dataconverter = DataConverter() - - label_data, train_data, valid_data, test_data = dataconverter.convert_lambdalearn_to_tuple( - breast_dataset, 0 - ) - print( - type(label_data).__name__, - type(train_data).__name__, - type(valid_data).__name__, - type(test_data).__name__, - ) - print(len(label_data)) - print(len(label_data[0]), len(label_data[1]), len(label_data[2])) - print(label_data[0][0], label_data[1][0], label_data[2][0]) - print() - - ( - label_data_examples, - train_data_examples, - valid_data_examples, - test_data_examples, - ) = dataconverter.convert_lambdalearn_to_listdata(breast_dataset, 0) - print( - type(label_data_examples).__name__, - type(train_data_examples).__name__, - type(valid_data_examples).__name__, - type(test_data_examples).__name__, - ) - print( - len(label_data_examples.X), - len(label_data_examples.gt_pseudo_label), - len(label_data_examples.Y), - ) - label_data_example = label_data_examples[0] - print(label_data_example.X, label_data_example.gt_pseudo_label, label_data_example.Y) diff --git a/ablkit/learning/model_converter.py b/ablkit/learning/model_converter.py deleted file mode 100644 index 13b36f1..0000000 --- a/ablkit/learning/model_converter.py +++ /dev/null @@ -1,211 +0,0 @@ -import torch -import copy -from typing import Any, Callable, List, Optional - -from .abl_model import ABLModel -from .basic_nn import BasicNN -from lambdaLearn.Base.DeepModelMixin import DeepModelMixin - - -class ModelConverter: - """ - This class provides functionality to convert LambdaLearn models to ABLkit models. - """ - - def __init__(self) -> None: - pass - - def convert_lambdalearn_to_ablmodel( - self, - lambdalearn_model, - loss_fn: torch.nn.Module, - optimizer_dict: dict, - scheduler_dict: Optional[dict] = None, - device: Optional[torch.device] = None, - batch_size: int = 32, - num_epochs: int = 1, - stop_loss: Optional[float] = 0.0001, - num_workers: int = 0, - save_interval: Optional[int] = None, - save_dir: Optional[str] = None, - train_transform: Callable[..., Any] = None, - test_transform: Callable[..., Any] = None, - collate_fn: Callable[[List[Any]], Any] = None, - ): - """ - Convert a lambdalearn model to an ABLModel. If the lambdalearn model is an instance of - DeepModelMixin, its network will be used as the model of BasicNN. Otherwise, the lambdalearn - model should implement ``fit`` and ``predict`` methods. - - Parameters - ---------- - lambdalearn_model : Union[DeepModelMixin, Any] - The LambdaLearn model to be converted. - loss_fn : torch.nn.Module - The loss function used for training. - optimizer_dict : dict - The dict contains necessary parameters to construct a optimizer used for training. - The optimizer class is specified by the ``optimizer`` key. - scheduler_dict : dict, optional - The dict contains necessary parameters to construct a learning rate scheduler used - for training, which will be called at the end of each run of the ``fit`` method. - The scheduler class is specified by the ``scheduler`` key. It should implement the - ``step`` method. Defaults to None. - device : torch.device, optional - The device on which the model will be trained or used for prediction, - Defaults to torch.device("cpu"). - batch_size : int, optional - The batch size used for training. Defaults to 32. - num_epochs : int, optional - The number of epochs used for training. Defaults to 1. - stop_loss : float, optional - The loss value at which to stop training. Defaults to 0.0001. - num_workers : int - The number of workers used for loading data. Defaults to 0. - save_interval : int, optional - The model will be saved every ``save_interval`` epoch during training. Defaults to None. - save_dir : str, optional - The directory in which to save the model during training. Defaults to None. - train_transform : Callable[..., Any], optional - A function/transform that takes an object and returns a transformed version used - in the `fit` and `train_epoch` methods. Defaults to None. - test_transform : Callable[..., Any], optional - A function/transform that takes an object and returns a transformed version in the - `predict`, `predict_proba` and `score` methods. Defaults to None. - collate_fn : Callable[[List[T]], Any], optional - The function used to collate data. Defaults to None. - - Returns - ------- - ABLModel - The converted ABLModel instance. - """ - if isinstance(lambdalearn_model, DeepModelMixin): - base_model = self.convert_lambdalearn_to_basicnn( - lambdalearn_model, - loss_fn, - optimizer_dict, - scheduler_dict, - device, - batch_size, - num_epochs, - stop_loss, - num_workers, - save_interval, - save_dir, - train_transform, - test_transform, - collate_fn, - ) - return ABLModel(base_model) - - if not (hasattr(lambdalearn_model, "fit") and hasattr(lambdalearn_model, "predict")): - raise NotImplementedError( - "The lambdalearn_model should be an instance of DeepModelMixin, or implement " - + "fit and predict methods." - ) - - return ABLModel(lambdalearn_model) - - def convert_lambdalearn_to_basicnn( - self, - lambdalearn_model: DeepModelMixin, - loss_fn: torch.nn.Module, - optimizer_dict: dict, - scheduler_dict: Optional[dict] = None, - device: Optional[torch.device] = None, - batch_size: int = 32, - num_epochs: int = 1, - stop_loss: Optional[float] = 0.0001, - num_workers: int = 0, - save_interval: Optional[int] = None, - save_dir: Optional[str] = None, - train_transform: Callable[..., Any] = None, - test_transform: Callable[..., Any] = None, - collate_fn: Callable[[List[Any]], Any] = None, - ): - """ - Convert a lambdalearn model to a BasicNN. If the lambdalearn model is an instance of - DeepModelMixin, its network will be used as the model of BasicNN. - - Parameters - ---------- - lambdalearn_model : Union[DeepModelMixin, Any] - The LambdaLearn model to be converted. - loss_fn : torch.nn.Module - The loss function used for training. - optimizer_dict : dict - The dict contains necessary parameters to construct a optimizer used for training. - scheduler_dict : dict, optional - The dict contains necessary parameters to construct a learning rate scheduler used - for training, which will be called at the end of each run of the ``fit`` method. - The scheduler class is specified by the ``scheduler`` key. It should implement the - ``step`` method. Defaults to None. - device : torch.device, optional - The device on which the model will be trained or used for prediction, - Defaults to torch.device("cpu"). - batch_size : int, optional - The batch size used for training. Defaults to 32. - num_epochs : int, optional - The number of epochs used for training. Defaults to 1. - stop_loss : float, optional - The loss value at which to stop training. Defaults to 0.0001. - num_workers : int - The number of workers used for loading data. Defaults to 0. - save_interval : int, optional - The model will be saved every ``save_interval`` epoch during training. Defaults to None. - save_dir : str, optional - The directory in which to save the model during training. Defaults to None. - train_transform : Callable[..., Any], optional - A function/transform that takes an object and returns a transformed version used - in the `fit` and `train_epoch` methods. Defaults to None. - test_transform : Callable[..., Any], optional - A function/transform that takes an object and returns a transformed version in the - `predict`, `predict_proba` and `score` methods. Defaults to None. - collate_fn : Callable[[List[T]], Any], optional - The function used to collate data. Defaults to None. - - Returns - ------- - BasicNN - The converted BasicNN instance. - """ - if isinstance(lambdalearn_model, DeepModelMixin): - if not isinstance(lambdalearn_model.network, torch.nn.Module): - raise NotImplementedError( - "Expected lambdalearn_model.network to be a torch.nn.Module, " - + f"but got {type(lambdalearn_model.network)}" - ) - # Only use the network part and device of the lambdalearn model - network = copy.deepcopy(lambdalearn_model.network) - optimizer_class = optimizer_dict["optimizer"] - optimizer_dict.pop("optimizer") - optimizer = optimizer_class(network.parameters(), **optimizer_dict) - if scheduler_dict is not None: - scheduler_class = scheduler_dict["scheduler"] - scheduler_dict.pop("scheduler") - scheduler = scheduler_class(optimizer, **scheduler_dict) - else: - scheduler = None - device = lambdalearn_model.device if device is None else device - base_model = BasicNN( - model=network, - loss_fn=loss_fn, - optimizer=optimizer, - scheduler=scheduler, - device=device, - batch_size=batch_size, - num_epochs=num_epochs, - stop_loss=stop_loss, - num_workers=num_workers, - save_interval=save_interval, - save_dir=save_dir, - train_transform=train_transform, - test_transform=test_transform, - collate_fn=collate_fn, - ) - return base_model - else: - raise NotImplementedError( - "The lambdalearn_model should be an instance of DeepModelMixin." - ) diff --git a/examples/mnist_add/main_with_model_converter.py b/examples/mnist_add/main_with_model_converter.py deleted file mode 100644 index 9e3b251..0000000 --- a/examples/mnist_add/main_with_model_converter.py +++ /dev/null @@ -1,160 +0,0 @@ -import argparse -import os.path as osp - -from torch import nn -from torch.optim import RMSprop, lr_scheduler - -from lambdaLearn.Algorithm.AbductiveLearning.bridge import SimpleBridge -from lambdaLearn.Algorithm.AbductiveLearning.data.evaluation import ReasoningMetric, SymbolAccuracy -from lambdaLearn.Algorithm.AbductiveLearning.learning import ABLModel -from lambdaLearn.Algorithm.AbductiveLearning.learning.model_converter import ModelConverter -from lambdaLearn.Algorithm.AbductiveLearning.reasoning import GroundKB, KBBase, PrologKB, Reasoner -from lambdaLearn.Algorithm.AbductiveLearning.utils import ABLLogger, print_log -from lambdaLearn.Algorithm.SemiSupervised.Classification.FixMatch import FixMatch - -from datasets import get_dataset -from models.nn import LeNet5 - - -class AddKB(KBBase): - def __init__(self, pseudo_label_list=list(range(10))): - super().__init__(pseudo_label_list) - - def logic_forward(self, nums): - return sum(nums) - - -class AddGroundKB(GroundKB): - def __init__(self, pseudo_label_list=list(range(10)), GKB_len_list=[2]): - super().__init__(pseudo_label_list, GKB_len_list) - - def logic_forward(self, nums): - return sum(nums) - - -def main(): - parser = argparse.ArgumentParser(description="MNIST Addition example") - parser.add_argument( - "--no-cuda", action="store_true", default=False, help="disables CUDA training" - ) - parser.add_argument( - "--epochs", - type=int, - default=1, - help="number of epochs in each learning loop iteration (default : 1)", - ) - parser.add_argument( - "--lr", type=float, default=3e-4, help="base model learning rate (default : 0.0003)" - ) - parser.add_argument("--alpha", type=float, default=0.9, help="alpha in RMSprop (default : 0.9)") - parser.add_argument( - "--batch-size", type=int, default=32, help="base model batch size (default : 32)" - ) - parser.add_argument( - "--loops", type=int, default=2, help="number of loop iterations (default : 2)" - ) - parser.add_argument( - "--segment_size", type=int, default=0.01, help="segment size (default : 0.01)" - ) - parser.add_argument("--save_interval", type=int, default=1, help="save interval (default : 1)") - parser.add_argument( - "--max-revision", - type=int, - default=-1, - help="maximum revision in reasoner (default : -1)", - ) - parser.add_argument( - "--require-more-revision", - type=int, - default=0, - help="require more revision in reasoner (default : 0)", - ) - kb_type = parser.add_mutually_exclusive_group() - kb_type.add_argument( - "--prolog", action="store_true", default=False, help="use PrologKB (default: False)" - ) - kb_type.add_argument( - "--ground", action="store_true", default=False, help="use GroundKB (default: False)" - ) - - args = parser.parse_args() - - # Build logger - print_log("Abductive Learning on the MNIST Addition example.", logger="current") - - # -- Working with Data ------------------------------ - print_log("Working with Data.", logger="current") - train_data = get_dataset(train=True, get_pseudo_label=True) - test_data = get_dataset(train=False, get_pseudo_label=True) - - # -- Building the Learning Part --------------------- - print_log("Building the Learning Part.", logger="current") - - # Build necessary components for BasicNN - model = FixMatch( - network=LeNet5(), - threshold=0.95, - lambda_u=1.0, - mu=7, - T=0.5, - epoch=1, - num_it_epoch=2**20, - num_it_total=2**20, - device="cuda", - ) - - loss_fn = nn.CrossEntropyLoss(label_smoothing=0.2) - optimizer_dict = dict(optimizer=RMSprop, lr=0.0003, alpha=0.9) - scheduler_dict = dict( - scheduler=lr_scheduler.OneCycleLR, max_lr=0.0003, pct_start=0.15, total_steps=200 - ) - - converter = ModelConverter() - base_model = converter.convert_lambdalearn_to_basicnn( - model, loss_fn=loss_fn, optimizer_dict=optimizer_dict, scheduler_dict=scheduler_dict - ) - - # Build ABLModel - model = ABLModel(base_model) - - # -- Building the Reasoning Part -------------------- - print_log("Building the Reasoning Part.", logger="current") - - # Build knowledge base - if args.prolog: - kb = PrologKB(pseudo_label_list=list(range(10)), pl_file="add.pl") - elif args.ground: - kb = AddGroundKB() - else: - kb = AddKB() - - # Create reasoner - reasoner = Reasoner( - kb, max_revision=args.max_revision, require_more_revision=args.require_more_revision - ) - - # -- Building Evaluation Metrics -------------------- - print_log("Building Evaluation Metrics.", logger="current") - metric_list = [SymbolAccuracy(prefix="mnist_add"), ReasoningMetric(kb=kb, prefix="mnist_add")] - - # -- Bridging Learning and Reasoning ---------------- - print_log("Bridge Learning and Reasoning.", logger="current") - bridge = SimpleBridge(model, reasoner, metric_list) - - # Retrieve the directory of the Log file and define the directory for saving the model weights. - log_dir = ABLLogger.get_current_instance().log_dir - weights_dir = osp.join(log_dir, "weights") - - # Train and Test - bridge.train( - train_data, - loops=args.loops, - segment_size=args.segment_size, - save_interval=args.save_interval, - save_dir=weights_dir, - ) - bridge.test(test_data) - - -if __name__ == "__main__": - main() From 1e25084c43c9f4058926a4ebc2b5253d4091a4bf Mon Sep 17 00:00:00 2001 From: Gao Enhao Date: Thu, 18 Jan 2024 11:04:48 +0800 Subject: [PATCH 3/8] [MNT] update mnist add experiment --- docs/Examples/MNISTAdd.rst | 8 ++++---- examples/mnist_add/README.md | 8 ++++---- examples/mnist_add/mnist_add.ipynb | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/Examples/MNISTAdd.rst b/docs/Examples/MNISTAdd.rst index 70d92ca..aa88d7b 100644 --- a/docs/Examples/MNISTAdd.rst +++ b/docs/Examples/MNISTAdd.rst @@ -387,11 +387,11 @@ We present the results of ABL as follows, which include the reasoning accuracy ( +--------------+----------+------------------------------+ | Method | Accuracy | Time to achieve the Acc. (s) | +==============+==========+==============================+ - | NeurASP | 0.964 | 354 | + | NeurASP | 0.962 | 966 | +--------------+----------+------------------------------+ - | DeepProbLog | 0.965 | 1965 | + | DeepProbLog | 0.971 | 2045 | +--------------+----------+------------------------------+ - | DeepStochLog | 0.975 | 727 | + | DeepStochLog | 0.975 | 257 | +--------------+----------+------------------------------+ - | ABL | 0.980 | 42 | + | ABL | 0.981 | 47 | +--------------+----------+------------------------------+ diff --git a/examples/mnist_add/README.md b/examples/mnist_add/README.md index 390c265..8aebe39 100644 --- a/examples/mnist_add/README.md +++ b/examples/mnist_add/README.md @@ -58,7 +58,7 @@ We present the results of ABL as follows, which include the reasoning accuracy ( | Method | Accuracy | Time to achieve the Acc. (s) | | :----------: | :------: | :--------------------------: | -| NeurASP | 0.964 | 354 | -| DeepProbLog | 0.965 | 1965 | -| DeepStochLog | 0.975 | 727 | -| ABL | 0.980 | 42 | \ No newline at end of file +| NeurASP | 0.962 | 966 | +| DeepProbLog | 0.971 | 2045 | +| DeepStochLog | 0.975 | 257 | +| ABL | 0.981 | 47 | \ No newline at end of file diff --git a/examples/mnist_add/mnist_add.ipynb b/examples/mnist_add/mnist_add.ipynb index 502dd50..d654f81 100644 --- a/examples/mnist_add/mnist_add.ipynb +++ b/examples/mnist_add/mnist_add.ipynb @@ -481,10 +481,10 @@ "source": [ "| Method | Accuracy | Time to achieve the Acc. (s) |\n", "| :----------: | :------: | :--------------------------: |\n", - "| NeurASP | 0.964 | 354 |\n", - "| DeepProbLog | 0.965 | 1965 |\n", - "| DeepStochLog | 0.975 | 727 |\n", - "| ABL | 0.980 | 42 |" + "| NeurASP | 0.962 | 966 |\n", + "| DeepProbLog | 0.971 | 2045 |\n", + "| DeepStochLog | 0.975 | 257 |\n", + "| ABL | 0.981 | 47 |" ] } ], From 31b8516766615d2441f672d17a9f58960f593fcb Mon Sep 17 00:00:00 2001 From: Gao Enhao Date: Thu, 18 Jan 2024 14:33:58 +0800 Subject: [PATCH 4/8] [MNT] modify doc --- docs/Intro/Basics.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/Intro/Basics.rst b/docs/Intro/Basics.rst index 3921995..5cb8384 100644 --- a/docs/Intro/Basics.rst +++ b/docs/Intro/Basics.rst @@ -28,8 +28,8 @@ such as ``SymbolAccuracy`` and ``ReasoningMetric`` (both specialized metrics inherited from the ``BaseMetric`` class), for evaluating performance from a data perspective. -:blue-bold:`Learning` part focuses on the construction, training, and -prediction of machine learning models. The ``ABLModel`` class is the +:blue-bold:`Learning` part focuses on creating, training, and utilizing +machine learning models. The ``ABLModel`` class is the central class that encapsulates the machine learning model. This class is compatible with various frameworks, including those based on scikit-learn or PyTorch neural networks constructed by the ``BasicNN`` class. From 98ca808b16713b20a1a4f7a5638cd90a50ea7d0b Mon Sep 17 00:00:00 2001 From: Gao Enhao Date: Fri, 19 Jan 2024 20:44:18 +0800 Subject: [PATCH 5/8] [MNT] modify table style of result of mnist --- docs/Examples/MNISTAdd.rst | 55 ++++++++++++++++++++++-------- examples/mnist_add/README.md | 37 ++++++++++++++++---- examples/mnist_add/mnist_add.ipynb | 47 +++++++++++++++++++++---- 3 files changed, 112 insertions(+), 27 deletions(-) diff --git a/docs/Examples/MNISTAdd.rst b/docs/Examples/MNISTAdd.rst index aa88d7b..fd24a8d 100644 --- a/docs/Examples/MNISTAdd.rst +++ b/docs/Examples/MNISTAdd.rst @@ -381,17 +381,44 @@ We present the results of ABL as follows, which include the reasoning accuracy ( - `DeepProbLog `_: An extension of ProbLog by introducing neural predicates in Probabilistic Logic Programming; - `DeepStochLog `_: A neural-symbolic framework based on stochastic logic program. -.. table:: - :class: centered - - +--------------+----------+------------------------------+ - | Method | Accuracy | Time to achieve the Acc. (s) | - +==============+==========+==============================+ - | NeurASP | 0.962 | 966 | - +--------------+----------+------------------------------+ - | DeepProbLog | 0.971 | 2045 | - +--------------+----------+------------------------------+ - | DeepStochLog | 0.975 | 257 | - +--------------+----------+------------------------------+ - | ABL | 0.981 | 47 | - +--------------+----------+------------------------------+ +.. raw:: html + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
MethodAccuracyTime to achieve the Acc. (s)
NeurASP96.2966
DeepProbLog97.12045
DeepStochLog97.5257
ABL98.147
diff --git a/examples/mnist_add/README.md b/examples/mnist_add/README.md index 8aebe39..8642523 100644 --- a/examples/mnist_add/README.md +++ b/examples/mnist_add/README.md @@ -56,9 +56,34 @@ We present the results of ABL as follows, which include the reasoning accuracy ( - [**DeepProbLog**](https://github.com/ML-KULeuven/deepproblog): An extension of ProbLog by introducing neural predicates in Probabilistic Logic Programming; - [**DeepStochLog**](https://github.com/ML-KULeuven/deepstochlog): A neural-symbolic framework based on stochastic logic program. -| Method | Accuracy | Time to achieve the Acc. (s) | -| :----------: | :------: | :--------------------------: | -| NeurASP | 0.962 | 966 | -| DeepProbLog | 0.971 | 2045 | -| DeepStochLog | 0.975 | 257 | -| ABL | 0.981 | 47 | \ No newline at end of file + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
MethodAccuracyTime to achieve the Acc. (s)
NeurASP96.2966
DeepProbLog97.12045
DeepStochLog97.5257
ABL98.147
\ No newline at end of file diff --git a/examples/mnist_add/mnist_add.ipynb b/examples/mnist_add/mnist_add.ipynb index d654f81..21a2f0d 100644 --- a/examples/mnist_add/mnist_add.ipynb +++ b/examples/mnist_add/mnist_add.ipynb @@ -479,12 +479,45 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "| Method | Accuracy | Time to achieve the Acc. (s) |\n", - "| :----------: | :------: | :--------------------------: |\n", - "| NeurASP | 0.962 | 966 |\n", - "| DeepProbLog | 0.971 | 2045 |\n", - "| DeepStochLog | 0.975 | 257 |\n", - "| ABL | 0.981 | 47 |" + "\n", + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + "\n", + "\n", + " \n", + " \n", + " \n", + "\n", + "\n", + " \n", + " \n", + " \n", + "\n", + "\n", + " \n", + " \n", + "\n", + "\n", + "\n", + "
MethodAccuracyTime to achieve the Acc. (s)
NeurASP96.2966
DeepProbLog97.12045
DeepStochLog97.5257
ABL98.147
\n" ] } ], @@ -504,7 +537,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.13" + "version": "3.8.18" }, "orig_nbformat": 4, "vscode": { From 3619df4780229aa248a5091786f1ce2709d78e97 Mon Sep 17 00:00:00 2001 From: Gao Enhao Date: Fri, 19 Jan 2024 22:11:32 +0800 Subject: [PATCH 6/8] [FIX] fix typo in mnist experiment --- docs/Examples/HWF.rst | 2 +- docs/Examples/MNISTAdd.rst | 12 ++++++------ examples/hwf/hwf.ipynb | 2 +- examples/mnist_add/mnist_add.ipynb | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/Examples/HWF.rst b/docs/Examples/HWF.rst index ae3b00b..cb0cb3e 100644 --- a/docs/Examples/HWF.rst +++ b/docs/Examples/HWF.rst @@ -192,7 +192,7 @@ sklearn-style interface. num_epochs=3, ) -``BasicNN`` offers methods like ``predict`` and ``predict_prob``, which +``BasicNN`` offers methods like ``predict`` and ``predict_proba``, which are used to predict the class index and the probabilities of each class for images. As shown below: diff --git a/docs/Examples/MNISTAdd.rst b/docs/Examples/MNISTAdd.rst index fd24a8d..5562dce 100644 --- a/docs/Examples/MNISTAdd.rst +++ b/docs/Examples/MNISTAdd.rst @@ -163,7 +163,7 @@ model with a sklearn-style interface. num_epochs=1, ) -``BasicNN`` offers methods like ``predict`` and ``predict_prob``, which +``BasicNN`` offers methods like ``predict`` and ``predict_proba``, which are used to predict the class index and the probabilities of each class for images. As shown below: @@ -385,7 +385,7 @@ We present the results of ABL as follows, which include the reasoning accuracy (