diff --git a/docs/Examples/HWF.rst b/docs/Examples/HWF.rst index 5d02d79..8bd403c 100644 --- a/docs/Examples/HWF.rst +++ b/docs/Examples/HWF.rst @@ -1,5 +1,467 @@ Handwritten Formula (HWF) ========================= -.. contents:: Table of Contents +Below shows an implementation of `Handwritten +Formula `__. In this task. In this +task, handwritten images of decimal formulas and their computed results +are given, alongwith a domain knowledge base containing information on +how to compute the decimal formula. The task is to recognize the symbols +(which can be digits or operators ‘+’, ‘-’, ‘×’, ‘÷’) of handwritten +images and accurately determine their results. +Intuitively, we first use a machine learning model (learning part) to +convert the input images to symbols (we call them pseudo-labels), and +then use the knowledge base (reasoning part) to calculate the results of +these symbols. Since we do not have ground-truth of the symbols, in +Abductive Learning, the reasoning part will leverage domain knowledge +and revise the initial symbols yielded by the learning part through +abductive reasoning. This process enables us to further update the +machine learning model. + +.. code:: ipython3 + + # Import necessary libraries and modules + import os.path as osp + import numpy as np + import torch + import torch.nn as nn + import matplotlib.pyplot as plt + from examples.hwf.datasets import get_dataset + from examples.models.nn import SymbolNet + from abl.learning import ABLModel, BasicNN + from abl.reasoning import KBBase, Reasoner + from abl.evaluation import ReasoningMetric, SymbolMetric + from abl.utils import ABLLogger, print_log + from abl.bridge import SimpleBridge + +Working with Data +----------------- + +First, we get the training and testing datasets: + +.. code:: ipython3 + + train_data = get_dataset(train=True, get_pseudo_label=True) + test_data = get_dataset(train=False, get_pseudo_label=True) + +Both ``train_data`` and ``test_data`` have the same structures: tuples +with three components: X (list where each element is a list of images), +gt_pseudo_label (list where each element is a list of symbols, i.e., +pseudo-labels) and Y (list where each element is the computed result). +The length and structures of datasets are illustrated as follows. + +.. note:: + + ``gt_pseudo_label`` is only used to evaluate the performance of + the learning part but not to train the model. + +.. code:: ipython3 + + print(f"Both train_data and test_data consist of 3 components: X, gt_pseudo_label, Y") + print() + train_X, train_gt_pseudo_label, train_Y = train_data + print(f"Length of X, gt_pseudo_label, Y in train_data: " + + f"{len(train_X)}, {len(train_gt_pseudo_label)}, {len(train_Y)}") + test_X, test_gt_pseudo_label, test_Y = test_data + print(f"Length of X, gt_pseudo_label, Y in test_data: " + + f"{len(test_X)}, {len(test_gt_pseudo_label)}, {len(test_Y)}") + print() + + X_0, gt_pseudo_label_0, Y_0 = train_X[0], train_gt_pseudo_label[0], train_Y[0] + print(f"X is a {type(train_X).__name__}, " + + f"with each element being a {type(X_0).__name__} of {type(X_0[0]).__name__}.") + print(f"gt_pseudo_label is a {type(train_gt_pseudo_label).__name__}, " + + f"with each element being a {type(gt_pseudo_label_0).__name__} " + + f"of {type(gt_pseudo_label_0[0]).__name__}.") + print(f"Y is a {type(train_Y).__name__}, " + + f"with each element being a {type(Y_0).__name__}.") + + +Out: + .. code:: none + :class: code-out + + Both train_data and test_data consist of 3 components: X, gt_pseudo_label, Y + + Length of X, gt_pseudo_label, Y in train_data: 10000, 10000, 10000 + Length of X, gt_pseudo_label, Y in test_data: 2000, 2000, 2000 + + X is a list, with each element being a list of Tensor. + gt_pseudo_label is a list, with each element being a list of str. + Y is a list, with each element being a int. + + +The ith element of X, gt_pseudo_label, and Y together constitute the ith +data example. Here we use two of them (the 1001st and the 3001st) as +illstrations: + +.. code:: ipython3 + + X_1000, gt_pseudo_label_1000, Y_1000 = train_X[1000], train_gt_pseudo_label[1000], train_Y[1000] + print(f"X in the 1001st data example (a list of images):") + for i, x in enumerate(X_1000): + plt.subplot(1, len(X_1000), i+1) + plt.axis('off') + plt.imshow(x.numpy().transpose(1, 2, 0)) + plt.show() + print(f"gt_pseudo_label in the 1001st data example (a list of pseudo-labels): {gt_pseudo_label_1000}") + print(f"Y in the 1001st data example (the computed result): {Y_1000}") + print() + X_3000, gt_pseudo_label_3000, Y_3000 = train_X[3000], train_gt_pseudo_label[3000], train_Y[3000] + print(f"X in the 3001st data example (a list of images):") + for i, x in enumerate(X_3000): + plt.subplot(1, len(X_3000), i+1) + plt.axis('off') + plt.imshow(x.numpy().transpose(1, 2, 0)) + plt.show() + print(f"gt_pseudo_label in the 3001st data example (a list of pseudo-labels): {gt_pseudo_label_3000}") + print(f"Y in the 3001st data example (the computed result): {Y_3000}") + + +Out: + .. code:: none + :class: code-out + + X in the 1001st data example (a list of images): + + .. image:: ../img/hwf_dataset1.png + :width: 300px + + .. code:: none + :class: code-out + + gt_pseudo_label in the 1001st data example (a list of pseudo-labels): ['5', '-', '3'] + Y in the 1001st data example (the computed result): 2 + + .. code:: none + :class: code-out + + X in the 3001st data example (a list of images): + + .. image:: ../img/hwf_dataset2.png + :width: 500px + + .. code:: none + :class: code-out + + gt_pseudo_label in the 3001st data example (a list of pseudo-labels): ['4', '/', '6', '*', '5'] + Y in the 3001st data example (the computed result): 3.333333333333333 + + +.. note:: + + The symbols in the HWF dataset can be one of digits or operators + '+', '-', '×', '÷'. + + We may see that, in the 1001st data example, the length of the + formula is 3, while in the 3001st data example, the length of the + formula is 5. In the HWF dataset, the length of the formula varies from + 1 to 7. + +Building the Learning Part +-------------------------- + +To build the learning part, we need to first build a machine learning +base model. We use SymbolNet, and encapsulate it within a ``BasicNN`` +object to create the base model. ``BasicNN`` is a class that +encapsulates a PyTorch model, transforming it into a base model with an +sklearn-style interface. + +.. code:: ipython3 + + # class of symbol may be one of ['0', '1', ..., '9', '+', '-', '*', '/'], total of 14 classes + cls = SymbolNet(num_classes=14, image_size=(45, 45, 1)) + loss_fn = nn.CrossEntropyLoss() + optimizer = torch.optim.Adam(cls.parameters(), lr=0.001, betas=(0.9, 0.99)) + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + + base_model = BasicNN( + model=cls, + loss_fn=loss_fn, + optimizer=optimizer, + device=device, + batch_size=128, + num_epochs=3, + ) + +``BasicNN`` offers methods like ``predict`` and ``predict_prob``, which +are used to predict the class index and the probabilities of each class +for images. As shown below: + +.. code:: ipython3 + + data_instances = [torch.randn(1, 45, 45).to(device) for _ in range(32)] + pred_idx = base_model.predict(X=data_instances) + print(f"Predicted class index for a batch of 32 instances: " + + f"{type(pred_idx).__name__} with shape {pred_idx.shape}") + pred_prob = base_model.predict_proba(X=data_instances) + print(f"Predicted class probabilities for a batch of 32 instances: " + + f"{type(pred_prob).__name__} with shape {pred_prob.shape}") + + +Out: + .. code:: none + :class: code-out + + Predicted class index for a batch of 32 instances: ndarray with shape (32,) + Predicted class probabilities for a batch of 32 instances: ndarray with shape (32, 14) + + +However, the base model built above deals with instance-level data +(i.e., individual images), and can not directly deal with example-level +data (i.e., a list of images comprising the formula). Therefore, we wrap +the base model into ``ABLModel``, which enables the learning part to +train, test, and predict on example-level data. + +.. code:: ipython3 + + model = ABLModel(base_model) + +As an illustration, consider this example of training on example-level +data using the ``predict`` method in ``ABLModel``. In this process, the +method accepts data examples as input and outputs the class labels and +the probabilities of each class for all instances within these data +examples. + +.. code:: ipython3 + + from abl.structures import ListData + # ListData is a data structure provided by ABL-Package that can be used to organize data examples + data_examples = ListData() + # We use the first 1001st and 3001st data examples in the training set as an illustration + data_examples.X = [X_1000, X_3000] + data_examples.gt_pseudo_label = [gt_pseudo_label_1000, gt_pseudo_label_3000] + data_examples.Y = [Y_1000, Y_3000] + + # Perform prediction on the two data examples + # Remind that, in the 1001st data example, the length of the formula is 3, + # while in the 3001st data example, the length of the formula is 5. + pred_label, pred_prob = model.predict(data_examples)['label'], model.predict(data_examples)['prob'] + print(f"Predicted class labels for the 100 data examples: a list of length {len(pred_label)}, \n" + + f"the first element is a {type(pred_label[0]).__name__} of shape {pred_label[0].shape}, "+ + f"and the second element is a {type(pred_label[1]).__name__} of shape {pred_label[1].shape}.\n") + print(f"Predicted class probabilities for the 100 data examples: a list of length {len(pred_prob)}, \n" + f"the first element is a {type(pred_prob[0]).__name__} of shape {pred_prob[0].shape}, " + + f"and the second element is a {type(pred_prob[1]).__name__} of shape {pred_prob[1].shape}.") + + +Out: + .. code:: none + :class: code-out + + Predicted class labels for the 100 data examples: a list of length 2, + the first element is a ndarray of shape (3,), and the second element is a ndarray of shape (5,). + + Predicted class probabilities for the 100 data examples: a list of length 2, + the first element is a ndarray of shape (3, 14), and the second element is a ndarray of shape (5, 14). + + +Building the Reasoning Part +--------------------------- + +In the reasoning part, we first build a knowledge base which contain +information on how to perform addition operations. We build it by +creating a subclass of ``KBBase``. In the derived subclass, we +initialize the ``pseudo_label_list`` parameter specifying list of +possible pseudo-labels, and override the ``logic_forward`` function +defining how to perform (deductive) reasoning. + +.. code:: ipython3 + + class HwfKB(KBBase): + def __init__(self, pseudo_label_list=["1", "2", "3", "4", "5", "6", "7", "8", "9", "+", "-", "*", "/"]): + super().__init__(pseudo_label_list) + + def _valid_candidate(self, formula): + if len(formula) % 2 == 0: + return False + for i in range(len(formula)): + if i % 2 == 0 and formula[i] not in ["1", "2", "3", "4", "5", "6", "7", "8", "9"]: + return False + if i % 2 != 0 and formula[i] not in ["+", "-", "*", "/"]: + return False + return True + + # Implement the deduction function + def logic_forward(self, formula): + if not self._valid_candidate(formula): + return np.inf + return eval("".join(formula)) + + kb = HwfKB() + +The knowledge base can perform logical reasoning (both deductive +reasoning and abductive reasoning). Below is an example of performing +(deductive) reasoning, and users can refer to :ref:`Performing abductive +reasoning in the knowledge base ` for details of abductive reasoning. + +.. code:: ipython3 + + pseudo_label_example = ["1", "-", "2", "*", "5"] + reasoning_result = kb.logic_forward(pseudo_label_example) + print(f"Reasoning result of pseudo-label example {pseudo_label_example} is {reasoning_result}.") + + +Out: + .. code:: none + :class: code-out + + Reasoning result of pseudo-label example ['1', '-', '2', '*', '5'] is -9. + + +.. note:: + + In addition to building a knowledge base based on ``KBBase``, we + can also establish a knowledge base with a ground KB using ``GroundKB``. + The corresponding code can be found in the ``examples/hwf/main.py`` file. Those + interested are encouraged to examine it for further insights. + + Also, when building the knowledge base, we can also set the + ``max_err`` parameter during initialization, which is shown in the + ``examples/hwf/main.py`` file. This parameter specifies the upper tolerance limit + when comparing the similarity between a pseudo-label example’s reasoning + result and the ground truth during abductive reasoning, with a default + value of 1e-10. + +Then, we create a reasoner by instantiating the class ``Reasoner``. Due +to the indeterminism of abductive reasoning, there could be multiple +candidates compatible to the knowledge base. When this happens, reasoner +can minimize inconsistencies between the knowledge base and +pseudo-labels predicted by the learning part, and then return only one +candidate that has the highest consistency. + +.. code:: ipython3 + + reasoner = Reasoner(kb) + +.. note:: + + During creating reasoner, the definition of “consistency” can be + customized within the ``dist_func`` parameter. In the code above, we + employ a consistency measurement based on confidence, which calculates + the consistency between the data example and candidates based on the + confidence derived from the predicted probability. In ``examples/hwf/main.py``, we + provide options for utilizing other forms of consistency measurement. + + Also, during process of inconsistency minimization, we can + leverage `ZOOpt library `__ for + acceleration. Options for this are also available in ``examples/hwf/main.py``. Those + interested are encouraged to explore these features. + +Building Evaluation Metrics +--------------------------- + +Next, we set up evaluation metrics. These metrics will be used to +evaluate the model performance during training and testing. +Specifically, we use ``SymbolMetric`` and ``ReasoningMetric``, which are +used to evaluate the accuracy of the machine learning model’s +predictions and the accuracy of the final reasoning results, +respectively. + +.. code:: ipython3 + + metric_list = [SymbolMetric(prefix="hwf"), ReasoningMetric(kb=kb, prefix="hwf")] + +Bridge Learning and Reasoning +----------------------------- + +Now, the last step is to bridge the learning and reasoning part. We +proceed this step by creating an instance of ``SimpleBridge``. + +.. code:: ipython3 + + bridge = SimpleBridge(model, reasoner, metric_list) + +Perform training and testing by invoking the ``train`` and ``test`` +methods of ``SimpleBridge``. + +.. code:: ipython3 + + # Build logger + print_log("Abductive Learning on the HWF example.", logger="current") + log_dir = ABLLogger.get_current_instance().log_dir + weights_dir = osp.join(log_dir, "weights") + + bridge.train(train_data, train_data, loops=3, segment_size=1000, save_dir=weights_dir) + bridge.test(test_data) + +Out: + .. code:: none + :class: code-out + + abl - INFO - Abductive Learning on the HWF example. + abl - INFO - loop(train) [1/3] segment(train) [1/10] + abl - INFO - model loss: 0.00024 + abl - INFO - loop(train) [1/3] segment(train) [2/10] + abl - INFO - model loss: 0.00053 + abl - INFO - loop(train) [1/3] segment(train) [3/10] + abl - INFO - model loss: 0.00260 + abl - INFO - loop(train) [1/3] segment(train) [4/10] + abl - INFO - model loss: 0.00162 + abl - INFO - loop(train) [1/3] segment(train) [5/10] + abl - INFO - model loss: 0.00073 + abl - INFO - loop(train) [1/3] segment(train) [6/10] + abl - INFO - model loss: 0.00055 + abl - INFO - loop(train) [1/3] segment(train) [7/10] + abl - INFO - model loss: 0.00148 + abl - INFO - loop(train) [1/3] segment(train) [8/10] + abl - INFO - model loss: 0.00034 + abl - INFO - loop(train) [1/3] segment(train) [9/10] + abl - INFO - model loss: 0.00167 + abl - INFO - loop(train) [1/3] segment(train) [10/10] + abl - INFO - model loss: 0.00185 + abl - INFO - Evaluation start: loop(val) [1] + abl - INFO - Evaluation ended, hwf/character_accuracy: 1.000 hwf/reasoning_accuracy: 0.999 + abl - INFO - Saving model: loop(save) [1] + abl - INFO - Checkpoints will be saved to weights_dir/model_checkpoint_loop_1.pth + abl - INFO - loop(train) [2/3] segment(train) [1/10] + abl - INFO - model loss: 0.00219 + abl - INFO - loop(train) [2/3] segment(train) [2/10] + abl - INFO - model loss: 0.00069 + abl - INFO - loop(train) [2/3] segment(train) [3/10] + abl - INFO - model loss: 0.00013 + abl - INFO - loop(train) [2/3] segment(train) [4/10] + abl - INFO - model loss: 0.00013 + abl - INFO - loop(train) [2/3] segment(train) [5/10] + abl - INFO - model loss: 0.00248 + abl - INFO - loop(train) [2/3] segment(train) [6/10] + abl - INFO - model loss: 0.00010 + abl - INFO - loop(train) [2/3] segment(train) [7/10] + abl - INFO - model loss: 0.00020 + abl - INFO - loop(train) [2/3] segment(train) [8/10] + abl - INFO - model loss: 0.00076 + abl - INFO - loop(train) [2/3] segment(train) [9/10] + abl - INFO - model loss: 0.00061 + abl - INFO - loop(train) [2/3] segment(train) [10/10] + abl - INFO - model loss: 0.00117 + abl - INFO - Evaluation start: loop(val) [2] + abl - INFO - Evaluation ended, hwf/character_accuracy: 1.000 hwf/reasoning_accuracy: 1.000 + abl - INFO - Saving model: loop(save) [2] + abl - INFO - Checkpoints will be saved to weights_dir/model_checkpoint_loop_2.pth + abl - INFO - loop(train) [3/3] segment(train) [1/10] + abl - INFO - model loss: 0.00120 + abl - INFO - loop(train) [3/3] segment(train) [2/10] + abl - INFO - model loss: 0.00114 + abl - INFO - loop(train) [3/3] segment(train) [3/10] + abl - INFO - model loss: 0.00071 + abl - INFO - loop(train) [3/3] segment(train) [4/10] + abl - INFO - model loss: 0.00027 + abl - INFO - loop(train) [3/3] segment(train) [5/10] + abl - INFO - model loss: 0.00017 + abl - INFO - loop(train) [3/3] segment(train) [6/10] + abl - INFO - model loss: 0.00018 + abl - INFO - loop(train) [3/3] segment(train) [7/10] + abl - INFO - model loss: 0.00141 + abl - INFO - loop(train) [3/3] segment(train) [8/10] + abl - INFO - model loss: 0.00099 + abl - INFO - loop(train) [3/3] segment(train) [9/10] + abl - INFO - model loss: 0.00145 + abl - INFO - loop(train) [3/3] segment(train) [10/10] + abl - INFO - model loss: 0.00215 + abl - INFO - Evaluation start: loop(val) [3] + abl - INFO - Evaluation ended, hwf/character_accuracy: 1.000 hwf/reasoning_accuracy: 1.000 + abl - INFO - Saving model: loop(save) [3] + abl - INFO - Checkpoints will be saved to weights_dir/model_checkpoint_loop_2.pth + abl - INFO - Evaluation ended, hwf/character_accuracy: 0.996 hwf/reasoning_accuracy: 0.977 + +More concrete examples are available in ``examples/hwf/main.py`` and ``examples/hwf/hwf.ipynb``. \ No newline at end of file diff --git a/docs/Examples/MNISTAdd.rst b/docs/Examples/MNISTAdd.rst index 6c3bc4e..7b83de7 100644 --- a/docs/Examples/MNISTAdd.rst +++ b/docs/Examples/MNISTAdd.rst @@ -1,7 +1,7 @@ MNIST Addition ============== -This notebook shows an implementation of `MNIST +Below shows an implementation of `MNIST Addition `__. In this task, pairs of MNIST handwritten images and their sums are given, alongwith a domain knowledge base containing information on how to perform addition @@ -12,7 +12,7 @@ Intuitively, we first use a machine learning model (learning part) to convert the input images to digits (we call them pseudo-labels), and then use the knowledge base (reasoning part) to calculate the sum of these digits. Since we do not have ground-truth of the digits, in -abductive learning, the reasoning part will leverage domain knowledge +Abductive Learning, the reasoning part will leverage domain knowledge and revise the initial digits yielded by the learning part through abductive reasoning. This process enables us to further update the machine learning model. @@ -42,11 +42,12 @@ First, we get the training and testing datasets: train_data = get_dataset(train=True, get_pseudo_label=True) test_data = get_dataset(train=False, get_pseudo_label=True) -Both ``train_data`` and ``test_data`` have the same structures: tuples -with three components: X (list where each element is a pair of images), -gt_pseudo_label (list where each element is a pair of digits) and Y -(list where each element is the sum of each digit pair). The length and -structures of datasets are illustrated as follows. +``train_data`` and ``test_data`` share identical structures: +tuples with three components: X (list where each element is a +list of two images), gt_pseudo_label (list where each element +is a list of two digits, i.e., pseudo-labels) and Y (list where +each element is the sum of the two digits). The length and structures +of datasets are illustrated as follows. .. note:: @@ -55,24 +56,25 @@ structures of datasets are illustrated as follows. .. code:: ipython3 - def describe_structure(lst): - if not isinstance(lst, list): - return type(lst).__name__ - return [describe_structure(item) for item in lst] - print(f"Both train_data and test_data consist of 3 components: X, gt_pseudo_label, Y") print("\n") train_X, train_gt_pseudo_label, train_Y = train_data - print(f"Length of X, gt_pseudo_label, Y in train_data: {len(train_X)}, {len(train_gt_pseudo_label)}, {len(train_Y)}") + print(f"Length of X, gt_pseudo_label, Y in train_data: " + + f"{len(train_X)}, {len(train_gt_pseudo_label)}, {len(train_Y)}") test_X, test_gt_pseudo_label, test_Y = test_data - print(f"Length of X, gt_pseudo_label, Y in test_data: {len(test_X)}, {len(test_gt_pseudo_label)}, {len(test_Y)}") + print(f"Length of X, gt_pseudo_label, Y in test_data: " + + f"{len(test_X)}, {len(test_gt_pseudo_label)}, {len(test_Y)}") print("\n") - structure_X = describe_structure(train_X[0]) - structure_gt_pseudo_label = describe_structure(train_gt_pseudo_label[0]) - structure_Y = describe_structure(train_Y[0]) - print(f"Type of X: {type(train_X).__name__}, and type of each element in X: {structure_X}.") - print(f"Type of gt_pseudo_label: {type(train_gt_pseudo_label).__name__}, and type of each element in gt_pseudo_label: {structure_gt_pseudo_label}.") - print(f"Type of Y: {type(train_Y).__name__}, and type of each element in Y: {structure_Y}.") + + X_0, gt_pseudo_label_0, Y_0 = train_X[0], train_gt_pseudo_label[0], train_Y[0] + print(f"X is a {type(train_X).__name__}, " + + f"with each element being a {type(X_0).__name__} " + + f"of {len(X_0)} {type(X_0[0]).__name__}.") + print(f"gt_pseudo_label is a {type(train_gt_pseudo_label).__name__}, " + + f"with each element being a {type(gt_pseudo_label_0).__name__} " + + f"of {len(gt_pseudo_label_0)} {type(gt_pseudo_label_0[0]).__name__}.") + print(f"Y is a {type(train_Y).__name__}, " + + f"with each element being a {type(Y_0).__name__}.") Out: @@ -80,15 +82,13 @@ Out: :class: code-out Both train_data and test_data consist of 3 components: X, gt_pseudo_label, Y - - + Length of X, gt_pseudo_label, Y in train_data: 30000, 30000, 30000 Length of X, gt_pseudo_label, Y in test_data: 5000, 5000, 5000 - - - Type of X: list, and type of each element in X: ['Tensor', 'Tensor']. - Type of gt_pseudo_label: list, and type of each element in gt_pseudo_label: ['int', 'int']. - Type of Y: list, and type of each element in Y: int. + + X is a list, with each element being a list of 2 Tensor. + gt_pseudo_label is a list, with each element being a list of 2 int. + Y is a list, with each element being a int. The ith element of X, gt_pseudo_label, and Y together constitute the ith @@ -97,24 +97,24 @@ training set, we have: .. code:: ipython3 - first_X, first_gt_pseudo_label, first_Y = train_X[0], train_gt_pseudo_label[0], train_Y[0] - print(f"X in the first data example (a pair of images):") + X_0, gt_pseudo_label_0, Y_0 = train_X[0], train_gt_pseudo_label[0], train_Y[0] + print(f"X in the first data example (a list of two images):") plt.subplot(1,2,1) plt.axis('off') - plt.imshow(first_X[0].numpy().transpose(1, 2, 0)) + plt.imshow(X_0[0].numpy().transpose(1, 2, 0)) plt.subplot(1,2,2) plt.axis('off') - plt.imshow(first_X[1].numpy().transpose(1, 2, 0)) + plt.imshow(X_0[1].numpy().transpose(1, 2, 0)) plt.show() - print(f"gt_pseudo_label in the first data example (a pair of ground truth pseudo-labels): {first_gt_pseudo_label[0]}, {first_gt_pseudo_label[1]}") - print(f"Y in the first data example (their sum result): {first_Y}") + print(f"gt_pseudo_label in the first data example (a list of two ground truth pseudo-labels): {gt_pseudo_label_0}") + print(f"Y in the first data example (their sum result): {Y_0}") Out: .. code:: none :class: code-out - X in the first data example (a pair of images): + X in the first data example (a list of two images): .. image:: ../img/mnist_add_datasets.png :width: 400px @@ -122,7 +122,7 @@ Out: .. parsed-literal:: - gt_pseudo_label in the first data example (a pair of ground truth pseudo-labels): 7, 5 + gt_pseudo_label in the first data example (a list of two ground truth pseudo-labels): [7, 5] Y in the first data example (their sum result): 12 @@ -193,24 +193,32 @@ examples. from abl.structures import ListData # ListData is a data structure provided by ABL-Package that can be used to organize data examples - data_example = ListData() - data_example.X = first_X - data_example.gt_pseudo_label = first_gt_pseudo_label - data_example.Y = first_Y - - # Perform prediction on the first data examples - prediction_result = model.predict(data_example) - print(f"Predicted class labels for the first data example: np.array with shape {prediction_result['label'].shape}") - print(f"Predicted class probabilities for the first data example: np.array with shape {prediction_result['prob'].shape}") + data_examples = ListData() + # We use the first 100 data examples in the training set as an illustration + data_examples.X = train_X[:100] + data_examples.gt_pseudo_label = train_gt_pseudo_label[:100] + data_examples.Y = train_Y[:100] + + # Perform prediction on the 100 data examples + pred_label, pred_prob = model.predict(data_examples)['label'], model.predict(data_examples)['prob'] + print(f"Predicted class labels for the 100 data examples: \n" + + f"a list of length {len(pred_label)}, and each element is " + + f"a {type(pred_label[0]).__name__} of shape {pred_label[0].shape}.\n") + print(f"Predicted class probabilities for the 100 data examples: \n" + + f"a list of length {len(pred_prob)}, and each element is " + + f"a {type(pred_prob[0]).__name__} of shape {pred_prob[0].shape}.") Out: .. code:: none :class: code-out - Predicted class labels for the first data example: np.array with shape (2,) - Predicted class probabilities for the first data example: np.array with shape (2, 10) - + Predicted class labels for the 100 data examples: + a list of length 100, and each element is a ndarray of shape (2,). + + Predicted class probabilities for the 100 data examples: + a list of length 100, and each element is a ndarray of shape (2, 10). + Building the Reasoning Part --------------------------- @@ -282,7 +290,7 @@ candidate that has the highest consistency. confidence derived from the predicted probability. In ``examples/mnist_add/main.py``, we provide options for utilizing other forms of consistency measurement. - Also, during process of inconsistency minimization, one can leverage + Also, during process of inconsistency minimization, we can leverage `ZOOpt library `__ for acceleration. Options for this are also available in ``examples/mnist_add/main.py``. Those interested are encouraged to explore these features. diff --git a/docs/Intro/Reasoning.rst b/docs/Intro/Reasoning.rst index 2af667f..f0c9c87 100644 --- a/docs/Intro/Reasoning.rst +++ b/docs/Intro/Reasoning.rst @@ -45,12 +45,12 @@ and override the ``logic_forward`` function: - ``logic_forward`` defines how to perform (deductive) reasoning, i.e. matching each pseudo-label example to its reasoning result. -.. warnings:: +.. note:: Generally, the overridden function ``logic_forward`` provided by the user accepts - only one parameter, ``pseudo_label`` (a pseudo-label example), However, for certain + only one parameter, ``pseudo_label`` (a pseudo-label example). However, for certain scenarios, deductive reasoning in the knowledge base may necessitate information - from the input. When this happens, ``logic_forward`` can also accept two parameters: + from the input. In these scenarios, ``logic_forward`` can also accept two parameters: ``pseudo_label`` and ``x``. After that, other operations, including how to perform abductive diff --git a/docs/img/hwf_dataset1.png b/docs/img/hwf_dataset1.png new file mode 100644 index 0000000..58008f4 Binary files /dev/null and b/docs/img/hwf_dataset1.png differ diff --git a/docs/img/hwf_dataset2.png b/docs/img/hwf_dataset2.png new file mode 100644 index 0000000..0ca7038 Binary files /dev/null and b/docs/img/hwf_dataset2.png differ diff --git a/examples/hwf/README.md b/examples/hwf/README.md new file mode 100644 index 0000000..c10e94f --- /dev/null +++ b/examples/hwf/README.md @@ -0,0 +1,46 @@ +# MNIST Addition Example + +This example shows a simple implementation of [Handwritten Formula](https://arxiv.org/abs/2006.06649) task, where handwritten images of decimal formulas and their computed results are given, alongwith a domain knowledge base containing information on how to compute the decimal formula. The task is to recognize the symbols (which can be digits or operators '+', '-', '×', '÷') of handwritten images and accurately determine their results. + +## Run + +```bash +pip install -r requirements.txt +python main.py +``` + +## Usage + +```bash +usage: main.py [-h] [--no-cuda] [--epochs EPOCHS] [--lr LR] + [--weight-decay WEIGHT_DECAY] [--batch-size BATCH_SIZE] + [--loops LOOPS] [--segment_size SEGMENT_SIZE] + [--save_interval SAVE_INTERVAL] [--max-revision MAX_REVISION] + [--require-more-revision REQUIRE_MORE_REVISION] + [--ground] [--max-err MAX_ERR] + +MNIST Addition example + +optional arguments: + -h, --help show this help message and exit + --no-cuda disables CUDA training + --epochs EPOCHS number of epochs in each learning loop iteration + (default : 1) + --lr LR base learning rate (default : 0.001) + --weight-decay WEIGHT_DECAY + weight decay value (default : 0.03) + --batch-size BATCH_SIZE + batch size (default : 32) + --loops LOOPS number of loop iterations (default : 5) + --segment_size SEGMENT_SIZE + segment size (default : 1/3) + --save_interval SAVE_INTERVAL + save interval (default : 1) + --max-revision MAX_REVISION + maximum revision in reasoner (default : -1) + --require-more-revision REQUIRE_MORE_REVISION + require more revision in reasoner (default : 0) + --ground use GroundKB (default: False) + --max-err MAX_ERR max tolerance during abductive reasoning (default : 1e-10) + +``` diff --git a/examples/hwf/hwf.ipynb b/examples/hwf/hwf.ipynb index d3b3946..6ddd79d 100644 --- a/examples/hwf/hwf.ipynb +++ b/examples/hwf/hwf.ipynb @@ -4,82 +4,345 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Handwritten Formula (HWF)" + "# Handwritten Formula (HWF)\n", + "\n", + "This notebook shows an implementation of [Handwritten Formula](https://arxiv.org/abs/2006.06649). In this task. In this task, handwritten images of decimal formulas and their computed results are given, alongwith a domain knowledge base containing information on how to compute the decimal formula. The task is to recognize the symbols (which can be digits or operators '+', '-', '×', '÷') of handwritten images and accurately determine their results.\n", + "\n", + "Intuitively, we first use a machine learning model (learning part) to convert the input images to symbols (we call them pseudo-labels), and then use the knowledge base (reasoning part) to calculate the results of these symbols. Since we do not have ground-truth of the symbols, in Abductive Learning, the reasoning part will leverage domain knowledge and revise the initial symbols yielded by the learning part through abductive reasoning. This process enables us to further update the machine learning model." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "# Import necessary libraries and modules\n", + "import os.path as osp\n", + "import numpy as np\n", + "import torch\n", + "import torch.nn as nn\n", + "import matplotlib.pyplot as plt\n", + "from examples.hwf.datasets import get_dataset\n", + "from examples.models.nn import SymbolNet\n", + "from abl.learning import ABLModel, BasicNN\n", + "from abl.reasoning import KBBase, Reasoner\n", + "from abl.evaluation import ReasoningMetric, SymbolMetric\n", + "from abl.utils import ABLLogger, print_log\n", + "from abl.bridge import SimpleBridge" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "This example shows a simple implementation of [Handwritten Formula](https://arxiv.org/abs/2006.06649). In this task, xxx.\n", - "" + "First, we get the training and testing datasets:" ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ - "import torch\n", - "import numpy as np\n", - "import torch.nn as nn\n", - "import os.path as osp\n", - "\n", - "from abl.reasoning import Reasoner, KBBase\n", - "from abl.learning import BasicNN, ABLModel\n", - "from abl.bridge import SimpleBridge\n", - "from abl.evaluation import SymbolMetric, ReasoningMetric\n", - "from abl.utils import ABLLogger, print_log\n", + "train_data = get_dataset(train=True, get_pseudo_label=True)\n", + "test_data = get_dataset(train=False, get_pseudo_label=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Both `train_data` and `test_data` have the same structures: tuples with three components: X (list where each element is a list of images), gt_pseudo_label (list where each element is a list of symbols, i.e., pseudo-labels) and Y (list where each element is the computed result). The length and structures of datasets are illustrated as follows.\n", "\n", - "from examples.models.nn import SymbolNet\n", - "from examples.hwf.datasets import get_dataset" + "Note: ``gt_pseudo_label`` is only used to evaluate the performance of the learning part but not to train the model." ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Both train_data and test_data consist of 3 components: X, gt_pseudo_label, Y\n", + "\n", + "Length of X, gt_pseudo_label, Y in train_data: 10000, 10000, 10000\n", + "Length of X, gt_pseudo_label, Y in test_data: 2000, 2000, 2000\n", + "\n", + "X is a list, with each element being a list of Tensor.\n", + "gt_pseudo_label is a list, with each element being a list of str.\n", + "Y is a list, with each element being a int.\n" + ] + } + ], + "source": [ + "print(f\"Both train_data and test_data consist of 3 components: X, gt_pseudo_label, Y\")\n", + "print()\n", + "train_X, train_gt_pseudo_label, train_Y = train_data\n", + "print(f\"Length of X, gt_pseudo_label, Y in train_data: \" +\n", + " f\"{len(train_X)}, {len(train_gt_pseudo_label)}, {len(train_Y)}\")\n", + "test_X, test_gt_pseudo_label, test_Y = test_data\n", + "print(f\"Length of X, gt_pseudo_label, Y in test_data: \" +\n", + " f\"{len(test_X)}, {len(test_gt_pseudo_label)}, {len(test_Y)}\")\n", + "print()\n", + "\n", + "X_0, gt_pseudo_label_0, Y_0 = train_X[0], train_gt_pseudo_label[0], train_Y[0]\n", + "print(f\"X is a {type(train_X).__name__}, \" +\n", + " f\"with each element being a {type(X_0).__name__} of {type(X_0[0]).__name__}.\")\n", + "print(f\"gt_pseudo_label is a {type(train_gt_pseudo_label).__name__}, \" +\n", + " f\"with each element being a {type(gt_pseudo_label_0).__name__} \" +\n", + " f\"of {type(gt_pseudo_label_0[0]).__name__}.\")\n", + "print(f\"Y is a {type(train_Y).__name__}, \" +\n", + " f\"with each element being a {type(Y_0).__name__}.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The ith element of X, gt_pseudo_label, and Y together constitute the ith data example. Here we use two of them (the 1001st and the 3001st) as illstrations:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "X in the 1001st data example (a list of images):\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgMAAAClCAYAAADBAf6NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAS7ElEQVR4nO3d248cRxXH8ZrZdXyRRZQ32LWDQOId24SECPGGJloMUrj8myCZxdKKV8QliWPeESgoOyveokiW443XMzzAdv+qds7xmZ6Z7tmt7+ep1zPd0z2X6vI5dapG8/l8ngAAQLXGQ58AAAAYFp0BAAAqR2cAAIDK0RkAAKBydAYAAKgcnQEAACpHZwAAgMrRGQAAoHK70SfO/vO9ZvvVfNZs74xi/YmX81fN9rXRTvRlLwV9P1LK35PysXMHd9+xDzhr36s0GrXbzvxQRyd/d89xW4y/+Y/eX1O/u0BXfHdzq7bp3v6n85fN9m5qH9O29fns62yfX37nx832/GX+2KY8nn7abHvndmv8xsL99T0Yp1H2WPTeGhH57hIZAACgcnQGAACoXDhNsElDpxCioZrJ3vcXH2BcnLOG+fUxSRn87vO/ZLvodeu2lWZIKaWDOw/MxwAgymsDlbaHq7bV3v7XR9cWnpvu8eHdd7N9Do/bNrWv+8hk7377+tMnzbaVFkgpfr/r+75IZAAAgMrRGQAAoHKd0gTeaPlZake8a7jJC30PkRqYaIhdw/olGc3/WMJA3khPDe/MUnutGvpKyQ4jWS68JqtPA1gDqw0u2+pIJdmz2Yvs79vjG0vtn1I+Gv/Dt3/UPiBt9dHJ0+Lc7PTGpmgV12SvvadoyiCl/P3VSgnrOSn56ZpNIDIAAEDl6AwAAFA5OgMAAFQuPGZgsn+v/SOaqx4tznmMdvLcyPzsLHoaa3M4/ajZ1lyN5vvLx14Zl+3vM5JtO19mlZHoGIyrNW8jgG2k+fqyRO7L2VfN9pujmwv31zECJWu8Wdkefnjnh8320Umef48cuy/ablvjB1LKZyq03h9vJts+EBkAAKBydAYAAKhcOE1wePxJs/3wroRwjvMQjjdj3jkNfac01MJFi1MD5blYoSz99+g+Hus9iM5GCABdadvizZ735nhxaqBLm+fN7tpl4bUhZrLV8j+vfY68J+V9se+7IpEBAAAqR2cAAIDKhdMEGnbR1ECn8M4WhLutML83o6KGbTTs41cTtMc7S+3z8tkILx5j0bEuhJqMag0AWIaVAvVSulZY3msPrcXevPtGNAUxRLo5mhKJpDD6nnHw4usDAICq0RkAAKBynRYq0oknyvCOhoF0sYZhKgaWd3ExoMUpDa+awDreToqlFpQbXmKhIgBrYIXivVY7kjJIKb8nWOmAoSfcWQe9htG1NSxE17PL944DAIC1ojMAAEDl6AwAAFC5TmMGdBaqC7MuSblbNKc0xMxR0QUzrDyOl9+xjhed6VB574cufkG/DkCfrPYsWm7eZdZCz7qPtyx9zfnLr7PHIjPJlufc9/VwBwEAoHJ0BgAAqFynNIEXvnissxPuP1j476Whyw7XHYKxjhctQVz1dQBgGV3aEjP0HZwZ1ZrdtRRNtUauIVrC2CWtremRR9OPiyO2M85aKerS6fys2b41Wr5UcVncTQAAqBydAQAAKtcpTeCFd7Kwi8yQ54VwhqgmAAD4ojOlWu2+R491On+ZPbabFi/QFr0/WPcUXSwupZROZxKKH7+xcJ8ylB85h3IhOuvcvMWJ9Hz6QGQAAIDK0RkAAKBy4TSBtTCPG0Ya288z9wEA9KrLBDe6z8H+/WbbmmSopKkBL6z+bPai2b4po+q987TuKRdeR6L0eo+apfbavHOzeFUCkUXpUspTCEw6BAAANo7OAAAAlaMzAABA5cJjBqxZk7wFdx5/3s7C9LNvv9ts//azP2f7MGYAALZDtH3XEnMdH+bly1U0F397fCP0PKuc8PmsXTTIK9fLr3Xx/uUxdNZBXTjOW3TI+veyzLDvGWaJDAAAUDk6AwAAVK5TaaG30ILO8KRhoPnZ4pmevGMDADZP210t+dsp/r+oz8sSCDMty8tnIPRm81v0mimlNE6vn3XQC997qQ7rdfV+pftHZwL07l2RRYeGvvdx5wUAoHJ0BgAAqNzK1QQla4Soru88ufNe9phWHQAAhqNteBm+t0Lpo2tt6Ftn7/uf5Rf2se4xWjFQhu/1XHWhIy9NoM+LLpinFQSH0yfyiJ0OsVIY3qJ/fS/gR2QAAIDK0RkAAKBy4TSBpRwB+eXsq2b7zfHNZjsLA83yBRmGHkUJADWzQtJl+N6awGf+sv33cfF/zMn+vfaPeRsW1xB7NAzuPa983XNetZpZHSE0LZBSuRBTu9cXr54322/t3DLPU1MD3vX0PRkfd2EAACpHZwAAgMrRGQAAoHIrjxko3R5dX/chAQAbZOWnoyVy6uH+g+zvo5OnC5/3cr7wn1NKeS4/sgBReW4WPVZ5vC4z4eo5eOMEImWCXsl+H+PqiAwAAFA5OgMAAFSu00JFntBMhaPR4n9P9sIRpUjYxZs9S626UNK6Q0/RfTY1Q9U6rucyLD5VXqeuJ26dc/mdHuLa1vnedrke7/thPa/vMimsh/d90MfycruYVcsJvbSAdR/xXlOvR8sJvWuLLmIUudah28ntbKUBAEBv6AwAAFC5TgsVKS8Ub+1zNM1Hl1ohGQ0zlotf6OtYz4sufmGt5b3oGOesmbi8Y3thUyv0q/toGLs8hrW/FwbWx7L1toPX82z2otm+Pb5hPk/f05tpWNF1zpX3uUXTWuvUJYVhfT/KY0VSENH3wHt/L0MaCZeXfg+jbXU0NXBV8SsEAKBydAYAAKjcytUE1uIQKdmLFpWjkUe7i09Dw49lGNpipQ9SioWFy+uxwpmRyTdSsheliI7ijoayu4Ras+sZ2eEz6z3wPhPdp6/weYS3friKhhaHvrboiH297rRitUo0lealUEgNYJP0u+emBu60EyQdnTxZ+JyhJwPqy9W5EgAA0AmdAQAAKkdnAACAynUqLYzmKXXRIq+ka37WlrVprlZz0n5eXh+L5dg1p6TjBMpz02vVR6KlUbqPV4pnveaqM7d551bmfs95+d1yHMa5aOnnEMtYRcsJ9TytPKN1/a879jpZ1+ONQ7FmV/TG1VhX45Xs6rGHHk+Beul3z22rZ69e+7yrNC7AU8dVAgAAE50BAAAqF04TKK/MaFeCi1Z4pQxN6mxPH7z9frP928/+vPA1y793jD5NdKa0aOh31VnTfv2dnzTb85dfO89c3uPpp812dAEhK4zrLeBjr8VtLz419CI1XWbPVNEyw76sc1ZLLy1mpRY03ZVSSr+681743Bu6WNncWdh+i/wxtlYbtkB0RtbRtTcWPk/bht0iYXZV0wZX86oAAEAYnQEAACrXaQZCb8SwNfpeeeFMrSyIzvIXHX1vhYHLBYCWPbYucJFSMkOgj6d/bba7hJq8sPZk7/7Sx8uM22s7/Pyj7KHIwknR1MTQvc/ozIhWZYGmDMrH+tKl2mSyf6/9wwvLy/dAZyrUfcY38kqYo5P2ex1NpQ2xwBPqEW1fNV2bVT0F24ahU6DrNHTbDAAABkZnAACAynWadEhDgdkCKMkOm1jh5fLYWlngrS9tTbzihR+7hLh//t22umF+eto+IKmAo5On5rnlx15tAiBvUSh9f6IjzC0H+w/Mx6LrfH/x6nmz/dbOrdA+Q4tMTlSmBS6kiPqmYf2ZXRWj39FVf7/RCgS1DekV1Cka1o989193jMuMyAAAAJWjMwAAQOXoDAAAULnRfB6b/mv2n+8129GyoC4lGNY+ZW5Wc9fW+ZT58YM7kgsPznoWndkvQs/nLOV5KOt99Eq1usww14W+zsH+4hJGbyxBlov/1j9XPp9l/XT8m+V3Cs6QFx1DsSmrzorZ5djryKFu8rw3ZfzNf/T+mtruIq7L98sao3YVxgxEvruX41cIAAA2hs4AAACV65Qm2KToAhNZuDpcXvX3lc+vdtbn46VxFKFWXFZ8dy+nLrNdZu3ZqCiXlVvmo+nHC4/tlXBHF/BbZzqCNAEAAHgtOgMAAFRuq9MEOjuaFzK5qgtHbDvvfdcw2+H0SbN9/Vv/2vh5lQi1Yh1IE1we0XtCpOrAC/kf3H2n/UNS1F1S0t5CdKsiTQAAAF6LzgAAAJULL1TUlyxU44RnImGgMuyym9rnXZaJTq4CUjcA+qRtji6S1WWBLG+yt6PjJ+XTU0oXq6t+d/xRs3191N529dhlWqDvibm4IwIAUDk6AwAAVI7OAAAAldu6MQPRPInmhHSf0/lZs90lP4Q4/QzK8Rm6wNNkr50t8o/2MBAAWAttj7z7gJav68gmq8Q9pdgYtbK08IO332+2f//vvy18zaERGQAAoHJ0BgAAqNzWpQm8xYms5ykNCZX7R2c0hM0q09ktAl6UbgIYipbpafh+XPz/d5yKRYj+r8u9Ql+zXHToD/9uSwsP7vyg2T48/sQ8l77bUFpsAAAqR2cAAIDKbV2aQHlhEg39lCHqc1/Nv87+vj2+sZ4Tq5imBiKfAQAMScP3ZerYusdY6YOU8hSAPk+P5e1/NH3abE/2HtjP67DY0SqIDAAAUDk6AwAAVG7r0gQaerZGhJaPWcq0QN8LP1xF+h56n4Eu1NF3uAsAznmL2n3x6nmz/dbOrWZb7w9lZYBVXRC9v+jxvLax7zaUOyIAAJWjMwAAQOXoDAAAULmtGzNglYGU+WkrD+Tlh7C68FiLkV1aAwB98e4D3zDKzfXe4+1v3W+8EsbImIOUUu9tKJEBAAAqR2cAAIDKbV2aQHkhaSvU4oV0aikn9EpcrNJN1aWUplzzG8B6RX67Kdm/S9KmF1n3hOi9wpppsNy/U/p6vrhN9b4H1uJ+kaup4+4IAABMdAYAAKjcVqcJEGctnlHSBYWsdEI0jKX7HMhsWSmldHTSLsahr0PvE+hGQ8LPZ/kibPqb1W39JXeZxRU+bQO9hdu6pGgeTz9ttid795ttbzbCVVLhtM0AAFSOzgAAAJUjTXAFeaGiSBjJCydqePLDu+8225oWSInJn4B1sEaO3xq/kT3PGkWep+jWMMIdJn1/vXZW21D9HLuE+MvKL7XsZ0pkAACAytEZAACgcnQGAACoHGMGrgjND0VnqNIcleaevJKjLFdpzJAFYD2i5X860+CplZMu9rHGGaAbL0ev77V+JtGScC0nnOzfa/99+nTBs7shMgAAQOXoDAAAUDnSBFeEhqE0tOitq628MiN97OH+g2ZbQ1fRxY0AdOP9RjXEXJYdLtp/0TGwvGh5ppWS7fQZSHrWa9+z1G/gsEQGAACoHJ0BAAAqR5rgiivTAlppoDNmWSNcU7JTA17VgjVKlt4n0I3+jqLhYd2HtEB/vM9HPwevokMrRGapfd7oehv0L9t3a3bDCNpmAAAqR2cAAIDKkSa4IjRcZE0slFIeQrw9bkP7GvL/xf472T7W+tm6ZjdrpQObdZba3+4v7r6XPyi/eWsimmezF9nft8c31ndyldKQv76/5XtrpVSt9E557Mlem6p9NP2TPivbZ9nUgCIyAABA5egMAABQOToDAABUjjEDV5CWpKSidEVzWZqj0nEC5RgBqxxRc1qzYtGilWfZAmAuIPb484+z52nuebL3/Wb70bR9XpnH9sYWIUbfQ31/y5LByBgqLR/8H2k3R22JaHSGWW23b7721YkMAABQPToDAABUjjTBhlmhOC+8Ew3fdZlpTEOISlMDXUJcpAKA9dPflTfjp9Lf8mRPyoQl1JxSyha8scqHS9H2TFltWHSfaMoxcm5ee9olbWKF5cvPx7pW3b/cR9vqo5PF5aLeeS5b3k1kAACAytEZAACgcqQJNswK43jhHSv0VLLCQFYqICV7oSGdyao8N0YdA8OwqglK1m/US/+pyd799o+xhOJn+ax4F1IN5zqkHKIh+2gKctW2ydrfm11Vz9P7fKw2fSctrgJJKf4+rgstOwAAlaMzAABA5UgTbJgVvitDQofTJ822hsU09PTB2z/I9pmfnS18TQ0vlYtfZBNZGL6cfZX9/eY4MmUFgHXTtmCVtepT8tN/q4aktZ2Z3Plh/mCZajgnKYdycaVIyN9b2Cda9XA6b9tQfU+j6ZlopYROBKfHm+zfa7YPp5+URzdfdxOIDAAAUDk6AwAAVI7OAAAAlWPMwFCKEp2HdyXPZuTYHk3/kv1t5bI0X+UtfqH7a46MMQLAdtAyNG+cgOakNdOctwX5YmJWyZ41s2lKsRn3jo6fJIs5hkpy5xcUi6C1J+fk1I02dLSb3/Lmr4zxDMFSSa9cUz3cf9D+Ied9ePxRaP8+EBkAAKBydAYAAKgcaYIN01CYlgaVpTTPZi+abau0UGcJLFllRztOf0/DfLtOGUt0gRQA66W/N/0djovfdRnOP6epgehMftHnWW1BWfKn52aV4h0e52V1VpmgKo+lbejt8Y1m22u/LpZeXzzniYb4S3oOmpooUhhHJ23qRM/nmtOe9t3uEhkAAKBydAYAAKjcaD63hmoCAIAaEBkAAKBydAYAAKgcnQEAACpHZwAAgMrRGQAAoHJ0BgAAqBydAQAAKkdnAACAytEZAACgcv8FLXXuSkEsp+4AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "gt_pseudo_label in the 1001st data example (a list of ground truth pseudo-labels): ['5', '-', '3']\n", + "Y in the 1001st data example (the computed result): 2\n", + "\n", + "X in the 3001st data example (a list of images):\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAABpCAYAAABF9zs7AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAABBV0lEQVR4nO2dZ3gc1dWA3zuzRV2WLRe594rB2AZjwPRuOqETQygOvQZIIR+QEAhgeq+BhB56Cx1TbcAGg23cey+ybKtumbnfj9GsZmdni8pKu9K8PDyPtTtzZ/bWc889RUgpJS4uLi4uLi4dGqWtX8DFxcXFxcWl7XEFAhcXFxcXFxdXIHBxcXFxcXFxBQIXFxcXFxcXXIHAxcXFxcXFBVcgcHFxcXFxccEVCFxcXFxcXFxwBQIXFxcXFxcXwJPqhfrGIel8jyhCUkNBoApneUWTOgAbtBpOveYaOn27hl3eXc/t3ecQkhoAXqG22vs2B6XHkmaX0Zpt05Fobtuko10CMgSAX3ijPjfHhNOYGfXAxfSZNottb/bj+93/G7k23vWZTjaNmbasa03q6MhGz4Wa1Bn6+XkMPuunyGeLnxzPiqOeTHpvpo2ZlypLePbUI9Hn/NrwoVJfH7oWe4OiIlQVGQ6BlCCE8bk9fl+8z1uYqlP24tVp0yjzFDSrnFTbJWWBINN4rbqEbyv3pKa7irJnH3r75rf1K7m4pB2lEUq96bUKX1QNR8uB0H6j6Vu0KuYaTepZKRS0FZrUea26hMV1ZQDkKUFOL/ql2RN2W7MiVMWLO8YRkio6Ar3Wg3bA2Mj3RaXVbfh2aUDqxqJuX9ClDlKkVIRSUIA2eiC6r2U3n6L+naQQbB+k4BWpvU9L0KoCQUCG0KQkT/EBNHk3H0bj70+cSZ93t9Dp/nXcM/gVhnoF4MsazYCLS2NINlacFvVzP7iAEXesJ/i3Om6b8hhDPCEgL3JtZDwKX9reu71RK4Pc8uiZ9Hl9LQChshJ4Aq7uvDxyTbq1AprUCcgwXqHiFWpEE5BIqwqJ+9AtG45gw7k9EdW1IAS5v/Vy89OPoSJRhE5/TxDIb/Hf0pIk1YiYC6uugRAIjxepadGaAimR4XDU3/GQw/tzwKMzOajgVxShx72usYSk8f4qkmIlQKnaevXeqgKBggKWilNwlnzsn1t3MU/t6MGn20ag+aBiTBcOLP2W0T6vu8txadfEGyuJUKsVwqvXonhK2dPvBbyx17Ti7qO94NshCa9aA4BXSiq1HMfr0jknpdpu1n6jINCJXuCWhaq4Y9OhfLN6AJ1H56OGJFJAbf8ge/lBFSqg4tR3MhXretHDs4ON+3aiYOCEyMZf6CAV438hQWgQqaZ467/9ewE7+qscXjiXcX4fRh213PuDWfd5LVZuKrSKQGD+QG+kcxkkGzBWSdtk2ksnMuDRpfgf38Z/f/8khYpKQKqRXU6is1QXl2wlmT1Nov4uZbRdjnmP3Q7BpRGYO0ddotuENXtbtOSc5NR2qlASLkfW59uv+1fFRNac3ZtOY/O5+7aH6K7WAlCoCCC32e/bmpi/LyQ1dKnhFSqTcsK8f90dhFIsQ5MQQuBFojrIXJoEc1XyCyhTm79g2/tHOvtPMjLShsD84VaB4IGKfvx37Vi0HMnmYwYxsedP9PYUGI1Py6lrXFyyDftE8czObjy5cl+kKik/by9G9VnRRm/W/vAKlfI9NIScCEA4R/Digs78tL0PDw54lb42W4K23KA4zaMmq8NVXLriNyxc352C/fPYMUwyzBugRM1uWwiI1oSoQknZvsOsp0THL/a6bI12Nd+nNcgIgSDeoLF26Ps+OJIh/zcX7dFOfHDWgxQrOYAao3VwKsfFpb1i7evmGerNXx/LsEvmok/ryvSb7sUvvFHnqvbxkW2eOW2JX3hZeMxDhI426uy7QD63Tj2H4LYufPVSP84sLI9c2xb16mQk6jQfflYzkNBlnehV5uWGhx9nvL+KYqV11dMtjXWxdqrzRMKZubH0RNaUhvFkLS8da0tTy0yHQXCrCARWNWVTEZpAr6kBASVKbtyKcIUBl47KMzt7cvvPhyGqPay7bBxDR60iV/iSjonW2n20F/zCG1HZFyp1CE0iQhqazTo9XfXanDlubbiKU3+dwoZNncg9OofaHhoDPTsoEHlZf9xqrjOpvr/1GM1oKwdByiIUQOtqfMxnteb4bDUNQaIKdFJtWa+P/FtKhMjeDuvikiqNnXg0qXP3rwcz8OxFrLhhLHOuehAdaRym1U+SSceXS4vSFrvJRO0MsChUTP7NhQwNBbnwxVc4Jm8nqiiIeC6oQoDM3j6RTM1vXWuMo2bD7iCRHYZ1QQ5jaH3UNMT0swszdu2EnXS0UUa0uib1aMMXyw+9ftMYBrw9FalKVv5jIkcPn9tWr+ni0mrEm9hCUkusbdM0RP2E7hVq1FloKka8zdXkdTR6qAGWTVFYelYJN352IiO/PYu14ao2eRd7+zkZp4WkB6HpICUKelTf8AoVD2rWCgPxsP6e92pyGPC/89lv7m8iGoJU7wXDU64xsUCa+p7Gs4Tj+6VzjLZqy8ebcMJohNEcO+Irs8cz9MLv0T3w09n3cVeP75v1LBeXbCHeWWdAhuNeK3UZFVjF6Tw53oRvaBTSG3mtPWDOLZrU6espYMURT/LPE59n2FPV9LrPy/Jw+g3znOY3HRnpG05trCPRLAuMTrQ2wSuchYFsmUet7WJHFQo6kle37sGI61cQfqE7GjKiUUm0Xli1CvZFOp3rjNNY1aROmCSbgmbQ6kaFOjJGNeNJ5DQTcR6V+EX818328y8XFyeswVZUoeB1GCsPbe/Dve8ejVQlix8cx4TdFsaoH1M5W3VtCVLDqR5H+zew9Bov+jYf5/33ImSfWmZOeigqqEyykOyNwcny3K4RsrJZq2bi9EtR1uWgTdFROwcY7duIKhqEl2TG3ZlOMnc9BYHHEgcnUL+YJ/p98TTXrYFTexh2Delrk1YVCMzdh70R4kmlkd2KEKAkroSIm0mCMl1csg1jN9AQfc3pPPHNDWMY/H8/sf7Cscy69qH66HWxO7tkk587ZprOUG8+i/Z/mtvLR/DNccOp2qU72/aB0vrmsp9ZtwQ6MmIfYhJvUdyoqfR50UPesi1MfuM7Lum0BogWBiLzbTsJZ+20JihCB9X4JCiloZlOoih32sSatGRsCXtZYTQUh/6SzrZp5UiFhkRrVcHE+3FXrJ/I9FfHoZZKlj+/G2eP+iqlsl1c2hMeVBQR38pZkzq6FMYkZxkCyYzLsj3BUSbgvIOrr1fbyUtL7+zsmoB4c2mVDDDm40vJW+yn5sQgnbp6mJS3BE36ovqQU5nZjlPE2zNLZ3DRw2dSs0XjyAevI7xHJfP2fjax15plrKRDEx2vrISac7LY7dAknibAabL6bNUQ+tz5HatunMDc/Z9K6svbnjpytmGca0fHAlNQ8AsPqlAIyFDEJ1tBieSycElOIutnTepUyQCariBy/OhxLkwkDLg0jXjn1IrQkV4P0gM1uocaPRjp762xiFgJyBBbNY3un3rp8sVqhr+5nrvKfiQkvY673vY2hzr9nn38OvMnPs+hC47Bc3U566eOQd87vgbAXo5d65BO2qI92jQwUTwBIYyGtBhGtXXAFNc+ITETZp+B8nbnqHjfml9w2oUfc3rxTxzy/LUULTc+r+4lePl3d7Orzzn+u0vqPLqjH088cgyBEij+j4fjen7lGICoLaKrtXfi1eFxhb/wySMjWLchxLl3XEnFaI35xz7YaCG4sXOO/bqQ1Bj+4UV0/s5LxSFBCn8nOLvLt0BOzHza3vtDXK2aEDRWsdyUVNLpqt90lJsRkQrtaFIaqagVYWSfyDI6WkrZHctLGPbqryAUUAT6zipEjp9PThzOLrlr6PNxEM/0OSj5eRTvMZRtU/JoiAju0his/eqXqt70fHER5UcN5cOp/6FYya7Y8+2Rod58PhzxLhcU7MO6K3PwVg8ldKyW/EYLzdXg7NBr2appFP3so8e7KxlwVpBXB30CdBwhvKPNwS1FmwgE8XYsphWuX3hQFGm4ULlkPH878r+8NX4MHqETloINd+1O/juz4W+DuaNoCvk/L0X06cnKu4s4ov9P7OKrRJPxo026xNIcLVW8e5Kmi3VJSLo0h4lc/+xaH6e/d//0Esre81F5VIAup1Xxh7KP6EjCgBmG2DQWdPKaUIQ0ElRZlphUhAh7vQdkCAWlScGDMlHznHEaggq9llVhL+GwgqdPT8J5bS8UNLbBMqmBW4PTCrZwSsFHrA3XskYr4JpOQ8kLh1G++IkcIaBbV4L9S7l+1DtMKdqKJt2dbHMIyBCLQhqbaougeyeCRU0zqHVjDqQPRTRul9+YxSGeBmGDVsN6zY9/RQ6dZq2HcxX+3e9LOpIwAKCjo0mZ8Dgg3xMk2KsHuhdmB6Cnp4peKWQutNoQGDEIpJFPOUWrgkzXXLRa+mPrTiRehXiFyvXrDmf5TSPwjPey+1s/cVnBR63xii1CS/oZZyMHv3sNA14L0+3X5Zihc9TSUtY8WsrvhnzBMfmrgbwOWz9NwWlH8uzOfjxz87HUdlaY+MyPnFn0CXnCF3eysSZusXofuJqB5mHVbAJRSXHCDhaejVkMEqXEdSpDFQoHfH0p/Z9QqDstxPg3lnBC0Y+EpKfDtbNfeCPCgGmTFvm8njv6v86Lr+7Bwjmd+Mv5U1l+usLSox5LWrZZl2abm4bT1melqjHIxHkw495odXUJOV/Ox1sFN3adwxF5gbZ+pZRwLbchd62K5/MfCW/cBELg6d+X0MjenDX4B67uvJySFsgd7gKbQ0V0mrWJwnVhrun2CWcWlrtut22MPQV7Z1814aG9CRYK3qjqx+xA0IgUmIZ5YkWoiteqipCbcvBtria3Sy3/VzqXUb6MUwBnDEO9+dzY9VdyCwJ4ps/Bv96bdIEOyBDTaxU+qPGzQ68zjiViPHictW7ZEjm31bIdOslLTulBFSTC4wFhDrLMlm41qVMrg3iFil94Iw2fidJfuojy1a0fEEpBASvvKuTG0a9zQO56QjIHHT2h9OwSi70fGamMNYSmIzSokyohqUXVqV0jYE8RHi8AjZsGuXFEa1qivTuu7fo1H/9rNbcvOIwXpxzOrccXMP/sB1N2AbVqhCB6d2slJDWOnvV7+v5dwtlw2mufskfOaqzeBJl4Vp0O4kX2ixd4yLw+kfG6Vbu9PBTi2n9cTt4WjfPufJ3TCzfFRDLME7HeJKZbtl8YcUbN8RmvTduSVu8hyaQkPQt3OtaEF+190MVDFQq1ZRpynzHIfcYQmDiMQ/sv4pSCHXRxrd9bhCq9jsd39OSddaMhFMZbFeb+zQfxWlVpZFylsguxx2KP951L0+mm5nNmYTldC6pR5i0jd3PL1+v8YC13lo+kZks+Wr4PugaYUrSVod6OZTPQHLoVVaHtMxrdC3duG8TMOmePECMbpErh6iAF8zbx9Op9eWpHX6pkahpsp4RImagxaFWdknXSati5NBCQIcK6gkdrCGKTiSSTCjuiUKBJnQ+Ou5uNR+cTkioqkt18VZg2A/XWI237klmIta99VtuZF66eTPHc9WibtuDdVsHaU7py23G7c/gfplGkGAuBgsBr2X042fCoxE5IrmYgNRqz41aENPzdibYxCkmNkNSizqCdcNpFms+fuvBMii8D/1kebvjP4/RRqwjJ3Bg7JnvUyvZKU4y/Xxj2PL8+Xcz5H5/H58fvysN/OpDFRzwWqUNzrISkRrB+PdLWrCP/vB68MHYyI+95nP2SyF9eoUaNLbtGKZNok0Mmp4ZbHKrm3s0Hs2ZLCWWH9aByUON8d9uCjnY0EA9zshnqzWeoFzQZql+AXJuBlqBKBrh36zjeWzuK0qXlyKoq6g7ZDd0nEDrofrhszVHkqg3RIhWhc2znn5icV5e0/I6iUm4LhhVtZs6RYwgWwyVrD+bwknmcVlhRv+A0T2tQVeencOUylHAP9vHrQF5czxG3bZ0p8xRQqgZBlWjLViJqugHOqYhNhN9P3ZDu1JSq3LpyMl+XLuWaLvMy8gigsbRp6GLr+djDW/dn5ZQ+dNo3l7vuvp8+agDIzchFN9Pepy1xah97/G/3bLp5LAj6+Owv+1L6zRK0HTuRE3bhnHveYlLucgCmLJhCxSn5bAsEEaqC1HSQOpf+47dMPuaJmPLs7VGjBwEczz9dYok3/p3O+/9Z9gWb7vqUyd9dxObj8/jrVadx2m8fiXu2He982ervbu74FSFBMRYq07rd+n5unAln7AKwV6iGDUG9/ZNTfalCwYfRvqJXD8ZO+xGAuReM5L2BB/CbaT8y1Ot1jA/hFDE0U9eQNjVDtR4J1Gpe2LYDJVjKEE+IIsXYXbZ1xW3VqrlyzWR0Kbi377t0s6QzdTGIKxS4tBjeqjBaRQUAUhUM9G1mkNfIVrd3txV8fvheKGEMdytpzG9KreTQBcegS4EiJH/s/z4H58Zq3pq7U3UxcDriLFByyBU6Y3utZcmRw0HAoQuO4cxe33FO0WYgdvxYy2lIPNTw2cw6jZtWHsv2LQV4Tx9LaERNzD3u+EtMs+pHVeifs5Ve3go+2m8iugpn/zqFvbut4NYe38VNSmQKcpncNm1iQxAlmUVdoEWMPTOl0n4OFrHp+gEITefbZ7szOW9HwlgDplTeUeIRmJ08UQyGBpesaB9ed+eSHCOqWigqyIqQEJIqmgyhCoVbu8+i5uZv0S0uT4oQjH/hajyTtxj35Pi54cXjOXi312Ke0R5UnW2BfccXz6pfR/Jkvw8J/f19xvzvcjzHlHPzXcdzznGPN9gS4HG0q7L70GtS59Y1k1FOD1F8qo+3/nYHxYoPb30cCuuRQTzvro5CvHkm1eBPjtoVxciB4BUax+Tt5MBr7uK69Qex9uRSvjpgApW3fEWpmh/3GC7T14RWPzKwG7fMD9byu/lT2LK5iMKz/VSODOLNgErboddy+pKTWLKhG759cwkVSvp4tqGQ2AjIVJdnesO3JMl+s7l7MaVj15K9CdjT6Vo+8AqVYmF4clh3IN123cT6C8caFwnYuSrMqJozkVLg94Z5ctd/M87vHhOkC2taYb/wRI5k9JoaRFhEvkukobHu/L8PhPj9L79l++ZCCs/yUjOuhs6qPyIsdMS5JxHmPJOKjcxuQ9aw4sq9EZrOqBlnctmI6VxQvCb2Qj1a4CoWuexXvJhbz9gdLVcyaeaF7Nt3OQ/3/jIr57lWPzKwN8r7VbtQeqVGzkiVB++7mxFeL15hmG22pbHT+rCk5s5eDFm6jUNe/5LLSpbgtZ2xxjsT6ogDMtFvtu6cMl1llg1IYRgNJtJSqULhy9Gvoo82JrAaGeTAW66m6+MLAfD0KuO1t8czrvsvrfbe7YVk81K8CIbxrlVR4o4N69h5rnxvel68E/8BnXjjtjsoVXOBaDW0O7YasMdysNtsWNvxzSEfErpWY+Tzl9L75IVMe+5QLjzgGeeCtfpsifWcVrCFUy59gOMWHwPHVfHD2buh/3E63vq4NNnUJhkRykqENYQOOUKLCAMmrV2ZARni8Pkns2plV7z7edAO7cpf8pY2KXmFS8uSbYOrJTBStSoI8zgghfN+q6raDFCUJ3wED9vJqrIJAEhF8uvMnrzo3ROkIK9zDf/b4zH6egqa9a4doX1SdeNz2iEeMHohM27eGxHWGfD++Vw44Quu77Ik9YfXH6t6hcAr1A5T580h3pk+RM/fkTle6obUnSLm0YwiJJpmrGXZSkYIBADIaDUotPxim0pe+Eo9SN2zPRj56QqGvFfOvWWziBe/qT0OxGRZ1ezYo+QlItGONpVrdGSHOxPVkQStv1ooIAS6VNBkOGE7haRGjQxRqPjwCy/z9noe9jK+mx0Ict35F+H5dLbxwV67svC5Evp6GlwXTRJZTlufBZnqXd3ypDL2na75V9+v4IKvGPLcRQw+fxZPPL8v1x+4JGl55veyPkOfbvscXK+CeCTT5JiEpBZzNNfSZLqLb6slN4LoSpgfrOXoTy9DrfCgXwm5vSvprKS3kpwawZzcQlJjv19OYcv8rsjxsGlSP64o/iyt75NJNPb4wylBTrIJKVVVa7zdVzaeyTUXBYEPi2eA1CPuUYnaSZM6XqFSKERkh2St/z6eEGunhggdNwGkQEi4+M3zIjuj3ME7mLHH0+QKX2RHbI4Vp/briG3TVDSpM2HfBXx7/wTEJsGgly7kksM+4urOyx2vn1mncfpHF6FWKeh/hk4DKsiL4xpn/TTTF59MwytUJkxawLf3TkBsIn67KM59/ZQeP3DjnSeiBHRGvXA5g8et5oPh78UEsstkWkUgMFWXyAZ129xAT4Y/UIOW7+OG/zxTH+2p5QPZJBoUEatcqaOjU/ded4b+ex68VcBbQ9/pUJK22UapWJxbPSnsZST2NmhIHWotC6Lbxx5cpaN4bDhhehlI86hASsOlMIleskEwU2M+VzFC6y6a9G/AaIMHtg/ko+PHoi1ZDkKw8/QJbB8XJlf1Rd0bz2ito7ZPU9CRPNd/OvSfzqgHLqbPtFm8OHQ8V5QsBWJ3/TNrBzHirq3UDO7CQ4/cz3CvH0hsDGr1OOg4s1jzebbfZ+j9PmX3By6j911GuzgJaool94E5h00p2sqU4x9n9x9Oo/tJS1n9pz1huHFNGC2y/mUyrSIQKAiU+kQrYE5yOlIVSEUQkiqh+gRBLU0iNadplTtxzqlUf9OV2hEaVY8O5M7er3a4HY/ZRqnsKuyBh6xl2F2vrGXZ69T6rFRSh3YkUt3dxTtDduq/EcHc5v6pIDgofyFP3r4PtdW7G9dW6Rz22HVGvBYBRfts5pvdXnHURHQkN9uWwKl/CyFjXHjnh4Kc8OaVeHcoBP4coFdZOT1VmVIbmGM0XuRCl2isdaggIiYEwpb0yCkHkr09rhj2Gbc8Oxm9XGPUgxfT55BVvDfsnRhBLxPHS6tlO4z5DGuMbw86daRblo0nFFTN7ErfO75n02uD+WmPl+q/ybzGSifJ1PXxrrd/ZsbIj/KHTnCPiald8ArrHR2XeLs74fGgewR1upeArA9I00jVsDkOdHQ0KclTfOzqyzFsDOq/P27JZPQbAuhV1QghWOkZS2DXcMRf3qWFECC8HqQUEWt4BQUdyfJQKUOer0aqguuef74+qJSzFtXIJBo7bjraxqY5mHWoCiWmXRIZJto5p2gz5xz4LwZ+ci59rvmJJV3HEh6moctYrWqm0bYjWzPCReaIxlV4WhAKuhQdIgmIFXNHYmKec8XL3x2SWtTn9n9bfa+tGgMnrGdqZvpoaz/oaG1hxStUFESUl4GnT28WPT6GVRfqXPvQBYx85TK2arUpn016hRplT+BBxS+c9wRX9fmIiudK2PJqPza90o9Aqca+t1zBvr+cHHWdGVfCFRJSx9qvTzjlKza91IfKH7uw5x1XcO+2kZFxo9BgL6LJxLY8TvNnNpxZZwqqUCLpwiG2XaZtG9ZwsS6j3A6tWNv2//Z8l/I3+gOw1z+u4OSlR0XaJFPbpc3eSkMgLOehbVFBNXqQDeEqpAJKcSE+T+YnVHJis1bd5P/XaTWsDdeyIVzFVq064SKs1dtaJFJD6sgmd3ozTnvDs2SHFgoidVG/q9CLC7h57ze5YPTX9HpnA72m69TEaYpEhpvG0YGM20aqUDg4V2PGbq/x/diXmDHuOSgO0eOF+Wxe2JXNWnUk/0GiZ7kk55Zuc/l67H/IXwe9/jWPH7b3QxUKFVoN27QCtBwPWq4H1cFmxH4s57ZD87DWn1O7pIrZLucUbeaHsa8A0O3R7/hlZa+WfeE00CZuh8autHUencjIba9ZZ1PwUjF1+4fJeV3hzl6vZuWgOv4P1zS/EAF1nQTXXPUKZxaWxzccs0fMs1m6JzIYtAdqSXRObi8rXpntHaMetAajwnqELqP8nZOdSVq/d9LaJLIF8KBy98RX+Pe7E2G20d/852/gk5FvpOye6tJAojoK6yrzg7Wc/NQfyN0kUf5Yzh7dV7ObbyeazI3rWZKo/d22SQ3rWqFJnZDp3RPHq8BK0jrWtUbFNmgr2kQgsO8wdanQ4Fnb8h3YyX9dkzrVK4ope/k7Nh4xlv8O+jBrB0zJlyubfrOiRGw5tO6d+HLHUEb7P497uZkoJxVMtZp5fUgq5AiNoV5fkw0HV4ZrqNS9Kb9DcxiT9iekhldoBEq8QB45SmycAJN4i0JjtCz2sWKWd3x+FccP/phBswZR+PJMFh8xDkamXKxLEkIFArqXsj2gM7N2AF3maeRsC/LbwZ8xpWgrmsxt61ds1yQbI82Zb/RcHU9ZD6Qm+CVYRz+PpFjJzPZs1TgEJl6h4hVhpBBpF5ri7YYiCAUsFr6QfZL0yPc2Net+RRhnYmtqa5n92BiuXDMq5XulcLa8jXyuN6RolQJ2DPTy6HX3s6ffaJfGGMJV6LUc/fR1lH0TSElqby7TP0j7I5KiSZ3BXo0T/v4xdbqXSTnrWFxXZpwty4bjhKTeNCSe9KwaGadxYHqBCAkIgRBuuO6mYm8jv/Bw/dSX+fLUoXz33O48s+Y4Audv45De8zkqfxWQH3UvZF/SnEzHrpFUhYJX1n+iS8K6EhlrKLGbInP9qNGDqEJEuW8/csizfLDHaJZ+MIDLL76MouvX8Mbg9zOyzVpdQxCQYX4KePihaiB1ZXkEC1VyRIjW8DAAo+FWh2uYG+xm/D18ELkFgbQ+O93c2eOnFilnTiDARTVDyVm1PfnFUqYURjfqWkWg5Zbwv527sSXfOQiLHTN6ZVCqbNN6kLdBkrNmR2rPbSfkCV/EFzog/UmvT4dLk2HPIdEKNdThg5FhwQc1fnbzlVPWjHDHLsacNMq3nrpCLwtXjKbgx7WM/OsG/lw6F6/Ij1zjZMybiYtKRyCeUaGOjmpbyw7NreXg3O941z+e3NWVLNrYjQ965THBX06pmu9YTlvR6m6HS0M6V95yCbnlGr3/soRjS+ewq0+DFH3gm4o1kM7xP11Aj1s9cBpMfnUGk/KWADkdfnCN8Cn84eYXKNeST/C6FGgo6FLBK8IJVWqheknbKzQ+KR/B53/ahxlbxjX4+uoJ1HH1WiQhQfep6NdVcPxV3zbqd2Uz5s7FtL/wCy9+89hANC3cd7KzZyc/dwVBnuLj1UMfYvak/ix79TjuOetU+Mc2PhzxbhN+WcfFXr81epDTn72Kfu/sIH/JQigsQBFGYCmr3Y293ezY7UAiqZWF6yraXKLmN10Skp6YsO2qUCiw5eKx8uyJDzPnqH488uwx3Pvg6fS9cwmP9/kyo9qm1TUEdVKleHkA35ZqDu68gJMKtsZkEUw3lTty6THrZzhtD6YWr0RJEvWro+AXXk4q2AnsTOn6VIPSWAW9fCXAo/7BkaBUSECRUWlFrccB1gh9uldhfI81TC1e39if1v5IVTuThMZORuP8Psb513NvUCBm/cqmqsEt8h4dGR2dvA0SOXs+GuApTEEgd4gU6pJe7MJ3Y9nLD+P9y3myGjxzlrKhtqiF3qzlaDUbAnuMeykEqtBjJCz7fS0lPUXZEkgRFWkvkyS0TCEVjwBrIKJE3hzW+07I30CvaU9QJxvO2JR6g1IdJepv4z69/vnGd2P927GeqXYUYmxh6m0IktESqXET2SfEU526JCY6Mp5iaMuEMNpVb+j/jTG+tZ+DG7ZabpCvxuAYlt1uM6AIvCKctG6tWhzz2rCZQEnXUURs1Mm2pk2zHVon/ni0pFAwOxDkP9v2RtapBI4Yj9qzxjGcbqY1UlvR2B1IMmEAIE/xcUCuDgRSyqLXgBkjouMJA5mCNZdBbd8QdYftTuVOjWs2jOX8Ll8zwtfyuUjaO/+t6sKXO4cRKBEEjhwPEqrzFT5fX8iNSohLOn9Ptww7Z+4IGMKVYMdgyN9/GKs3V3OTdiyeSiP+RmO8DrJpPWk1G4KmyKmpLBSJUufav7t04el0Pr+WnPM9PPTIXXRXdVQRa8HrErszSdSpm9u+qTyjo2LNN9CU+klXnX57+D2sOcTPBfdfwYK/9ePmZzvx0oCG7KCJPBXaW74K+7wRbx5y0q7d9PJpDHpiDZ4HKnjo90+jIJkbLOPBa09lxpbxDHh6C+cUbY55Znuqv0zBnsHVL7x8duqdLDyhhL/efD7Bj7Ygti2Cwf3iasasmtJsnM/aVEPQVFIxhDI/06TO7KDGP9cczsb1JcgDuhIcUssoX2b6gbq4WFEQkaiCiUiHQa79GMj67zJPAcVKECUEctNWasLJd7GqUNClm2xHkzrP7OzJW5vHIFUo378343vMi8xJAbkZT7WGp6KGOj159lGXlsM+fnqpeeT5t7NtF/DUDaB4emreUQkRGPFfMpCsEAjsLjfQIBTYG9B+9q0KhX+uOZy6kwXFp/h45bY76Kx4gFhrUDfqmkumEJVD3fadLmN3oObOu6X2jaZdiCoEKs42CKoQkfNQ+44p3tjxCjUlm5NsItVgUNbPbv3geIb9bRGh+wK8eNb9dFd9mCmNrero1gjA5WJg12aZtgSlaj6zzryb704q4vapU/Bv2Bm3Xayam3jriBQgVCUj2zarRmNUBj0HtZzpHmUmhTEJ6h5kVTVCh+6qnwLF2TUkXkIfF5e2xto3d8ldw8rTe7FljIfjZv+e89fsj9bCmdQMYdsweHN6BzBCGu/cq5YN5+3G/GW9mPjzScwOBJ2Kiyk72y3kk80TMccD9X8/uqMfY384E6HD+rNGMH7gKnp6/PiFJ1JmJyXM6sN9rJnclTt/OowjFk5ms1advh/j4ohVYC1WchnirWD1ET5WntSVQb7kweDiCbuRgG0ZSFYJBCZOwkAYjTBaZPCZ6XQbQxiNgAw7PsPFpTWxa7/M/g0wOa+OOZc8wMSjf6HfZRXMe3wXQvV9vyWFWr/wRnY8IWmMDbtQvvigp3j3D3fQ/TMPnc+v5T/b9o4qI977ZOsZKxh1kSjBl4n9N6pCYdqMwyn7zRJ0r+Tr6+/m+QEfRaLamWX29RTwy+n38ccLXmbgvRr6DaXMDWaei1p7xOyvTv1zgLeAWWfczTe/n8Z+OckFX3BaqwzvIKlpGemhkxVHBmAm1mmwHbBPMopNtlEQfFXn4YLvphDe4SPnGg/qmB1R6V+jyq4voxUi4rq4NBqn/q0IiQyGEDa5Nx0LbUNK3tijAZ8Qxq6nfpJr7546jdVuRNWHBBk2Nh1+4Y1bll94yFcCRtAuXda73WZnNtZsIdExj9l+ecKH2WRN6edqC8UPSRdZIxBA9ERnpHA1GsvJalkVCm9uH8uQq9ZTcfBA3rn9zvowkUpEoxBB1seudi13XTIMu+WzSRiNsDSEZOtRZEssxIms4p2u1aSM+NHrUolKkJRqwKpsIlWbAfM6q7umYVDWoHXxC0/kWnss/VTcsl1aBnNN8KA6rzO2NcLUEiVbMRxtZeqNCjPRhiCrBAIrhq1ArAEHwOygxqmfXIxSpcIfoWDADvJEtLWux3JvKm5CLi5tQbw+6EFFsaitW9J6vzH9XhUKhYqHmpN3sGVcfxZ+1Z/3Cnbl5UMfZk9/Ygv5ZCGUM50ow88E764guK18JP/67AAUCUvv3oMD95wXE47YWmaEDN9RthcMDbTxb+sibl9nTFLVEpnXmf3bg0r+0RtZPGgU/AgDfp3KI4c+yxF5mZFPJytGYKIzSPN/6xnrdzWDGXH7Vvp+qPHuCXfz854vkqf4osqy3mt/liEVZp705uJiogoFj6KBMKLcaWnqr042APa/C5QcZu/xHK+dcB99PtYYMW0b39YMaXTa5Wwj1XlCFQovLhnH4Kt/wFspWHbKozzV92tHjaRZpvG/VUPjagvSjdX2zGzXeOtEvKNrpzKBqPK+3vV1ZvzmLrp9DyNuWskHO0a39E9pMlkhEKTig+1BjVb31EvWehxJLt4k1yAVupK5S2YQb9I5pcv3LL63N1v31Jn05LXs9v3pLe4lk+pkaITJrf9bStQ4sRPMhDtOKdGzDad5wp6R8M5tgxj21EUElxWx+MndmXT4LymVadhr6EYoYxpCd7ukFwWBX3gc+6PpLmsaq6dqGOt0HK0AmbjEZH0vs+/4Q1Iz8lYrxs6pTqoEZCjhvaZWIFUVoItLa5FogT84V2Ppgf9i0PD19L/9R5heEm0b08rvpSCNGUUxcsfbvXzMCVVvJ2fj9nnCOp+Yc8r7G3Zh4G0/k7desPDwR3iizzcx18crUxU60qMghSCE2mivKZfGk2zutwoE8chm1/WssSGIl1nP+u85gQCnvnAlnipB+G9VhOo0fvfAlVQOCzF38gNR8QfsLlrWcl0bApdMweyD9oBbMSgNnjItTTwjQzt9PAqdrlvNgo3deeK5o3ikQPLyGfcyxu+P3OOnwYguniFXthE3vLTUo4MMNaJtNKkz2reV8C3bWbWpC3989Fwu76Xz9UnTKPMkz4bo0jSc1hnrAm8agSYiFY12XYrJyVqbrBmFVq+CeKwOlzDgjUr6fFLJzWPe4dARC+jz5npKZ3qoi6PmsZ7tRD8rA1vLpcOi1//X2pgTZCoUKDm8OeRDHhj3Ir0/r2LA6ztZHS6JuiZ2V539Y03HjOoYrS3Yqdeh6QoiLxd7BOJEu0izzsvUXD4d+TZXjPmMPm9tpO+HGtv1rJmysxL7OuOUp6Ipwqu1HOMZmUlWaAgaIhAm3kmoQgchUJau5cE/n8LO/ipFD2/koO4zKHaITmi6+oSkhm6xLM3G80yX9o1feNtEFWmOkUThhu0eAqrQkUIghECvd/O17rYaVOIKXmJjG2QbHlQUEX3k+MD2gTx375HUdhX0fHkNZ3X7OGGqdyvWOtekbhyBCoFrRpBerOuM+XeqxoNOZcVrY69QyclQ75GsEAjA2UVJkzobtBpq6iM+LQn0N+KqV1ZS9MVSpDKYx4Y/z3CvHzXJIq+jt2Ak+I7NhnAVlY2MwlUopKsKtRG70DpPMLmeEFrXLugeWBwK0l3V62NutCxOY8TxHBxJqNiHEtRZGSxlpW8j/T15jkJBe8hnYAZNAwjIMBvCQb7eNphuX2xm48HdeHXYKxQriZOpxTuWiZxXC5GRRmjtjXgCQFO1Albh10qmGhVmhUDgZLgDUKHXcuhT19H9B8NoUK3T8S1eDMMHU3tPLceXfcIAT/LzSUNyd4WBxuJ0BlyjB5n08rX0/jzc0OFT0AivOVRl4SkPxdXOmGrUjqK9acyO5LZ+b/DE65P49ZduXHLJ5az6jc6Kw5+Kuc406GvKmb2xa429x6mc3f3VHHzHV3y+eSgv3XoET5UdxZuX3cEgb0Hce9qa5iZbMu97o6qM+285hWCxYMS/FjC55HtyUjh3jkdEe6LrWPXMyeycsjGuQ7qJV2dObd/UukuH/Vm8MtPRxhklEKTyAxeH6lgTLkYVOlvCvSlYLclfsBlt/UbQNJReZewcUszfB77CAbk6ZgYxl/SzIFjDklApXeaC/3+z8PTqicz1R1ynAEQojLZuQyR8q0nRwL3rz+46xoLfkozy5XJv2Sw+XjkM//8W4t9zguNY0jGiCnrSvDMpVnK5oXQhZd7tvPbjJPJ6FVEtG4wJrbSXRatGD/JLUOWzihEUL6+lYmgu/+j9DgO8BWhSbfLkbQR+qiPYuxPBYpWvagcTkCsY5fPEPWrJZiv35jA/WEu5nstobw0lal5K93TUuopHRggEUW4cSQbO0W9ezeCXapBCoPtVwldV0Puizay4YjjeTTsIPxHmwt6vsqe/DlcYSC/mrtE8XzbbpsvSxci8PJZM68LFo7+MXB+SKu9v2IW8qb0Ir1zdICgoKslOGKznqubf7Rmrd4GOHkmAYycgQ46hu62YrlJeoeJXnKN7prs+MzGRi5VEaZnN+vMLT9x6+rKukJtu/B0AY++dwwHFC+jpafCugOi+a7dmT1T/xxUsQ3v4A55bPYGXrjiK+8b5+PriaTGLXnM0QNlOjR7kN89cQ48ZQfb85yxu7z4n8p2Th46p3Uw2dhqL07GY9Zlmm5sjT1hsCeLNbU7lOH3eErS6QJAjNHYM8JNX6OGTipHkK3M4JG+TkTTChiZ1FoYCzKwdYPyNgq9CQYQ0BCA9ChN7ruS80i+5Lm8kHr+Xk3p+z5mF5bjCQOuhCgVdaggNREhD79cDLc/HIYMWcGXJSsAYsG9Xd8evhkF4QSggNdROxeiD+1DbPbstzTOBroXViHEjkR7JEzv6sFfucsb4/REPBSUz5P+MxezHjUWTOtu1PErm7yRc4OO0LjPZLwc0GbvQNGUSL1XzmVq8nnld1rLslxqKSgfERKaMt9PtKEcHOjr5ayW5s5azoS4zMkOmq+5NoSIdtPoMMcLr5c4bHuPN7WOZc8Pu3FU0kl63PcQefufzuxNmXMjgv9VE/q67LsDFL70BgE9ojPdvY23YnejaGgXB6yfdy8pjOwMNbQOGcduMQC4P/ekUiuZsQlu9FnRj4q08aDjn/OMtxuWswi+MHZXdujfVnVR7wvq7vRbtgF0jYNUcPDf8OX5+sZRLP5nC2ydM5I4/H87yQ5/GL7xR19ntMawpX1scRQFFEHTwNoDM0/Q47RaNKIzxd5E6Eq3evkJI0KWCVp9GPZFhmdNniTRCipBIh5wV1t2u6Y1ijY3QYZCALtFt7hjWhETm31abmJbo+3bNUry2NnuR+a2UkrDufKTUFnYirb6SeoXKpJwwGwuWsWjDUHw7fQRRUev77YJgDc9vn0BIquhSEKrwE+hpvKYU0LOsgmPzaywl5rM2HEiqcnZJP7v6ctjVF902ARni8e2DeXfjaAoXbSe8fCUAakkJwd0GsHW0yokFy1M+8+toxFonx58censK6O2p41KPjrZoKeyc0Bqv2CzaagfbUs+t0YM8un04n2wZTsWoIoJFgkKljmRTq9UzwU68Nu7lr2DWvuOo6aZw08aD2LdoMacVVljui54E7Yl12jMKCpUDIX+fIazbXMnN+SOZWvJ9xnouqQi2D1bwVvfnu03buc2/gwtLZqfkHZROIa9NttaGr7+GVAVSif5xf1p1AqHf5UIgCIDvIpU7nnwQFYkiJD1UjYD0RZ2TWaOBmfnYzWAnHcUqvS0xJVmns7qloTAv3XQExV+uQC/fGMkxUTduIOc99CZ75ayiRI0etKnspNo71t2ElXgx1gMyjBfLTra+nuNZUMdLZWx+32LoOkKLzW1gutSFpBb93m2M0zk/RE/C1t+xIARv/PVQcrYGOfzBrziz0/cM8OREaULi2Xk41bNVa2O/7pJOCzjqrnlctOgMlp/ek+m/GcdJlz1g7HiJ374hqRGWobh2KO2BPMXHx7+9k7mnlvL3G89hxsLdyXs2wLWdl0WuidfHWqK/N6b/alKnQPHz7nl3MDvQi/v+chpfrptA2VPbOa94Y1LbnnTOhW2ma+/q2cmG/YpRQvB/S4+nW14lAPPW9qTrBD9K/VGeNrCWcf4GewDrIA3IELduHcO3WwdSPsqPMrwrfXzlkWs7lLqsDTEnT8NYxmibkNS4ZeuufLphGIXLq9A2bTauLSmhar8hbNnNw945q+jradAMOO1kzJ1UpqqY002qv1u1GCf16r2NylMmgA6nLjuC83p+xRG5NY5lpMPKukKr4cZN+zNzU3/0vUup6SHopASx2vU0/K7MSyQWq5WJfb+ADHHz5nHM2DqAql4qlX1y2Sd/MUO90Ts8wy6h6fYx1nfJU3yM8kGRvw5tw068lWURY1GTeNqAdIS0zjT6egrIE+VUjBDo3iKeW7onS3t246YeH7eapiAVF0FzThvkLaBObsRbpePZUkmN7m9VQ18n2kQgCEmN8T6ND666g3vL92XuWcOoWrMNFEGnk3J4+ra76KQYFVOoeICc6MRD9eVsDtfwwbT96DJzE/u88CM39fiUYsUHGbTj6ChEOnv93xV6HR9M24/Ob85Dr244RtAH9+bsf77NMQXL6GbRDFgTwzgJBZGkOLJjCQVhNDQpY4xu7ROHdYf4wS4vUHlnmEkvX0vdcWGueuQUjpj0TMwZvpkIqaV3jj8GC/n12tF0Cuqc8dRrHJ2/glJbW1vPvDOBuPka4iywW7QAn9+xNyVzyhn73Eyu7/YVxYoPTcZa+Cfa9afyHnYUIdEUBaQRCMkrVUehwCyrI82FpWo+M8++i6/qSrnn0jNYuWMInz27mNMKtsRojVt6k2Ht18lqvEFDpiAkICV10kMYLemYSOcRUKsKBNZEQn7hoZvqYWLBUt47em/82w1jtG27a/TzeMhToidAawePlAd4AhJRU0ehp45uar7rV9pG2DupF8GWPXTCebsYH9Rvkqp7CcblrKJLnMht9iBUkWMhBFmUeqPFUFBAxDf6c/qsQMkhtz4VsV5Zia45W7s3Z+eaCE0qqLVhREijs1rleC6qoLSpZiDVCdXJ6O+q9XvzzfoBBAcoVPXuynmFH0bNPckWGh0Zd8Fwmufs7FmyklfPOYjabpKDf/odx/Sbx81d50dd05HnwRI1j1G+zaw7wEvOFh9///ko/tt9M48MeI1uabZVMjUxTobR9jZXEFHH3SoSxWZ8G490aUxbVSAwE0dYJaDj86s48vL7MJKc6IavtIMLIiT+8abtgEvrEznGsexMChQ/C056EO3EhmQhGhIVEZV10iSeha31e2t8946iJTCjaJr5NszdTaLfb7aHkBjunfXE27mmE3suA/M97E9u6zZN5dma1KnSA8x8fCxlby+j4LUtPN3/PXKFD2jwgDHTrVut2K3PSObamGwsXN9lPldf/wsTZ51Nj7M28PI1+3PzBdECgdP97X3MWH/nIG8Bc6fcz8uVZbxw1hFU5/dkyb8KKLOseC1ZH8m0MUZQsFhbHq8ZkUAIvMIY3/ax7vQco0yZNG5PY2lVgcDc5dk7qF94GxXXOSQ1Tl12BHNW9sG7m2Drrv04t+DrNLyxSyo4qVYj7nKWdk11B2V+5wp4DTR2N60g6Dt2HWuuHY++TWPY9PO4efzb9TE6DDrKQpEIpzqwu7qaqEJBEcIIIRwO41N0coXP0RMkUXspiJhnJDp7tmO6QSpCotfWISzyRWvZiWQi9jnDL7yM8q9n6WkFeKsUzvnqXAb02spbw19x3JS0BHajVDDjBsQfY9I2RybqO1atQzrygLTqTGB4F6gx6YXNXV+8v62YUvqq5wYz/PJljN5vCYvOfYSTCnYaalA3dXGbELMDjOeHa2mjmCMgW7snKqO941QXTouF0zgxr/105NvMueQBcterDD5nPvctPSjqujBaxI6gJVGFddcTjkyOido7k9pUR1Ijg451o1KfZEgIFOG8OzP93CP31LeHVbulI+vtYiweUgnmLbMMa7m6FCB1RJLprqOMGYgV7Mb4PCw6/WHOOeljhv9jB8GHytikhROU0LznholuU7MP2ftEFPX9KSSNtdHaX0xPHEeNaRratE2MCu0SUGNcKwyhQonZeUafNbtkMmYbJTKs6igTWDxS+f2pXKMg6HHwWpaVjENfIhm4/PcAyBydxw58hoNzA81+16aSCW3sbJEvyBEePPUHG2esOJCZPw5t+H6EZNvoIfyzy4tNfp6hQYtO/ew0d5mLi115HJIaJw/8iWduOxCp6gx87fccvOc8nujzTaPfqT1j1ve++Yt49KoDUWsEh759DaUDtzF9t+djbNWa+yxdSlubxmqLrEJwV1Wy7qwQcl03Fn56GA912Z+PJj3AIG9BRPNqp90YFZokd6FK/L2CYqhZhIIuRZTBRiZMMh0Ne+S7ZCQS9iLW7zJW2k90b3unOe5In458m5rhQSb9/QpKH5sJgKdXTz4dP4rD8ua05GsChlGhnUxrt0STqt1rY9YXwxny5xkACL+fqrd68cXoV+vLib4vEVZ1r92OojH1Y4w3neu7zOeGMxcy8KPzGHLObD67ey+0U7+Kek5jy24POP3ufXIUlh77KFPX7MeGo/2UHzmUml1D+GX8/BRNwToHOtnKmJhrVqmaz5IDnuGjGi/TfnsGSlhn/oRuDPLWxPwG87c5zY8tRdbF/N3nlxPZ+kN3QqN0tt8zkLvLXna1Am1Moo7flLKcOntHm9TsNPX3m/d5hUrpKWtYPG4P4wtdsOTDfvw3bwIvHfMge/ob7/5nX1Q3a9Xs9enleNb7CJ2tkVNay0jvVqAg6eLU2lohHRnXcCseW6dOZNuEEDf2f7tZ79qYe53mNvtO9OzdZ/DM4/ugVMHIpy/hgMPn8HCvbzqsYJBI0Du59AcunnYWVOrs88wfENJyhi+IeEMdfuQs7u/5Q/rez3ZErqd4eh9vfmwpsk4g2P5FDwZMm0X5G/2ZuftL9RXTMTp6R6GjTFythWmA9OGId2GE8dkvwTqunHopOWt38sMhAxnnWxW5PlWLezvlmqDfSwq5yzZx5JuzuaxkFZCZoWNNz5dYZXw0mnlGLwRVB1ezYtK/0/5uVuK1hVVQuLHrr9x49K8MevlC+t/4PR/1HkO415cdLmZHKhyRF2D5YU9x+IKjUY6tQK+pj5Fiq6f3BuzKXWWGNi3ZEWdTcCwrxX1tOts0qwQCcxKSmkbOUyXsNuxSfv/b9+onHhcXF5OYyGi273qqGur1m1i8sZRVzx3D47XGd5UDNb457q6oyG5OxxVOZeoOM5pTOOvoe2Kt+VsDTwo6raMWHcW6t/vT7/uamAWjrXGq/ykHfclLL4/Ds8LHHndfwX6nzeaunl93uHTI8TxHrIG4Lu/3Cdf/5yR0PbbPSilQlucwftplSAG6D87/7fuRzK0tjSoUFHSELhtSwrcRWSMQBGSISj2IacCc/+5PFH7biW+OGeQKBC4uDjip4c3JskTJ5eMR7/Bevxzue/g01NkLAag5ZFeWT84jR9SgCBGbZyBOmVUyQLmei+4R4PPiFYYld0hqGGGV4/tnk0Jkt5YmlQVywcLeDL3nW4TXh8jPQ1Uz131PR3JD6Txu7Porw3+9mN5PzOOrSQPRyr7C456oAlAnw6gI/MLL5Lw6Ju/1fNxrR/58MT0f+REUBaWokOlHDuN3xQsA4sZSaRFcgSA19vxhCv43OtF3Vjm6Ilhx0zgGTFzNlWX/iwn44OLi4myUaf98D385Rf9cy+aaMlRFZ0N5HZf/8xJUI7cYW/YJs/ioR6PCvZoJk8yyqmSA3V+7ipJfBZVnVbNLz20cVbAIKMAvoqcYa7Il8zzU/nlrE9dgs34h3XDJeLods4Zb+77Zui+WAk5HN+ed9CEf7TuS2/u9jl+0rNFcNmCPR2B+lofP0SbDSXA+65RP+Xj/4QCENJWKD3pw6IarkQLqugrumvoEh+WF0vcj2oisEQhqFneix7MzkPn5qKVd6D5+Ix8Mf4+QTE/EJheXbCaeMGCnm5rP8wP/Bxiq1Os3jWHuzUNgyzaQkkDxcJYeGiBHmLlFBMVKTsQdrlyrZmXYR49voeTLFUyaupV7y2Zh2g7Y38OuETAnb8OsqvX0BI0xtNs5PMzPI95J9ys1C+tCd23nZZYsfx1zTowXH8JOvBwqfy5dxJ9LFwGGsezRL/6B0ncXgaKiDSrj+7MGMcQ7C4AcQcamWW4sWSEQRCY0RWXF9bsy8bB5/LXb+2BRZ3a0kLYuLqlitza3Yw0lfmHnr7nnP16qw34AFi6q4/dXXhnZyW8+s5a5+zwTyWy51+vX0OdjnZLZK6PUnWb4XmvZdne+ZJ+3FnFduzL3hCCCXeBKdEzU0ebGVCOi2u8Jo0XZXZQoOZx87UcsvKgMRUhWVlXw5n0H8tHW/QAoH+nhfxfdQd+WEAqEiAT2sr+/vW3Tsd5lhUBg7kaEIggNrOPxPtNdV0MXlzQwwFsQ5W617/ZuFMyTiLBhkLVu/57MCKio6ISkl4JVCgXz1oPfh9alkAJ1fVu9eouiSYmQ7hzTHmnsQuoVqqFxqde6fFTj5daN55A/f5NRnr8n02v6M8i3ucnvNKtmICKkgUjc58x3T1c46qwQCEykLhHCWfVjJk5yXW1cXAzsu51k48IpDvtjw5/n3Td2jVyz8NOe/O3ccxH12oCaqbXsc46hWs1TA/ymcB6G7YDXMVyxfYcTkOFIbP7WJNEuWhUKecKHVDI3BLrV5iKet0ZjA4a1N+LFYTBjUFg3lVbvGZXoUNv2vrJ3TiVnTHuPirCRxfOddR6evuoEvJVNtylQghrKgmXIEQMigb2cYrFYbXg6dmAiiWMYRxM9G/R7Li6tiJEvQqKI+Ol2nbAu4sO9fkZ1WRL5+zFxMJ5v5iHDIYSqkn/VkMhZq0GD2tS6WEFskJ2IEN/qPgYpkkUKArsNhrXeXZyJqp9GaA0KlBymFjdowjYEi1n8YxXapqZrCIz3Sa3LpdPeRkjZxn4OLi4uLi4uLm2Oq1t3cXFxcXFxcQUCFxcXFxcXF1cgcHFxcXFxccEVCFxcXFxcXFxwBQIXFxcXFxcXXIHAxcXFxcXFBVcgcHFxcXFxccEVCFxcXFxcXFxwBQIXFxcXFxcX4P8BHkesO04OVroAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "gt_pseudo_label in the 3001st data example (a list of ground truth pseudo-labels): ['4', '/', '6', '*', '5']\n", + "Y in the 3001st data example (the computed result): 3.333333333333333\n" + ] + } + ], + "source": [ + "X_1000, gt_pseudo_label_1000, Y_1000 = train_X[1000], train_gt_pseudo_label[1000], train_Y[1000]\n", + "print(f\"X in the 1001st data example (a list of images):\")\n", + "for i, x in enumerate(X_1000):\n", + " plt.subplot(1, len(X_1000), i+1)\n", + " plt.axis('off') \n", + " plt.imshow(x.numpy().transpose(1, 2, 0))\n", + "plt.show()\n", + "print(f\"gt_pseudo_label in the 1001st data example (a list of ground truth pseudo-labels): {gt_pseudo_label_1000}\")\n", + "print(f\"Y in the 1001st data example (the computed result): {Y_1000}\")\n", + "print()\n", + "X_3000, gt_pseudo_label_3000, Y_3000 = train_X[3000], train_gt_pseudo_label[3000], train_Y[3000]\n", + "print(f\"X in the 3001st data example (a list of images):\")\n", + "for i, x in enumerate(X_3000):\n", + " plt.subplot(1, len(X_3000), i+1)\n", + " plt.axis('off') \n", + " plt.imshow(x.numpy().transpose(1, 2, 0))\n", + "plt.show()\n", + "print(f\"gt_pseudo_label in the 3001st data example (a list of ground truth pseudo-labels): {gt_pseudo_label_3000}\")\n", + "print(f\"Y in the 3001st data example (the computed result): {Y_3000}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note: The symbols in the HWF dataset can be one of digits or operators '+', '-', '×', '÷'. \n", + "\n", + "Note: We may see that, in the 1001st data example, the length of the formula is 3, while in the 3001st data example, the length of the formula is 5. In the HWF dataset, the length of the formula varies from 1 to 7." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Building the Learning Part" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To build the learning part, we need to first build a machine learning base model. We use SymbolNet, and encapsulate it within a `BasicNN` object to create the base model. `BasicNN` is a class that encapsulates a PyTorch model, transforming it into a base model with an sklearn-style interface. " + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, "outputs": [], "source": [ - "# Get training and testing data\n", - "train_data = get_dataset(train=True, get_pseudo_label=True)\n", - "test_data = get_dataset(train=False, get_pseudo_label=True)" + "# class of symbol may be one of ['0', '1', ..., '9', '+', '-', '*', '/'], total of 14 classes\n", + "cls = SymbolNet(num_classes=14, image_size=(45, 45, 1))\n", + "loss_fn = nn.CrossEntropyLoss()\n", + "optimizer = torch.optim.Adam(cls.parameters(), lr=0.001, betas=(0.9, 0.99))\n", + "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n", + "\n", + "base_model = BasicNN(\n", + " model=cls,\n", + " loss_fn=loss_fn,\n", + " optimizer=optimizer,\n", + " device=device,\n", + " batch_size=128,\n", + " num_epochs=3,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`BasicNN` offers methods like `predict` and `predict_prob`, which are used to predict the class index and the probabilities of each class for images. As shown below:" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Predicted class index for a batch of 32 instances: ndarray with shape (32,)\n", + "Predicted class probabilities for a batch of 32 instances: ndarray with shape (32, 14)\n" + ] + } + ], + "source": [ + "data_instances = [torch.randn(1, 45, 45).to(device) for _ in range(32)]\n", + "pred_idx = base_model.predict(X=data_instances)\n", + "print(f\"Predicted class index for a batch of 32 instances: \" +\n", + " f\"{type(pred_idx).__name__} with shape {pred_idx.shape}\")\n", + "pred_prob = base_model.predict_proba(X=data_instances)\n", + "print(f\"Predicted class probabilities for a batch of 32 instances: \" +\n", + " f\"{type(pred_prob).__name__} with shape {pred_prob.shape}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "However, the base model built above deals with instance-level data (i.e., individual images), and can not directly deal with example-level data (i.e., a list of images comprising the formula). Therefore, we wrap the base model into `ABLModel`, which enables the learning part to train, test, and predict on example-level data." + ] + }, + { + "cell_type": "code", + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ - "# Initialize logger and print basic information\n", - "print_log(\"Abductive Learning on the HWF example.\", logger=\"current\")\n", + "model = ABLModel(base_model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As an illustration, consider this example of training on example-level data using the `predict` method in `ABLModel`. In this process, the method accepts data examples as input and outputs the class labels and the probabilities of each class for all instances within these data examples." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Predicted class labels for the 100 data examples: a list of length 2, \n", + "the first element is a ndarray of shape (3,), and the second element is a ndarray of shape (5,).\n", + "\n", + "Predicted class probabilities for the 100 data examples: a list of length 2, \n", + "the first element is a ndarray of shape (3, 14), and the second element is a ndarray of shape (5, 14).\n" + ] + } + ], + "source": [ + "from abl.structures import ListData\n", + "# ListData is a data structure provided by ABL-Package that can be used to organize data examples\n", + "data_examples = ListData()\n", + "# We use the first 1001st and 3001st data examples in the training set as an illustration\n", + "data_examples.X = [X_1000, X_3000]\n", + "data_examples.gt_pseudo_label = [gt_pseudo_label_1000, gt_pseudo_label_3000]\n", + "data_examples.Y = [Y_1000, Y_3000]\n", "\n", - "# Retrieve the directory of the Log file and define the directory for saving the model weights.\n", - "log_dir = ABLLogger.get_current_instance().log_dir\n", - "weights_dir = osp.join(log_dir, \"weights\")" + "# Perform prediction on the two data examples\n", + "# Remind that, in the 1001st data example, the length of the formula is 3, \n", + "# while in the 3001st data example, the length of the formula is 5.\n", + "pred_label, pred_prob = model.predict(data_examples)['label'], model.predict(data_examples)['prob']\n", + "print(f\"Predicted class labels for the 100 data examples: a list of length {len(pred_label)}, \\n\" +\n", + " f\"the first element is a {type(pred_label[0]).__name__} of shape {pred_label[0].shape}, \"+\n", + " f\"and the second element is a {type(pred_label[1]).__name__} of shape {pred_label[1].shape}.\\n\")\n", + "print(f\"Predicted class probabilities for the 100 data examples: a list of length {len(pred_prob)}, \\n\"\n", + " f\"the first element is a {type(pred_prob[0]).__name__} of shape {pred_prob[0].shape}, \" +\n", + " f\"and the second element is a {type(pred_prob[1]).__name__} of shape {pred_prob[1].shape}.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Building the Reasoning Part" ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ - "### Logic Part" + "In the reasoning part, we first build a knowledge base which contain information on how to perform addition operations. We build it by creating a subclass of `KBBase`. In the derived subclass, we initialize the `pseudo_label_list` parameter specifying list of possible pseudo-labels, and override the `logic_forward` function defining how to perform (deductive) reasoning." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ - "# Initialize knowledge base and reasoner\n", - "class HWF_KB(KBBase):\n", + "class HwfKB(KBBase):\n", + " def __init__(self, pseudo_label_list=[\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"+\", \"-\", \"*\", \"/\"]):\n", + " super().__init__(pseudo_label_list)\n", + "\n", " def _valid_candidate(self, formula):\n", " if len(formula) % 2 == 0:\n", " return False\n", @@ -89,98 +352,98 @@ " if i % 2 != 0 and formula[i] not in [\"+\", \"-\", \"*\", \"/\"]:\n", " return False\n", " return True\n", - "\n", + " \n", + " # Implement the deduction function\n", " def logic_forward(self, formula):\n", " if not self._valid_candidate(formula):\n", - " return np.info\n", + " return np.inf\n", " return eval(\"\".join(formula))\n", "\n", - "\n", - "kb = HWF_KB(\n", - " pseudo_label_list=[\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"+\", \"-\", \"*\", \"/\"],\n", - " max_err=1e-10,\n", - " use_cache=False,\n", - ")\n", - "reasoner = Reasoner(kb, dist_func=\"confidence\")" + "kb = HwfKB()" ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ - "### Machine Learning Part" + "The knowledge base can perform logical reasoning (both deductive reasoning and abductive reasoning). Below is an example of performing (deductive) reasoning, and users can refer to [Documentation]() for details of abductive reasoning." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Reasoning result of pseudo-label example ['1', '-', '2', '*', '5'] is -9.\n" + ] + } + ], "source": [ - "# Initialize necessary component for machine learning part\n", - "cls = SymbolNet(num_classes=len(kb.pseudo_label_list), image_size=(45, 45, 1))\n", - "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n", - "loss_fn = nn.CrossEntropyLoss()\n", - "optimizer = torch.optim.Adam(cls.parameters(), lr=0.001, betas=(0.9, 0.99))" + "pseudo_label_example = [\"1\", \"-\", \"2\", \"*\", \"5\"]\n", + "reasoning_result = kb.logic_forward(pseudo_label_example)\n", + "print(f\"Reasoning result of pseudo-label example {pseudo_label_example} is {reasoning_result}.\")" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "# Initialize BasicNN\n", - "# The function of BasicNN is to wrap NN models into the form of an sklearn estimator\n", - "base_model = BasicNN(\n", - " model=cls,\n", - " loss_fn=loss_fn,\n", - " optimizer=optimizer,\n", - " device=device,\n", - " save_interval=1,\n", - " save_dir=weights_dir,\n", - " batch_size=128,\n", - " num_epochs=3,\n", - ")" + "Note: In addition to building a knowledge base based on `KBBase`, we can also establish a knowledge base with a ground KB using `GroundKB`. The corresponding code can be found in the `main.py` file. Those interested are encouraged to examine it for further insights.\n", + "\n", + "Note: Also, when building the knowledge base, we can also set the `max_err` parameter during initialization, which is shown in the `main.py` file. This parameter specifies the upper tolerance limit when comparing the similarity between a pseudo-label example’s reasoning result and the ground truth during abductive reasoning, with a default value of 1e-10." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then, we create a reasoner by instantiating the class ``Reasoner``. Due to the indeterminism of abductive reasoning, there could be multiple candidates compatible to the knowledge base. When this happens, reasoner can minimize inconsistencies between the knowledge base and pseudo-labels predicted by the learning part, and then return only one candidate that has the highest consistency." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ - "# Initialize ABL model\n", - "# The main function of the ABL model is to serialize data and\n", - "# provide a unified interface for different machine learning models\n", - "model = ABLModel(base_model)" + "reasoner = Reasoner(kb)" ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ - "### Metric" + "Note: During creating reasoner, the definition of \"consistency\" can be customized within the `dist_func` parameter. In the code above, we employ a consistency measurement based on confidence, which calculates the consistency between the data example and candidates based on the confidence derived from the predicted probability. In `main.py`, we provide options for utilizing other forms of consistency measurement.\n", + "\n", + "Note: Also, during process of inconsistency minimization, we can leverage [ZOOpt library](https://github.com/polixir/ZOOpt) for acceleration. Options for this are also available in `main.py`. Those interested are encouraged to explore these features." ] }, { - "cell_type": "code", - "execution_count": null, + "attachments": {}, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "# Add metric\n", - "metric_list = [SymbolMetric(prefix=\"hwf\"), ReasoningMetric(kb=kb, prefix=\"hwf\")]" + "## Building Evaluation Metrics" ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ - "### Dataset" + "Next, we set up evaluation metrics. These metrics will be used to evaluate the model performance during training and testing. Specifically, we use `SymbolMetric` and `ReasoningMetric`, which are used to evaluate the accuracy of the machine learning model’s predictions and the accuracy of the final reasoning results, respectively." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "metric_list = [SymbolMetric(prefix=\"hwf\"), ReasoningMetric(kb=kb, prefix=\"hwf\")]" ] }, { @@ -188,24 +451,25 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Bridge Machine Learning and Logic Reasoning" + "## Bridge Learning and Reasoning\n", + "\n", + "Now, the last step is to bridge the learning and reasoning part. We proceed this step by creating an instance of `SimpleBridge`." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ - "bridge = SimpleBridge(model=model, reasoner=reasoner, metric_list=metric_list)" + "bridge = SimpleBridge(model, reasoner, metric_list)" ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ - "### Train and Test" + "Perform training and testing by invoking the `train` and `test` methods of `SimpleBridge`." ] }, { @@ -214,14 +478,19 @@ "metadata": {}, "outputs": [], "source": [ - "bridge.train(train_data, train_data, loops=3, segment_size=1000, save_interval=1, save_dir=weights_dir)\n", + "# Build logger\n", + "print_log(\"Abductive Learning on the HWF example.\", logger=\"current\")\n", + "log_dir = ABLLogger.get_current_instance().log_dir\n", + "weights_dir = osp.join(log_dir, \"weights\")\n", + "\n", + "bridge.train(train_data, train_data, loops=3, segment_size=1000, save_dir=weights_dir)\n", "bridge.test(test_data)" ] } ], "metadata": { "kernelspec": { - "display_name": "ABL", + "display_name": "abl", "language": "python", "name": "python3" }, @@ -240,7 +509,7 @@ "orig_nbformat": 4, "vscode": { "interpreter": { - "hash": "fb6f4ceeabb9a733f366948eb80109f83aedf798cc984df1e68fb411adb27d58" + "hash": "9c8d454494e49869a4ee4046edcac9a39ff683f7d38abf0769f648402670238e" } } }, diff --git a/examples/hwf/main.py b/examples/hwf/main.py index c003717..6954a6b 100644 --- a/examples/hwf/main.py +++ b/examples/hwf/main.py @@ -1,32 +1,26 @@ -# %% -import torch -import numpy as np -import torch.nn as nn +import os import os.path as osp +import argparse -from abl.reasoning import Reasoner, KBBase -from abl.learning import BasicNN, ABLModel -from abl.bridge import SimpleBridge -from abl.evaluation import SymbolMetric, ReasoningMetric -from abl.utils import ABLLogger, print_log +import numpy as np +import torch +from torch import nn +from examples.hwf.datasets import get_dataset from examples.models.nn import SymbolNet -from examples.hwf.datasets.get_dataset import get_hwf - -# %% -# Initialize logger and print basic information -print_log("Abductive Learning on the HWF example.", logger="current") - -# Retrieve the directory of the Log file and define the directory for saving the model weights. -log_dir = ABLLogger.get_current_instance().log_dir -weights_dir = osp.join(log_dir, "weights") +from abl.learning import ABLModel, BasicNN +from abl.reasoning import KBBase, GroundKB, Reasoner +from abl.evaluation import ReasoningMetric, SymbolMetric +from abl.utils import ABLLogger, print_log +from abl.bridge import SimpleBridge -# %% [markdown] -# ### Logic Part +class HwfKB(KBBase): + def __init__(self, + pseudo_label_list=["1", "2", "3", "4", "5", "6", "7", "8", "9", "+", "-", "*", "/"], + max_err=1e-10, + ): + super().__init__(pseudo_label_list, max_err) -# %% -# Initialize knowledge base and reasoner -class HWF_KB(KBBase): def _valid_candidate(self, formula): if len(formula) % 2 == 0: return False @@ -36,76 +30,118 @@ class HWF_KB(KBBase): if i % 2 != 0 and formula[i] not in ["+", "-", "*", "/"]: return False return True - + + # Implement the deduction function def logic_forward(self, formula): if not self._valid_candidate(formula): - return np.info + return np.inf return eval("".join(formula)) +class HwfGroundKB(GroundKB): + def __init__(self, + pseudo_label_list=["1", "2", "3", "4", "5", "6", "7", "8", "9", "+", "-", "*", "/"], + GKB_len_list=[1,3,5,7], + max_err=1e-10, + ): + super().__init__(pseudo_label_list, GKB_len_list, max_err) -kb = HWF_KB( - pseudo_label_list=["1", "2", "3", "4", "5", "6", "7", "8", "9", "+", "-", "*", "/"], - max_err=1e-10, - use_cache=False, -) -reasoner = Reasoner(kb, dist_func="confidence") - -# %% [markdown] -# ### Machine Learning Part - -# %% -# Initialize necessary component for machine learning part -cls = SymbolNet(num_classes=len(kb.pseudo_label_list), image_size=(45, 45, 1)) -device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") -loss_fn = nn.CrossEntropyLoss() -optimizer = torch.optim.Adam(cls.parameters(), lr=0.001, betas=(0.9, 0.99)) - -# %% -# Initialize BasicNN -# The function of BasicNN is to wrap NN models into the form of an sklearn estimator -base_model = BasicNN( - model=cls, - loss_fn=loss_fn, - optimizer=optimizer, - device=device, - save_interval=1, - save_dir=weights_dir, - batch_size=128, - num_epochs=3, -) - -# %% -# Initialize ABL model -# The main function of the ABL model is to serialize data and -# provide a unified interface for different machine learning models -model = ABLModel(base_model) - -# %% [markdown] -# ### Metric - -# %% -# Add metric -metric_list = [SymbolMetric(prefix="hwf"), ReasoningMetric(kb=kb, prefix="hwf")] - -# %% [markdown] -# ### Dataset - -# %% -# Get training and testing data -train_data = get_hwf(train=True, get_pseudo_label=True) -test_data = get_hwf(train=False, get_pseudo_label=True) - -# %% [markdown] -# ### Bridge Machine Learning and Logic Reasoning - -# %% -bridge = SimpleBridge(model=model, reasoner=reasoner, metric_list=metric_list) - -# %% [markdown] -# ### Train and Test - -# %% -bridge.train(train_data, train_data, loops=3, segment_size=1000, save_interval=1, save_dir=weights_dir) -bridge.test(test_data) - + def _valid_candidate(self, formula): + if len(formula) % 2 == 0: + return False + for i in range(len(formula)): + if i % 2 == 0 and formula[i] not in ["1", "2", "3", "4", "5", "6", "7", "8", "9"]: + return False + if i % 2 != 0 and formula[i] not in ["+", "-", "*", "/"]: + return False + return True + + # Implement the deduction function + def logic_forward(self, formula): + if not self._valid_candidate(formula): + return np.inf + return eval("".join(formula)) +def main(): + parser = argparse.ArgumentParser(description='MNIST Addition example') + parser.add_argument('--no-cuda', action='store_true', default=False, + help='disables CUDA training') + parser.add_argument('--epochs', type=int, default=3, + help='number of epochs in each learning loop iteration (default : 3)') + parser.add_argument('--lr', type=float, default=1e-3, + help='base learning rate (default : 0.001)') + parser.add_argument('--weight-decay', type=int, default=3e-2, + help='weight decay value (default : 0.03)') + parser.add_argument('--batch-size', type=int, default=128, + help='batch size (default : 128)') + parser.add_argument('--loops', type=int, default=5, + help='number of loop iterations (default : 5)') + parser.add_argument('--segment_size', type=int or float, default=1000, + help='segment size (default : 1000)') + parser.add_argument('--save_interval', type=int, default=1, + help='save interval (default : 1)') + parser.add_argument('--max-revision', type=int or float, default=-1, + help='maximum revision in reasoner (default : -1)') + parser.add_argument('--require-more-revision', type=int, default=5, + help='require more revision in reasoner (default : 0)') + parser.add_argument("--ground", action="store_true", default=False, + help='use GroundKB (default: False)') + parser.add_argument("--max-err", type=float, default=1e-10, + help='max tolerance during abductive reasoning (default : 1e-10)') + + args = parser.parse_args() + + ### Working with Data + train_data = get_dataset(train=True, get_pseudo_label=True) + test_data = get_dataset(train=False, get_pseudo_label=True) + + ### Building the Learning Part + # Build necessary components for BasicNN + cls = SymbolNet(num_classes=14, image_size=(45, 45, 1)) + loss_fn = nn.CrossEntropyLoss() + optimizer = torch.optim.Adam(cls.parameters(), lr=args.lr) + use_cuda = not args.no_cuda and torch.cuda.is_available() + device = torch.device("cuda" if use_cuda else "cpu") + + # Build BasicNN + base_model = BasicNN( + cls, + loss_fn, + optimizer, + device=device, + batch_size=args.batch_size, + num_epochs=args.epochs, + ) + + # Build ABLModel + model = ABLModel(base_model) + + ### Building the Reasoning Part + # Build knowledge base + if args.ground: + kb = HwfGroundKB() + else: + kb = HwfKB() + + # Create reasoner + reasoner = Reasoner(kb, max_revision=args.max_revision, require_more_revision=args.require_more_revision) + + ### Building Evaluation Metrics + metric_list = [SymbolMetric(prefix="hwf"), ReasoningMetric(kb=kb, prefix="hwf")] + + ### Bridge Learning and Reasoning + bridge = SimpleBridge(model, reasoner, metric_list) + + # Build logger + print_log("Abductive Learning on the HWF example.", logger="current") + + # Retrieve the directory of the Log file and define the directory for saving the model weights. + log_dir = ABLLogger.get_current_instance().log_dir + weights_dir = osp.join(log_dir, "weights") + + # Train and Test + bridge.train(train_data, loops=args.loops, segment_size=args.segment_size, save_interval=args.save_interval, save_dir=weights_dir) + bridge.test(test_data) + + +if __name__ == "__main__": + main() diff --git a/examples/hwf/requirements.txt b/examples/hwf/requirements.txt index 11aaa3a..ee7cc5e 100644 --- a/examples/hwf/requirements.txt +++ b/examples/hwf/requirements.txt @@ -1,2 +1,3 @@ abl -gdown \ No newline at end of file +gdown +matplotlib \ No newline at end of file diff --git a/examples/mnist_add/README.md b/examples/mnist_add/README.md index cdf92eb..51bdaad 100644 --- a/examples/mnist_add/README.md +++ b/examples/mnist_add/README.md @@ -1,6 +1,6 @@ # MNIST Addition Example -This example shows a simple implementation of [MNIST Addition](https://link) task, where the inputs are pairs of MNIST handwritten images, and the outputs are their sums. +This example shows a simple implementation of [MNIST Addition](https://arxiv.org/abs/1805.10872) task, where pairs of MNIST handwritten images and their sums are given, alongwith a domain knowledge base containing information on how to perform addition operations. The task is to recognize the digits of handwritten images and accurately determine their sum. ## Run diff --git a/examples/mnist_add/mnist_add.ipynb b/examples/mnist_add/mnist_add.ipynb index 0ecc210..6f407aa 100644 --- a/examples/mnist_add/mnist_add.ipynb +++ b/examples/mnist_add/mnist_add.ipynb @@ -8,7 +8,7 @@ "\n", "This notebook shows an implementation of [MNIST Addition](https://arxiv.org/abs/1805.10872). In this task, pairs of MNIST handwritten images and their sums are given, alongwith a domain knowledge base containing information on how to perform addition operations. The task is to recognize the digits of handwritten images and accurately determine their sum.\n", "\n", - "Intuitively, we first use a machine learning model (learning part) to convert the input images to digits (we call them pseudo-labels), and then use the knowledge base (reasoning part) to calculate the sum of these digits. Since we do not have ground-truth of the digits, in abductive learning, the reasoning part will leverage domain knowledge and revise the initial digits yielded by the learning part through abductive reasoning. This process enables us to further update the machine learning model." + "Intuitively, we first use a machine learning model (learning part) to convert the input images to digits (we call them pseudo-labels), and then use the knowledge base (reasoning part) to calculate the sum of these digits. Since we do not have ground-truth of the digits, in Abductive Learning, the reasoning part will leverage domain knowledge and revise the initial digits yielded by the learning part through abductive reasoning. This process enables us to further update the machine learning model." ] }, { @@ -54,7 +54,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Both `train_data` and `test_data` have the same structures: tuples with three components: X (list where each element is a pair of images), gt_pseudo_label (list where each element is a pair of digits) and Y (list where each element is the sum of each digit pair). The length and structures of datasets are illustrated as follows.\n", + "`train_data` and `test_data` share identical structures: tuples with three components: X (list where each element is a list of two images), gt_pseudo_label (list where each element is a list of two digits, i.e., pseudo-labels) and Y (list where each element is the sum of the two digits). The length and structures of datasets are illustrated as follows.\n", "\n", "Note: ``gt_pseudo_label`` is only used to evaluate the performance of the learning part but not to train the model." ] @@ -70,14 +70,12 @@ "text": [ "Both train_data and test_data consist of 3 components: X, gt_pseudo_label, Y\n", "\n", - "\n", "Length of X, gt_pseudo_label, Y in train_data: 30000, 30000, 30000\n", "Length of X, gt_pseudo_label, Y in test_data: 5000, 5000, 5000\n", "\n", - "\n", - "Type of X: list, and type of each element in X: ['Tensor', 'Tensor'].\n", - "Type of gt_pseudo_label: list, and type of each element in gt_pseudo_label: ['int', 'int'].\n", - "Type of Y: list, and type of each element in Y: int.\n" + "X is a list, with each element being a list of 2 Tensor.\n", + "gt_pseudo_label is a list, with each element being a list of 2 int.\n", + "Y is a list, with each element being a int.\n" ] } ], @@ -88,18 +86,24 @@ " return [describe_structure(item) for item in lst]\n", "\n", "print(f\"Both train_data and test_data consist of 3 components: X, gt_pseudo_label, Y\")\n", - "print(\"\\n\")\n", + "print()\n", "train_X, train_gt_pseudo_label, train_Y = train_data\n", - "print(f\"Length of X, gt_pseudo_label, Y in train_data: {len(train_X)}, {len(train_gt_pseudo_label)}, {len(train_Y)}\")\n", + "print(f\"Length of X, gt_pseudo_label, Y in train_data: \" +\n", + " f\"{len(train_X)}, {len(train_gt_pseudo_label)}, {len(train_Y)}\")\n", "test_X, test_gt_pseudo_label, test_Y = test_data\n", - "print(f\"Length of X, gt_pseudo_label, Y in test_data: {len(test_X)}, {len(test_gt_pseudo_label)}, {len(test_Y)}\")\n", - "print(\"\\n\")\n", - "structure_X = describe_structure(train_X[0])\n", - "structure_gt_pseudo_label = describe_structure(train_gt_pseudo_label[0])\n", - "structure_Y = describe_structure(train_Y[0])\n", - "print(f\"Type of X: {type(train_X).__name__}, and type of each element in X: {structure_X}.\")\n", - "print(f\"Type of gt_pseudo_label: {type(train_gt_pseudo_label).__name__}, and type of each element in gt_pseudo_label: {structure_gt_pseudo_label}.\")\n", - "print(f\"Type of Y: {type(train_Y).__name__}, and type of each element in Y: {structure_Y}.\")" + "print(f\"Length of X, gt_pseudo_label, Y in test_data: \" +\n", + " f\"{len(test_X)}, {len(test_gt_pseudo_label)}, {len(test_Y)}\")\n", + "print()\n", + "\n", + "X_0, gt_pseudo_label_0, Y_0 = train_X[0], train_gt_pseudo_label[0], train_Y[0]\n", + "print(f\"X is a {type(train_X).__name__}, \" +\n", + " f\"with each element being a {type(X_0).__name__} \" +\n", + " f\"of {len(X_0)} {type(X_0[0]).__name__}.\")\n", + "print(f\"gt_pseudo_label is a {type(train_gt_pseudo_label).__name__}, \" +\n", + " f\"with each element being a {type(gt_pseudo_label_0).__name__} \" +\n", + " f\"of {len(gt_pseudo_label_0)} {type(gt_pseudo_label_0[0]).__name__}.\")\n", + "print(f\"Y is a {type(train_Y).__name__}, \" +\n", + " f\"with each element being a {type(Y_0).__name__}.\")" ] }, { @@ -118,7 +122,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "X in the first data example (a pair of images):\n" + "X in the first data example (a list of two images):\n" ] }, { @@ -135,23 +139,23 @@ "name": "stdout", "output_type": "stream", "text": [ - "gt_pseudo_label in the first data example (a pair of ground truth pseudo-labels): 7, 5\n", + "gt_pseudo_label in the first data example (a list of two ground truth pseudo-labels): [7, 5]\n", "Y in the first data example (their sum result): 12\n" ] } ], "source": [ - "first_X, first_gt_pseudo_label, first_Y = train_X[0], train_gt_pseudo_label[0], train_Y[0]\n", - "print(f\"X in the first data example (a pair of images):\")\n", + "X_0, gt_pseudo_label_0, Y_0 = train_X[0], train_gt_pseudo_label[0], train_Y[0]\n", + "print(f\"X in the first data example (a list of two images):\")\n", "plt.subplot(1,2,1)\n", "plt.axis('off') \n", - "plt.imshow(first_X[0].numpy().transpose(1, 2, 0))\n", + "plt.imshow(X_0[0].numpy().transpose(1, 2, 0))\n", "plt.subplot(1,2,2)\n", "plt.axis('off') \n", - "plt.imshow(first_X[1].numpy().transpose(1, 2, 0))\n", + "plt.imshow(X_0[1].numpy().transpose(1, 2, 0))\n", "plt.show()\n", - "print(f\"gt_pseudo_label in the first data example (a pair of ground truth pseudo-labels): {first_gt_pseudo_label[0]}, {first_gt_pseudo_label[1]}\")\n", - "print(f\"Y in the first data example (their sum result): {first_Y}\")" + "print(f\"gt_pseudo_label in the first data example (a list of two ground truth pseudo-labels): {gt_pseudo_label_0}\")\n", + "print(f\"Y in the first data example (their sum result): {Y_0}\")" ] }, { @@ -206,17 +210,19 @@ "name": "stdout", "output_type": "stream", "text": [ - "Predicted class index for a batch of 32 instances: np.ndarray with shape (32,)\n", - "Predicted class probabilities for a batch of 32 instances: np.ndarray with shape (32, 10)\n" + "Predicted class index for a batch of 32 instances: ndarray with shape (32,)\n", + "Predicted class probabilities for a batch of 32 instances: ndarray with shape (32, 10)\n" ] } ], "source": [ "data_instances = [torch.randn(1, 28, 28).to(device) for _ in range(32)]\n", "pred_idx = base_model.predict(X=data_instances)\n", - "print(f\"Predicted class index for a batch of 32 instances: np.ndarray with shape {pred_idx.shape}\")\n", + "print(f\"Predicted class index for a batch of 32 instances: \" +\n", + " f\"{type(pred_idx).__name__} with shape {pred_idx.shape}\")\n", "pred_prob = base_model.predict_proba(X=data_instances)\n", - "print(f\"Predicted class probabilities for a batch of 32 instances: np.ndarray with shape {pred_prob.shape}\")" + "print(f\"Predicted class probabilities for a batch of 32 instances: \" +\n", + " f\"{type(pred_prob).__name__} with shape {pred_prob.shape}\")" ] }, { @@ -251,23 +257,31 @@ "name": "stdout", "output_type": "stream", "text": [ - "Predicted class labels for the first data example: np.array with shape (2,)\n", - "Predicted class probabilities for the first data example: np.array with shape (2, 10)\n" + "Predicted class labels for the 100 data examples: \n", + "a list of length 100, and each element is a ndarray of shape (2,).\n", + "\n", + "Predicted class probabilities for the 100 data examples: \n", + "a list of length 100, and each element is a ndarray of shape (2, 10).\n" ] } ], "source": [ "from abl.structures import ListData\n", "# ListData is a data structure provided by ABL-Package that can be used to organize data examples\n", - "data_example = ListData()\n", - "data_example.X = first_X\n", - "data_example.gt_pseudo_label = first_gt_pseudo_label\n", - "data_example.Y = first_Y\n", + "data_examples = ListData()\n", + "# We use the first 100 data examples in the training set as an illustration\n", + "data_examples.X = train_X[:100]\n", + "data_examples.gt_pseudo_label = train_gt_pseudo_label[:100]\n", + "data_examples.Y = train_Y[:100]\n", "\n", - "# Perform prediction on the first data examples\n", - "prediction_result = model.predict(data_example)\n", - "print(f\"Predicted class labels for the first data example: np.array with shape {prediction_result['label'].shape}\")\n", - "print(f\"Predicted class probabilities for the first data example: np.array with shape {prediction_result['prob'].shape}\")" + "# Perform prediction on the 100 data examples\n", + "pred_label, pred_prob = model.predict(data_examples)['label'], model.predict(data_examples)['prob']\n", + "print(f\"Predicted class labels for the 100 data examples: \\n\" +\n", + " f\"a list of length {len(pred_label)}, and each element is \" +\n", + " f\"a {type(pred_label[0]).__name__} of shape {pred_label[0].shape}.\\n\")\n", + "print(f\"Predicted class probabilities for the 100 data examples: \\n\" +\n", + " f\"a list of length {len(pred_prob)}, and each element is \" +\n", + " f\"a {type(pred_prob[0]).__name__} of shape {pred_prob[0].shape}.\")" ] }, { @@ -356,7 +370,7 @@ "source": [ "Note: During creating reasoner, the definition of \"consistency\" can be customized within the `dist_func` parameter. In the code above, we employ a consistency measurement based on confidence, which calculates the consistency between the data example and candidates based on the confidence derived from the predicted probability. In `main.py`, we provide options for utilizing other forms of consistency measurement.\n", "\n", - "Also, during process of inconsistency minimization, one can leverage [ZOOpt library](https://github.com/polixir/ZOOpt) for acceleration. Options for this are also available in `main.py`. Those interested are encouraged to explore these features." + "Note: Also, during process of inconsistency minimization, one can leverage [ZOOpt library](https://github.com/polixir/ZOOpt) for acceleration. Options for this are also available in `main.py`. Those interested are encouraged to explore these features." ] }, {