diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..30b55ca --- /dev/null +++ b/.coveragerc @@ -0,0 +1,14 @@ +[report] +show_missing = True + +[run] +disable_warnings = include-ignored +include = */abl/* +omit = + */abl/__init__.py + abl/bridge/__init__.py + abl/dataset/__init__.py + abl/data/__init__.py + abl/learning/__init__.py + abl/reasoning/__init__.py + abl/utils/__init__.py \ No newline at end of file diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml new file mode 100644 index 0000000..f650f61 --- /dev/null +++ b/.github/workflows/build-and-test.yaml @@ -0,0 +1,62 @@ +name: ABL-Package-CI + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + python-version: ['3.7', '3.11'] + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Cache Python virtual environment + uses: actions/cache@v2 + with: + path: | + ~/.venv + !~/.venv/*/lib/python*/no-global-site-packages.txt + key: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-python-${{ matrix.python-version }}- + + - name: Display python version + run: python -c "import sys; print(sys.version)" + + - name: Install SWI-Prolog on Ubuntu + if: matrix.os == 'ubuntu-latest' + run: sudo apt-get install swi-prolog + - name: Install SWI-Prolog on Windows + if: matrix.os == 'windows-latest' + run: choco install swi-prolog + - name: Install SWI-Prolog on MACOS + if: matrix.os == 'macos-latest' + run: brew install swi-prolog + + - name: Install package dependencies + run: | + python -m pip install --upgrade pip + pip install pytest pytest-cov + - name: Install + run: pip install -v -e . + + - name: Run tests + run: | + pytest --cov-config=.coveragerc --cov-report=xml --cov=abl ./tests + + - name: Publish code coverage + uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: ./coverage.xml \ No newline at end of file diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml new file mode 100644 index 0000000..7386a88 --- /dev/null +++ b/.github/workflows/lint.yaml @@ -0,0 +1,24 @@ +name: flake8 Lint + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + flake8-lint: + runs-on: ubuntu-latest + name: Lint + steps: + - name: Check out source repository + uses: actions/checkout@v3 + - name: Set up Python environment + uses: actions/setup-python@v4 + with: + python-version: "3.8" + - name: flake8 Lint + uses: py-actions/flake8@v2 + with: + max-line-length: "100" + args: --ignore=E203,W503,F821,E266 \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..931fddd --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +*.pyc +examples/**/*.png +*.pk +*.pth +*.json +*.ckpt +results +raw/ +abl.egg-info/ +examples/**/*.jpg +.idea/ +build/ +docs/API/generated/ +.history \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..c2497c6 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 LAMDA + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README b/README deleted file mode 100644 index 0aa2a11..0000000 --- a/README +++ /dev/null @@ -1,24 +0,0 @@ -# ABL Package - -This is the code repository of abductive learning Package. - -## Environment dependency -... - -## Example -share_example.py and nonshare_exaple.py are examples of grounded abductive learning. - -```bash -python share_example.py -``` - - -## Authors - -- [Yu-Xuan Huang](http://www.lamda.nju.edu.cn/huangyx/) (Nanjing University) -- [](http://www.lamda.nju.edu.cn//) (Nanjing University) - - -## NOTICE -They can only be used for academic purpose. For other purposes, please contact with LAMDA Group(www.lamda.nju.edu.cn). - diff --git a/README.md b/README.md new file mode 100644 index 0000000..88ba424 --- /dev/null +++ b/README.md @@ -0,0 +1,173 @@ +[![license](https://img.shields.io/github/license/mashape/apistatus.svg?maxAge=2592000)](https://github.com/AbductiveLearning/ABL-Package/blob/Dev/LICENSE) +[![flake8 Lint](https://github.com/AbductiveLearning/ABL-Package/actions/workflows/lint.yaml/badge.svg?branch=Dev)](https://github.com/AbductiveLearning/ABL-Package/actions/workflows/lint.yaml) +[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) +[![ABL-Package-CI](https://github.com/AbductiveLearning/ABL-Package/actions/workflows/build-and-test.yaml/badge.svg?branch=Dev)](https://github.com/AbductiveLearning/ABL-Package/actions/workflows/build-and-test.yaml) + +# ABL-Package + +**ABL-Package** is an open source library for **Abductive Learning (ABL)**. +ABL is a novel paradigm that integrates machine learning and +logical reasoning in a unified framework. It is suitable for tasks +where both data and (logical) domain knowledge are available. + +Key Features of ABL-Package: + +- **Great Flexibility**: Adaptable to various machine learning modules and logical reasoning components. +- **User-Friendly**: Provide data, model, and KB, and get started with just a few lines of code. +- **High-Performance**: Optimization for high accuracy and fast training speed. + +ABL-Package encapsulates advanced ABL techniques, providing users with +an efficient and convenient package to develop dual-driven ABL systems, +which leverage the power of both data and knowledge. + +To learn how to use it, please refer to - [document](https://www.lamda.nju.edu.cn/abl_test/docs/build/html/index.html). + +## Installation + +ABL is distributed on [PyPI](https://pypi.org/) and can be installed with ``pip``: + +```bash +# (TODO) +$ pip install abl +``` + +For testing purposes, you can install it using: + +```bash +$ pip install -i https://test.pypi.org/simple/ --extra-index-url https://mirrors.nju.edu.cn/pypi/web/simple/ abl +``` + +Alternatively, to install ABL by source code, sequentially run following commands in your terminal/command line. + +```bash +$ git clone https://github.com/AbductiveLearning/ABL-Package.git +$ cd ABL-Package +$ pip install -v -e . +``` + +(Optional) If the use of a [Prolog-based knowledge base](https://www.lamda.nju.edu.cn/abl_test/docs/build/html/Intro/Reasoning.html#prolog) is necessary, the installation of [Swi-Prolog](https://www.swi-prolog.org/) is also required: + +For Linux users: + +```bash +$ sudo apt-get install swi-prolog +``` + +For Windows and Mac users, please refer to the [Swi-Prolog Install Guide](https://github.com/yuce/pyswip/blob/master/INSTALL.md). + +## Examples + +We provide several examples in `examples/`. Each example is stored in a separate folder containing a README file. + ++ [MNIST Addition](https://github.com/AbductiveLearning/ABL-Package/blob/Dev/examples/mnist_add) ++ [Handwritten Formula](https://github.com/AbductiveLearning/ABL-Package/blob/Dev/examples/hwf) ++ [Handwritten Equation Decipherment](https://github.com/AbductiveLearning/ABL-Package/tree/Dev/examples/hed) ++ [Zoo](https://github.com/AbductiveLearning/ABL-Package/tree/Dev/examples/zoo) + +## Quick Start + +We use the MNIST Addition task as a quick start example. In this task, pairs of MNIST handwritten images and their sums are given, alongwith a domain knowledge base which contain information on how to perform addition operations. Our objective is to input a pair of handwritten images and accurately determine their sum. + +### Working with Data + + +ABL-Package requires data in the format of `(X, gt_pseudo_label, Y)` where `X` is a list of input examples containing instances, `gt_pseudo_label` is the ground-truth label of each example in `X` and `Y` is the ground-truth reasoning result of each example in `X`. Note that `gt_pseudo_label` is only used to evaluate the machine learning model's performance but not to train it. + +In the MNIST Addition task, the data loading looks like: + +```python +# The 'datasets' module below is located in 'examples/mnist_add/' +from datasets import get_dataset + +# train_data and test_data are tuples in the format of (X, gt_pseudo_label, Y) +train_data = get_dataset(train=True) +test_data = get_dataset(train=False) +``` + +### Building the Learning Part + +Learning part is constructed by first defining a base model for machine learning. The ABL-Package offers considerable flexibility, supporting any base model that conforms to the scikit-learn style (which requires the implementation of fit and predict methods), or a PyTorch-based neural network (which has defined the architecture and implemented forward method). In this example, we build a simple LeNet5 network as the base model. + +```python +# The 'models' module below is located in 'examples/mnist_add/' +from models.nn import LeNet5 + +cls = LeNet5(num_classes=10) +``` + +To facilitate uniform processing, ABL-Package provides the `BasicNN` class to convert a PyTorch-based neural network into a format compatible with scikit-learn models. To construct a `BasicNN` instance, aside from the network itself, we also need to define a loss function, an optimizer, and the computing device. + +```python +​import torch +​from abl.learning import BasicNN +​ +​loss_fn = torch.nn.CrossEntropyLoss() +​optimizer = torch.optim.RMSprop(cls.parameters(), lr=0.001, alpha=0.9) +​device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +​base_model = BasicNN(model=cls, loss_fn=loss_fn, optimizer=optimizer, device=device) +``` + +The base model built above are trained to make predictions on instance-level data (e.g., a single image), while ABL deals with example-level data. To bridge this gap, we wrap the base_model into an instance of `ABLModel`. This class serves as a unified wrapper for base models, facilitating the learning part to train, test, and predict on example-level data, (e.g., images that comprise an equation). + +```python +from abl.learning import ABLModel +​ +​model = ABLModel(base_model) +``` + +### Building the Reasoning Part + +To build the reasoning part, we first define a knowledge base by creating a subclass of `KBBase`. In the subclass, we initialize the `pseudo_label_list` parameter and override the `logic_forward` method, which specifies how to perform (deductive) reasoning that processes pseudo-labels of an example to the corresponding reasoning result. Specifically for the MNIST Addition task, this `logic_forward` method is tailored to execute the sum operation. + +```python +from abl.reasoning import KBBase +​ +class AddKB(KBBase): + def __init__(self, pseudo_label_list=list(range(10))): + super().__init__(pseudo_label_list) + +​ def logic_forward(self, nums): + return sum(nums) +​ +kb = AddKB() +``` + +Next, we create a reasoner by instantiating the class `Reasoner`, passing the knowledge base as a parameter. Due to the indeterminism of abductive reasoning, there could be multiple candidate pseudo-labels compatible to the knowledge base. In such scenarios, the reasoner can minimize inconsistency and return the pseudo-label with the highest consistency. + +```python +from abl.reasoning import Reasoner +​ +reasoner = Reasoner(kb) +``` + +### Building Evaluation Metrics + +ABL-Package provides two basic metrics, namely `SymbolAccuracy` and `ReasoningMetric`, which are used to evaluate the accuracy of the machine learning model's predictions and the accuracy of the `logic_forward` results, respectively. + +```python +from abl.data.evaluation import ReasoningMetric, SymbolAccuracy +​ +metric_list = [SymbolAccuracy(), ReasoningMetric(kb=kb)] +``` + +### Bridging Learning and Reasoning + +Now, we use `SimpleBridge` to combine learning and reasoning in a +unified ABL framework. + +```python +from abl.bridge import SimpleBridge +​ +bridge = SimpleBridge(model, reasoner, metric_list) +``` + +Finally, we proceed with training and testing. + +```python +​bridge.train(train_data, loops=1, segment_size=0.01) +bridge.test(test_data) +``` + +## References + +For more information about ABL, please refer to: [Zhou, 2019](https://link.springer.com/epdf/10.1007/s11432-018-9801-4?author_access_token=jgJe1Ox3Mk-K7ORSnX7jtfe4RwlQNchNByi7wbcMAY7_PxTx-xNLP7Lp0mIZ04ORp3VG4wioIBHSCIAO3B_TBJkj87YzapmdnYVSQvgBIO3aEpQWppxZG25KolINetygc2W_Cj2gtoBdiG_J1hU3pA==) and [Zhou and Huang, 2022](https://www.lamda.nju.edu.cn/publication/chap_ABL.pdf). \ No newline at end of file diff --git a/abducer/abducer_base.py b/abducer/abducer_base.py deleted file mode 100644 index 13e1f75..0000000 --- a/abducer/abducer_base.py +++ /dev/null @@ -1,104 +0,0 @@ -# coding: utf-8 -#================================================================# -# Copyright (C) 2021 Freecss All rights reserved. -# -# File Name ：abducer_base.py -# Author ：freecss -# Email ：karlfreecss@gmail.com -# Created Date ：2021/06/03 -# Description ： -# -#================================================================# - -import abc -from abducer.kb import ClsKB, RegKB -#from kb import ClsKB, RegKB -import numpy as np - -def hamming_dist(A, B): - B = np.array(B) - A = np.expand_dims(A, axis = 0).repeat(axis=0, repeats=(len(B))) - return np.sum(A != B, axis = 1) - -def confidence_dist(A, B): - B = np.array(B) - - #print(A) - A = np.clip(A, 1e-9, 1) - A = np.expand_dims(A, axis=0) - A = A.repeat(axis=0, repeats=(len(B))) - rows = np.array(range(len(B))) - rows = np.expand_dims(rows, axis = 1).repeat(axis = 1, repeats = len(B[0])) - cols = np.array(range(len(B[0]))) - cols = np.expand_dims(cols, axis = 0).repeat(axis = 0, repeats = len(B)) - return 1 - np.prod(A[rows, cols, B], axis = 1) - -class AbducerBase(abc.ABC): - def __init__(self, kb, dist_func = "hamming", pred_res_parse = None): - self.kb = kb - if dist_func == "hamming": - dist_func = hamming_dist - elif dist_func == "confidence": - dist_func = confidence_dist - self.dist_func = dist_func - if pred_res_parse is None: - pred_res_parse = lambda x : x["cls"] - self.pred_res_parse = pred_res_parse - - def abduce(self, data, max_address_num, require_more_address, length = -1): - pred_res, ans = data - - if length == -1: - length = len(pred_res) - - candidates = self.kb.get_candidates(ans, length) - pred_res = np.array(pred_res) - - cost_list = self.dist_func(pred_res, candidates) - address_num = np.min(cost_list) - threshold = min(address_num + require_more_address, max_address_num) - idxs = np.where(cost_list <= address_num+require_more_address)[0] - - #return [candidates[idx] for idx in idxs], address_num - if len(idxs) > 1: - return None - return [candidates[idx] for idx in idxs][0] - - def batch_abduce(self, Y, C, max_address_num = 3, require_more_address = 0): - return [ - self.abduce((y, c), max_address_num, require_more_address)\ - for y, c in zip(self.pred_res_parse(Y), C) - ] - - def __call__(self, Y, C, max_address_num = 3, require_more_address = 0): - return batch_abduce(Y, C, max_address_num, require_more_address) - -if __name__ == "__main__": - #["1+1", "0+1", "1+0", "2+0"] - X = [[1,3,1], [0,3,1], [1,2,0], [3,2,0]] - Y = [2, 1, 1, 2] - kb = RegKB(X, Y) - - abd = AbducerBase(kb) - res = abd.abduce(([0,2,0], None), 1, 0) - print(res) - res = abd.abduce(([0, 2, 0], 0.99), 1, 0) - print(res) - - A = np.array([[0.5, 0.25, 0.25, 0], [0.3, 0.3, 0.3, 0.1], [0.1, 0.2, 0.3, 0.4]]) - B = [[1, 2, 3], [0, 1, 3]] - res = confidence_dist(A, B) - print(res) - - A = np.array([[0.5, 0.25, 0.25, 0], [0.3, 1.0, 0.3, 0.1], [0.1, 0.2, 0.3, 1.0]]) - B = [[0, 1, 3]] - res = confidence_dist(A, B) - print(res) - - kb_str = ['10010001011', '00010001100', '00111101011', '11101000011', '11110011001', '11111010001', '10001010010', '11100100001', '10001001100', '11011010001', '00110000100', '11000000111', '01110111111', '11000101100', '10101011010', '00000110110', '11111110010', '11100101100', '10111001111', '10000101100', '01001011101', '01001110000', '01110001110', '01010010001', '10000100010', '01001011011', '11111111100', '01011101101', '00101110101', '11101001101', '10010110000', '10000000011'] - X = [[int(c) for c in s] for s in kb_str] - kb = RegKB(X, len(X) * [None]) - - abd = AbducerBase(kb) - res = abd.abduce(((1, 1, 0, 1, 1, 0, 1, 0, 0, 0, 1), None), 1, 0) - print(res) diff --git a/abducer/kb.py b/abducer/kb.py deleted file mode 100644 index d33be07..0000000 --- a/abducer/kb.py +++ /dev/null @@ -1,137 +0,0 @@ -# coding: utf-8 -#================================================================# -# Copyright (C) 2021 LAMDA All rights reserved. -# -# File Name ：kb.py -# Author ：freecss -# Email ：karlfreecss@gmail.com -# Created Date ：2021/06/03 -# Description ： -# -#================================================================# - -import abc -import bisect -import copy -import numpy as np - -from collections import defaultdict - -class KBBase(abc.ABC): - def __init__(self, X = None, Y = None): - pass - - def get_candidates(self, key = None, length = None): - pass - - def get_all_candidates(self): - pass - - def _length(self, length): - if length is None: - length = list(self.base.keys()) - if type(length) is int: - length = [length] - return length - - def __len__(self): - pass - -class ClsKB(KBBase): - def __init__(self, X, Y = None): - super().__init__() - self.base = {} - - if X is None: - return - - if Y is None: - Y = [None] * len(X) - - for x, y in zip(X, Y): - self.base.setdefault(len(x), defaultdict(list))[y].append(np.array(x)) - - def get_candidates(self, key, length = None): - if key is None: - return self.get_all_candidates() - - length = self._length(length) - - return sum([self.base[l][key] for l in length], []) - - def get_all_candidates(self): - return sum([sum(v.values(), []) for v in self.base.values()], []) - - def _dict_len(self, dic): - return sum(len(c) for c in dic.values()) - - def __len__(self): - return sum(self._dict_len(v) for v in self.base.values()) - -class RegKB(KBBase): - def __init__(self, X, Y = None): - super().__init__() - tmp_dict = {} - for x, y in zip(X, Y): - tmp_dict.setdefault(len(x), defaultdict(list))[y].append(np.array(x)) - - self.base = {} - for l in tmp_dict.keys(): - data = sorted(list(zip(tmp_dict[l].keys(), tmp_dict[l].values()))) - X = [x for y, x in data] - Y = [y for y, x in data] - self.base[l] = (X, Y) - - def get_candidates(self, key, length = None): - if key is None: - return self.get_all_candidates() - - length = self._length(length) - - min_err = 999999 - candidates = [] - for l in length: - X, Y = self.base[l] - - idx = bisect.bisect_left(Y, key) - begin = max(0, idx - 1) - end = min(idx + 2, len(X)) - - for idx in range(begin, end): - err = abs(Y[idx] - key) - if abs(err - min_err) < 1e-9: - candidates.extend(X[idx]) - elif err < min_err: - candidates = copy.deepcopy(X[idx]) - min_err = err - return candidates - - def get_all_candidates(self): - return sum([sum(D[0], []) for D in self.base.values()], []) - - def __len__(self): - return sum([sum(len(x) for x in D[0]) for D in self.base.values()]) - -if __name__ == "__main__": - X = ["1+1", "0+1", "1+0", "2+0", "1+0+1"] - Y = [2, 1, 1, 2, 2] - kb = ClsKB(X, Y) - print(len(kb)) - res = kb.get_candidates(2, 5) - print(res) - res = kb.get_candidates(2, 3) - print(res) - res = kb.get_candidates(None) - print(res) - - X = ["1+1", "0+1", "1+0", "2+0", "1+0.5", "0.75+0.75"] - Y = [2, 1, 1, 2, 1.5, 1.5] - kb = RegKB(X, Y) - print(len(kb)) - res = kb.get_candidates(1.6) - print(res) - res = kb.get_candidates(1.6, length = 9) - print(res) - res = kb.get_candidates(None) - print(res) - diff --git a/abl/__init__.py b/abl/__init__.py new file mode 100644 index 0000000..60ab54f --- /dev/null +++ b/abl/__init__.py @@ -0,0 +1,9 @@ +from . import bridge, data, learning, reasoning, utils + +__all__ = [ + "bridge", + "data", + "learning", + "reasoning", + "utils", +] diff --git a/abl/bridge/__init__.py b/abl/bridge/__init__.py new file mode 100644 index 0000000..502a118 --- /dev/null +++ b/abl/bridge/__init__.py @@ -0,0 +1,4 @@ +from .base_bridge import BaseBridge +from .simple_bridge import SimpleBridge + +__all__ = ["BaseBridge", "SimpleBridge"] diff --git a/abl/bridge/base_bridge.py b/abl/bridge/base_bridge.py new file mode 100644 index 0000000..8ec0ebd --- /dev/null +++ b/abl/bridge/base_bridge.py @@ -0,0 +1,89 @@ +from abc import ABCMeta, abstractmethod +from typing import Any, List, Optional, Tuple, Union + +from ..data.structures import ListData +from ..learning import ABLModel +from ..reasoning import Reasoner + + +class BaseBridge(metaclass=ABCMeta): + """ + A base class for bridging learning and reasoning parts. + + This class provides necessary methods that need to be overridden in subclasses + to construct a typical pipeline of Abductive Learning (corresponding to ``train``), + which involves the following four methods: + + - predict: Predict class indices on the given data examples. + - idx_to_pseudo_label: Map indices into pseudo-labels. + - abduce_pseudo_label: Revise pseudo-labels based on abdutive reasoning. + - pseudo_label_to_idx: Map revised pseudo-labels back into indices. + + Parameters + ---------- + model : ABLModel + The machine learning model wrapped in ``ABLModel``, which is mainly used for + prediction and model training. + reasoner : Reasoner + The reasoning part wrapped in ``Reasoner``, which is used for pseudo-label revision. + """ + + def __init__(self, model: ABLModel, reasoner: Reasoner) -> None: + if not isinstance(model, ABLModel): + raise TypeError( + "Expected an instance of ABLModel, but received type: {}".format(type(model)) + ) + if not isinstance(reasoner, Reasoner): + raise TypeError( + "Expected an instance of Reasoner, but received type: {}".format(type(reasoner)) + ) + + self.model = model + self.reasoner = reasoner + + @abstractmethod + def predict(self, data_examples: ListData) -> Tuple[List[List[Any]], List[List[Any]]]: + """Placeholder for predicting class indices from input.""" + + @abstractmethod + def abduce_pseudo_label(self, data_examples: ListData) -> List[List[Any]]: + """Placeholder for revising pseudo-labels based on abdutive reasoning.""" + + @abstractmethod + def idx_to_pseudo_label(self, data_examples: ListData) -> List[List[Any]]: + """Placeholder for mapping indices to pseudo-labels.""" + + @abstractmethod + def pseudo_label_to_idx(self, data_examples: ListData) -> List[List[Any]]: + """Placeholder for mapping pseudo-labels to indices.""" + + def filter_pseudo_label(self, data_examples: ListData) -> List[List[Any]]: + """Default filter function for pseudo-label.""" + non_empty_idx = [ + i + for i in range(len(data_examples.abduced_pseudo_label)) + if data_examples.abduced_pseudo_label[i] + ] + data_examples.update(data_examples[non_empty_idx]) + return data_examples + + @abstractmethod + def train( + self, + train_data: Union[ListData, Tuple[List[List[Any]], Optional[List[List[Any]]], List[Any]]], + ): + """Placeholder for training loop of ABductive Learning.""" + + @abstractmethod + def valid( + self, + val_data: Union[ListData, Tuple[List[List[Any]], Optional[List[List[Any]]], List[Any]]], + ) -> None: + """Placeholder for model test.""" + + @abstractmethod + def test( + self, + test_data: Union[ListData, Tuple[List[List[Any]], Optional[List[List[Any]]], List[Any]]], + ) -> None: + """Placeholder for model validation.""" diff --git a/abl/bridge/simple_bridge.py b/abl/bridge/simple_bridge.py new file mode 100644 index 0000000..f8a1c9a --- /dev/null +++ b/abl/bridge/simple_bridge.py @@ -0,0 +1,356 @@ +import os.path as osp +from typing import Any, List, Optional, Tuple, Union + +from numpy import ndarray + +from ..data.evaluation import BaseMetric +from ..data.structures import ListData +from ..learning import ABLModel +from ..reasoning import Reasoner +from ..utils import print_log +from .base_bridge import BaseBridge + + +class SimpleBridge(BaseBridge): + """ + A basic implementation for bridging machine learning and reasoning parts. + + This class implements the typical pipeline of Abductive Learning, which involves + the following five steps: + + - Predict class probabilities and indices for the given data examples. + - Map indices into pseudo-labels. + - Revise pseudo-labels based on abdutive reasoning. + - Map the revised pseudo-labels to indices. + - Train the model. + + Parameters + ---------- + model : ABLModel + The machine learning model wrapped in ``ABLModel``, which is mainly used for + prediction and model training. + reasoner : Reasoner + The reasoning part wrapped in ``Reasoner``, which is used for pseudo-label revision. + metric_list : List[BaseMetric] + A list of metrics used for evaluating the model's performance. + """ + + def __init__( + self, + model: ABLModel, + reasoner: Reasoner, + metric_list: List[BaseMetric], + ) -> None: + super().__init__(model, reasoner) + self.metric_list = metric_list + + def predict(self, data_examples: ListData) -> Tuple[List[ndarray], List[ndarray]]: + """ + Predict class indices and probabilities (if ``predict_proba`` is implemented in + ``self.model.base_model``) on the given data examples. + + Parameters + ---------- + data_examples : ListData + Data examples on which predictions are to be made. + + Returns + ------- + Tuple[List[ndarray], List[ndarray]] + A tuple containing lists of predicted indices and probabilities. + """ + self.model.predict(data_examples) + return data_examples.pred_idx, data_examples.pred_prob + + def abduce_pseudo_label(self, data_examples: ListData) -> List[List[Any]]: + """ + Revise predicted pseudo-labels of the given data examples using abduction. + + Parameters + ---------- + data_examples : ListData + Data examples containing predicted pseudo-labels. + + Returns + ------- + List[List[Any]] + A list of abduced pseudo-labels for the given data examples. + """ + self.reasoner.batch_abduce(data_examples) + return data_examples.abduced_pseudo_label + + def idx_to_pseudo_label(self, data_examples: ListData) -> List[List[Any]]: + """ + Map indices of data examples into pseudo-labels. + + Parameters + ---------- + data_examples : ListData + Data examples containing the indices. + + Returns + ------- + List[List[Any]] + A list of pseudo-labels converted from indices. + """ + pred_idx = data_examples.pred_idx + data_examples.pred_pseudo_label = [ + [self.reasoner.idx_to_label[_idx] for _idx in sub_list] for sub_list in pred_idx + ] + return data_examples.pred_pseudo_label + + def pseudo_label_to_idx(self, data_examples: ListData) -> List[List[Any]]: + """ + Map pseudo-labels of data examples into indices. + + Parameters + ---------- + data_examples : ListData + Data examples containing pseudo-labels. + + Returns + ------- + List[List[Any]] + A list of indices converted from pseudo-labels. + """ + abduced_idx = [ + [ + self.reasoner.label_to_idx[_abduced_pseudo_label] + for _abduced_pseudo_label in sub_list + ] + for sub_list in data_examples.abduced_pseudo_label + ] + data_examples.abduced_idx = abduced_idx + return data_examples.abduced_idx + + def data_preprocess( + self, + prefix: str, + data: Union[ListData, Tuple[List[List[Any]], Optional[List[List[Any]]], List[Any]]], + ) -> ListData: + """ + Transform data in the form of (X, gt_pseudo_label, Y) into ListData. + + Parameters + ---------- + prefix : str + A prefix indicating the type of data processing (e.g., 'train', 'test'). + data : Union[ListData, Tuple[List[List[Any]], Optional[List[List[Any]]], List[Any]]] + Data to be preprocessed. Can be ListData or a tuple of lists. + + Returns + ------- + ListData + The preprocessed ListData object. + """ + if isinstance(data, ListData): + data_examples = data + if not ( + hasattr(data_examples, "X") + and hasattr(data_examples, "gt_pseudo_label") + and hasattr(data_examples, "Y") + ): + raise ValueError( + f"{prefix}data should have X, gt_pseudo_label and Y attribute but " + f"only {data_examples.all_keys()} are provided." + ) + else: + X, gt_pseudo_label, Y = data + data_examples = ListData(X=X, gt_pseudo_label=gt_pseudo_label, Y=Y) + + return data_examples + + def concat_data_examples( + self, unlabel_data_examples: ListData, label_data_examples: Optional[ListData] + ) -> ListData: + """ + Concatenate unlabeled and labeled data examples. ``abduced_pseudo_label`` of unlabeled data + examples and ``gt_pseudo_label`` of labeled data examples will be used to train the model. + + Parameters + ---------- + unlabel_data_examples : ListData + Unlabeled data examples to concatenate. + label_data_examples : ListData, optional + Labeled data examples to concatenate, if available. + + Returns + ------- + ListData + Concatenated data examples. + """ + if label_data_examples is None: + return unlabel_data_examples + + unlabel_data_examples.X = unlabel_data_examples.X + label_data_examples.X + unlabel_data_examples.abduced_pseudo_label = ( + unlabel_data_examples.abduced_pseudo_label + label_data_examples.gt_pseudo_label + ) + unlabel_data_examples.Y = unlabel_data_examples.Y + label_data_examples.Y + return unlabel_data_examples + + def train( + self, + train_data: Union[ListData, Tuple[List[List[Any]], Optional[List[List[Any]]], List[Any]]], + label_data: Optional[ + Union[ListData, Tuple[List[List[Any]], List[List[Any]], List[Any]]] + ] = None, + val_data: Optional[ + Union[ListData, Tuple[List[List[Any]], Optional[List[List[Any]]], Optional[List[Any]]]] + ] = None, + loops: int = 50, + segment_size: Union[int, float] = 1.0, + eval_interval: int = 1, + save_interval: Optional[int] = None, + save_dir: Optional[str] = None, + ): + """ + A typical training pipeline of Abuductive Learning. + + Parameters + ---------- + train_data : Union[ListData, Tuple[List[List[Any]], Optional[List[List[Any]]], List[Any]]] + Training data should be in the form of ``(X, gt_pseudo_label, Y)`` or a ``ListData`` + object with ``X``, ``gt_pseudo_label`` and ``Y`` attributes. + - ``X`` is a list of sublists representing the input data. + - ``gt_pseudo_label`` is only used to evaluate the performance of the ``ABLModel`` but + not to train. ``gt_pseudo_label`` can be ``None``. + - ``Y`` is a list representing the ground truth reasoning result for each sublist + in ``X``. + label_data : Union[ListData, Tuple[List[List[Any]], List[List[Any]], List[Any]]], optional + Labeled data should be in the same format as ``train_data``. The only difference is + that the ``gt_pseudo_label`` in ``label_data`` should not be ``None`` and will be + utilized to train the model. Defaults to None. + val_data : Union[ListData, Tuple[List[List[Any]], Optional[List[List[Any]]], Optional[List[Any]]]], optional # noqa: E501 + Validation data should be in the same format as ``train_data``. Both ``gt_pseudo_label`` + and ``Y`` can be either None or not, which depends on the evaluation metircs in + ``self.metric_list``. If ``val_data`` is None, ``train_data`` will be used to validate + the model during training time. Defaults to None. + loops : int + Machine Learning part and Reasoning part will be iteratively optimized + for ``loops`` times, by default 50. + segment_size : Union[int, float] + Data will be split into segments of this size and data in each segment + will be used together to train the model, by default 1.0. + eval_interval : int + The model will be evaluated every ``eval_interval`` loops during training, + by default 1. + save_interval : int, optional + The model will be saved every ``eval_interval`` loops during training, by + default None. + save_dir : str, optional + Directory to save the model, by default None. + """ + data_examples = self.data_preprocess("train", train_data) + + if label_data is not None: + label_data_examples = self.data_preprocess("label", label_data) + else: + label_data_examples = None + + if val_data is not None: + val_data_examples = self.data_preprocess("val", val_data) + else: + val_data_examples = data_examples + + if isinstance(segment_size, int): + if segment_size <= 0: + raise ValueError("segment_size should be positive.") + elif isinstance(segment_size, float): + if 0 < segment_size <= 1: + segment_size = int(segment_size * len(data_examples)) + else: + raise ValueError("segment_size should be in (0, 1].") + else: + raise ValueError("segment_size should be int or float.") + + for loop in range(loops): + for seg_idx in range((len(data_examples) - 1) // segment_size + 1): + print_log( + f"loop(train) [{loop + 1}/{loops}] segment(train) " + f"[{(seg_idx + 1)}/{(len(data_examples) - 1) // segment_size + 1}] ", + logger="current", + ) + + sub_data_examples = data_examples[ + seg_idx * segment_size : (seg_idx + 1) * segment_size + ] + self.predict(sub_data_examples) + self.idx_to_pseudo_label(sub_data_examples) + self.abduce_pseudo_label(sub_data_examples) + self.filter_pseudo_label(sub_data_examples) + self.concat_data_examples(sub_data_examples, label_data_examples) + self.pseudo_label_to_idx(sub_data_examples) + self.model.train(sub_data_examples) + + if (loop + 1) % eval_interval == 0 or loop == loops - 1: + print_log(f"Eval start: loop(val) [{loop + 1}]", logger="current") + self._valid(val_data_examples) + + if save_interval is not None and ((loop + 1) % save_interval == 0 or loop == loops - 1): + print_log(f"Saving model: loop(save) [{loop + 1}]", logger="current") + self.model.save( + save_path=osp.join(save_dir, f"model_checkpoint_loop_{loop + 1}.pth") + ) + + def _valid(self, data_examples: ListData) -> None: + """ + Internal method for validating the model with given data examples. + + Parameters + ---------- + data_examples : ListData + Data examples to be used for validation. + """ + self.predict(data_examples) + self.idx_to_pseudo_label(data_examples) + + for metric in self.metric_list: + metric.process(data_examples) + + res = dict() + for metric in self.metric_list: + res.update(metric.evaluate()) + msg = "Evaluation ended, " + for k, v in res.items(): + msg += k + f": {v:.3f} " + print_log(msg, logger="current") + + def valid( + self, + val_data: Union[ + ListData, Tuple[List[List[Any]], Optional[List[List[Any]]], Optional[List[Any]]] + ], + ) -> None: + """ + Validate the model with the given validation data. + + Parameters + ---------- + val_data : Union[ListData, Tuple[List[List[Any]], Optional[List[List[Any]]], Optional[List[Any]]]] # noqa: E501 + Validation data should be in the form of ``(X, gt_pseudo_label, Y)`` or a ``ListData`` object + with ``X``, ``gt_pseudo_label`` and ``Y`` attributes. Both ``gt_pseudo_label`` and ``Y`` can be + either None or not, which depends on the evaluation metircs in ``self.metric_list``. + """ + val_data_examples = self.data_preprocess("val", val_data) + self._valid(val_data_examples) + + def test( + self, + test_data: Union[ + ListData, Tuple[List[List[Any]], Optional[List[List[Any]]], Optional[List[Any]]] + ], + ) -> None: + """ + Test the model with the given test data. + + Parameters + ---------- + test_data : Union[ListData, Tuple[List[List[Any]], Optional[List[List[Any]]], Optional[List[Any]]]] # noqa: E501 + Test data should be in the form of ``(X, gt_pseudo_label, Y)`` or a ``ListData`` object + with ``X``, ``gt_pseudo_label`` and ``Y`` attributes. Both ``gt_pseudo_label`` and ``Y`` + can be either None or not, which depends on the evaluation metircs in ``self.metric_list``. + """ + print_log("Test start:", logger="current") + test_data_examples = self.data_preprocess("test", test_data) + self._valid(test_data_examples) diff --git a/abl/data/__init__.py b/abl/data/__init__.py new file mode 100644 index 0000000..8084dd7 --- /dev/null +++ b/abl/data/__init__.py @@ -0,0 +1,4 @@ +from .evaluation import BaseMetric, ReasoningMetric, SymbolAccuracy +from .structures import ListData + +__all__ = ["BaseMetric", "ReasoningMetric", "SymbolAccuracy", "ListData"] diff --git a/abl/data/data_converter.py b/abl/data/data_converter.py new file mode 100644 index 0000000..b4e495d --- /dev/null +++ b/abl/data/data_converter.py @@ -0,0 +1,141 @@ +from typing import Any, Tuple + +from abl.utils import tab_data_to_tuple +from .structures.list_data import ListData +from lambdaLearn.Base.TabularMixin import TabularMixin + + +class DataConverter: + """ + This class provides functionality to convert LambdaLearn data to ABL-Package data. + """ + + def __init__(self) -> None: + pass + + def convert_lambdalearn_to_tuple( + self, dataset: TabularMixin, reasoning_result: Any + ) -> Tuple[Tuple, Tuple, Tuple, Tuple]: + """ + Convert a lambdalearn dataset to a tuple of tuples (label_data, train_data, valid_data, test_data), # noqa: E501 + each containing (data, label, reasoning_result). + + Parameters + ---------- + dataset : TabularMixin + The LambdaLearn dataset to be converted. + reasoning_result : Any + The reasoning result of the dataset. + Returns + ------- + Tuple[Tuple, Tuple, Tuple, Tuple] + A tuple of (label_data, train_data, valid_data, test_data), where each element is + a tuple of (data, label, reasoning_result). + """ + + if not isinstance(dataset, TabularMixin): + raise NotImplementedError( + "Only support converting the datasets that are instances of TabularMixin. " + + "Please refer to the documentation and manually convert the dataset into a tuple." + ) + + label_data = tab_data_to_tuple( + dataset.labeled_X, dataset.labeled_y, reasoning_result=reasoning_result + ) + train_data = tab_data_to_tuple( + dataset.unlabeled_X, dataset.unlabeled_y, reasoning_result=reasoning_result + ) + valid_data = tab_data_to_tuple( + dataset.valid_X, dataset.valid_y, reasoning_result=reasoning_result + ) + test_data = tab_data_to_tuple( + dataset.test_X, dataset.test_y, reasoning_result=reasoning_result + ) + + return label_data, train_data, valid_data, test_data + + def convert_lambdalearn_to_listdata( + self, dataset: TabularMixin, reasoning_result: Any + ) -> Tuple[ListData, ListData, ListData, ListData]: + """ + Convert a lambdalearn dataset to a tuple of ListData + (label_data_examples, train_data_examples, valid_data_examples, test_data_examples). + + Parameters + ---------- + dataset : TabularMixin + The LambdaLearn dataset to be converted. + reasoning_result : Any + The reasoning result of the dataset. + Returns + ------- + Tuple[ListData, ListData, ListData, ListData] + A tuple of ListData (label_data_examples, train_data_examples, valid_data_examples, test_data_examples) # noqa: E501 + """ + + if not isinstance(dataset, TabularMixin): + raise NotImplementedError( + "Only support converting the datasets that are instances of TabularMixin. " + + "Please refer to the documentation and manually convert the dataset " + + "into a ListData." + ) + + label_data, train_data, valid_data, test_data = self.convert_lambdalearn_to_tuple( + dataset, reasoning_result + ) + + if label_data is not None: + X, gt_pseudo_label, Y = label_data + label_data_examples = ListData(X=X, gt_pseudo_label=gt_pseudo_label, Y=Y) + if train_data is not None: + X, gt_pseudo_label, Y = train_data + train_data_examples = ListData(X=X, gt_pseudo_label=gt_pseudo_label, Y=Y) + if valid_data is not None: + X, gt_pseudo_label, Y = valid_data + valid_data_examples = ListData(X=X, gt_pseudo_label=gt_pseudo_label, Y=Y) + if test_data is not None: + X, gt_pseudo_label, Y = test_data + test_data_examples = ListData(X=X, gt_pseudo_label=gt_pseudo_label, Y=Y) + + return label_data_examples, train_data_examples, valid_data_examples, test_data_examples + + +if __name__ == "__main__": + from lambdaLearn.Dataset.Tabular.BreastCancer import BreastCancer + + breast_dataset = BreastCancer(labeled_size=0.1, stratified=True, shuffle=True) + dataconverter = DataConverter() + + label_data, train_data, valid_data, test_data = dataconverter.convert_lambdalearn_to_tuple( + breast_dataset, 0 + ) + print( + type(label_data).__name__, + type(train_data).__name__, + type(valid_data).__name__, + type(test_data).__name__, + ) + print(len(label_data)) + print(len(label_data[0]), len(label_data[1]), len(label_data[2])) + print(label_data[0][0], label_data[1][0], label_data[2][0]) + print() + + ( + label_data_examples, + train_data_examples, + valid_data_examples, + test_data_examples, + ) = dataconverter.convert_lambdalearn_to_listdata(breast_dataset, 0) + print( + type(label_data_examples).__name__, + type(train_data_examples).__name__, + type(valid_data_examples).__name__, + type(test_data_examples).__name__, + ) + print( + len(label_data_examples.X), + len(label_data_examples.gt_pseudo_label), + len(label_data_examples.Y), + ) + label_data_example = label_data_examples[0] + print(label_data_example.X, label_data_example.gt_pseudo_label, label_data_example.Y) diff --git a/abl/data/evaluation/__init__.py b/abl/data/evaluation/__init__.py new file mode 100644 index 0000000..3816941 --- /dev/null +++ b/abl/data/evaluation/__init__.py @@ -0,0 +1,5 @@ +from .base_metric import BaseMetric +from .reasoning_metric import ReasoningMetric +from .symbol_accuracy import SymbolAccuracy + +__all__ = ["BaseMetric", "ReasoningMetric", "SymbolAccuracy"] diff --git a/abl/data/evaluation/base_metric.py b/abl/data/evaluation/base_metric.py new file mode 100644 index 0000000..61f6428 --- /dev/null +++ b/abl/data/evaluation/base_metric.py @@ -0,0 +1,85 @@ +import logging +from abc import ABCMeta, abstractmethod +from typing import Any, List, Optional + +from ...utils import print_log +from ..structures import ListData + + +class BaseMetric(metaclass=ABCMeta): + """ + Base class for a metrics. + + The metrics first processes each batch of data_examples and appends the processed + results to the results list. Then, it computes the metrics of the entire dataset. + + Parameters + ---------- + prefix : str, optional + The prefix that will be added in the metrics names to disambiguate homonymous + metrics of different tasks. If prefix is not provided in the argument, + self.default_prefix will be used instead. Default to None. + + """ + + def __init__( + self, + prefix: Optional[str] = None, + ) -> None: + self.default_prefix = "" + self.results: List[Any] = [] + self.prefix = prefix or self.default_prefix + + @abstractmethod + def process(self, data_examples: ListData) -> None: + """ + Process one batch of data examples. The processed results should be stored + in ``self.results``, which will be used to compute the metrics when all + batches have been processed. + + Parameters + ---------- + data_examples : ListData + A batch of data examples. + """ + + @abstractmethod + def compute_metrics(self) -> dict: + """ + Compute the metrics from processed results. + + Returns + ------- + dict + The computed metrics. The keys are the names of the metrics, + and the values are corresponding results. + """ + + def evaluate(self) -> dict: + """ + Evaluate the model performance of the whole dataset after processing + all batches. + + Returns + ------- + dict + Evaluation metrics dict on the val dataset. The keys are the + names of the metrics, and the values are corresponding results. + """ + if len(self.results) == 0: + print_log( + f"{self.__class__.__name__} got empty `self.results`. Please " + "ensure that the processed results are properly added into " + "`self.results` in `process` method.", + logger="current", + level=logging.WARNING, + ) + + metrics = self.compute_metrics() + # Add prefix to metrics names + if self.prefix: + metrics = {"/".join((self.prefix, k)): v for k, v in metrics.items()} + + # reset the results list + self.results.clear() + return metrics diff --git a/abl/data/evaluation/reasoning_metric.py b/abl/data/evaluation/reasoning_metric.py new file mode 100644 index 0000000..9a010bb --- /dev/null +++ b/abl/data/evaluation/reasoning_metric.py @@ -0,0 +1,79 @@ +from typing import Optional + +from ...reasoning import KBBase +from ..structures import ListData +from .base_metric import BaseMetric + + +class ReasoningMetric(BaseMetric): + """ + A metrics class for evaluating the model performance on tasks need reasoning. + + This class is designed to calculate the accuracy of the reasoing results. Reasoning + results are generated by first using the learning part to predict pseudo-labels + and then using a knowledge base (KB) to perform logical reasoning. The reasoning results + are then compared with the ground truth to calculate the accuracy. + + Parameters + ---------- + kb : KBBase + An instance of a knowledge base, used for logical reasoning and validation. + If not provided, reasoning checks are not performed. Default to None. + prefix : str, optional + The prefix that will be added to the metrics names to disambiguate homonymous + metrics of different tasks. Inherits from BaseMetric. Default to None. + + Notes + ----- + The `ReasoningMetric` expects data_examples to have the attributes `pred_pseudo_label`, + `Y`, and `X`, corresponding to the predicted pseduo labels, ground truth of reasoning + results, and input data, respectively. + """ + + def __init__(self, kb: KBBase, prefix: Optional[str] = None) -> None: + super().__init__(prefix) + self.kb = kb + + def process(self, data_examples: ListData) -> None: + """ + Process a batch of data examples. + + This method takes in a batch of data examples, each containing predicted pseudo-labels + (pred_pseudo_label), ground truth of reasoning results (Y), and input data (X). It + evaluates the reasoning accuracy of each example by comparing the logical reasoning + result (derived using the knowledge base) of the predicted pseudo-labels against Y + The result of this comparison (1 for correct reasoning, 0 for incorrect) is appended + to ``self.results``. + + Parameters + ---------- + data_examples : ListData + A batch of data examples. + """ + pred_pseudo_label_list = data_examples.pred_pseudo_label + y_list = data_examples.Y + x_list = data_examples.X + for pred_pseudo_label, y, x in zip(pred_pseudo_label_list, y_list, x_list): + if self.kb._check_equal( + self.kb.logic_forward(pred_pseudo_label, *(x,) if self.kb._num_args == 2 else ()), y + ): + self.results.append(1) + else: + self.results.append(0) + + def compute_metrics(self) -> dict: + """ + Compute the reasoning accuracy metrics from ``self.results``. It calculates the + percentage of correctly reasoned examples over all examples. + + Returns + ------- + dict + A dictionary containing the computed metrics. It includes the key + 'reasoning_accuracy' which maps to the calculated reasoning accuracy, + represented as a float between 0 and 1. + """ + results = self.results + metrics = dict() + metrics["reasoning_accuracy"] = sum(results) / len(results) + return metrics diff --git a/abl/data/evaluation/symbol_accuracy.py b/abl/data/evaluation/symbol_accuracy.py new file mode 100644 index 0000000..33d1df4 --- /dev/null +++ b/abl/data/evaluation/symbol_accuracy.py @@ -0,0 +1,73 @@ +from typing import Optional + +import numpy as np + +from ..structures import ListData +from .base_metric import BaseMetric + + +class SymbolAccuracy(BaseMetric): + """ + A metrics class for evaluating symbol-level accuracy. + + This class is designed to assess the accuracy of symbol prediction. Symbol accuracy + are calculated by comparing predicted presudo labels and their ground truth. + + Parameters + ---------- + prefix : str, optional + The prefix that will be added to the metrics names to disambiguate homonymous + metrics of different tasks. Inherits from BaseMetric. Default to None. + """ + + def __init__(self, prefix: Optional[str] = None) -> None: + super().__init__(prefix) + + def process(self, data_examples: ListData) -> None: + """ + Processes a batch of data examples. + + This method takes in a batch of data examples, each containing a list of predicted + pseudo-labels (pred_pseudo_label) and their ground truth (gt_pseudo_label). It + calculates the accuracy by comparing the two lists. Then, a tuple of correct symbol + count and total symbol count is appended to ``self.results``. + + Parameters + ---------- + data_examples : ListData + A batch of data examples, each containing: + - ``pred_pseudo_label``: List of predicted pseudo-labels. + - ``gt_pseudo_label``: List of ground truth pseudo-labels. + + Raises + ------ + ValueError + If the lengths of predicted and ground truth symbol lists are not equal. + """ + pred_pseudo_label_list = data_examples.flatten("pred_pseudo_label") + gt_pseudo_label_list = data_examples.flatten("gt_pseudo_label") + + if not len(pred_pseudo_label_list) == len(gt_pseudo_label_list): + raise ValueError("lengthes of pred_pseudo_label and gt_pseudo_label should be equal") + + correct_num = np.sum(np.array(pred_pseudo_label_list) == np.array(gt_pseudo_label_list)) + + self.results.append((correct_num, len(pred_pseudo_label_list))) + + def compute_metrics(self) -> dict: + """ + Compute the symbol accuracy metrics from ``self.results``. It calculates the + percentage of correctly predicted pseudo-labels over all pseudo-labels. + + Returns + ------- + dict + A dictionary containing the computed metrics. It includes the key + 'character_accuracy' which maps to the calculated symbol-level accuracy, + represented as a float between 0 and 1. + + """ + results = self.results + metrics = dict() + metrics["character_accuracy"] = sum(t[0] for t in results) / sum(t[1] for t in results) + return metrics diff --git a/abl/data/structures/__init__.py b/abl/data/structures/__init__.py new file mode 100644 index 0000000..0e3535e --- /dev/null +++ b/abl/data/structures/__init__.py @@ -0,0 +1,4 @@ +from .base_data_element import BaseDataElement +from .list_data import ListData + +__all__ = ["BaseDataElement", "ListData"] diff --git a/abl/data/structures/base_data_element.py b/abl/data/structures/base_data_element.py new file mode 100644 index 0000000..a5822cf --- /dev/null +++ b/abl/data/structures/base_data_element.py @@ -0,0 +1,625 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import copy +from typing import Any, Iterator, Optional, Tuple, Type, Union + +import numpy as np +import torch + + +# Modified from +# https://github.com/open-mmlab/mmengine/blob/main/mmengine/structures/base_data_element.py +class BaseDataElement: + """A base data interface that supports Tensor-like and dict-like + operations. + + A typical data elements refer to predicted results or ground truth labels + on a task, such as predicted bboxes, instance masks, semantic + segmentation masks, etc. Because groundtruth labels and predicted results + often have similar properties (for example, the predicted bboxes and the + groundtruth bboxes), MMEngine uses the same abstract data interface to + encapsulate predicted results and groundtruth labels, and it is recommended + to use different name conventions to distinguish them, such as using + ``gt_instances`` and ``pred_instances`` to distinguish between labels and + predicted results. Additionally, we distinguish data elements at instance + level, pixel level, and label level. Each of these types has its own + characteristics. Therefore, MMEngine defines the base class + ``BaseDataElement``, and implement ``InstanceData``, ``PixelData``, and + ``LabelData`` inheriting from ``BaseDataElement`` to represent different + types of ground truth labels or predictions. + + Another common data element is data example. A data example consists of input + data (such as an image) and its annotations and predictions. In general, + an image can have multiple types of annotations and/or predictions at the + same time (for example, both pixel-level semantic segmentation annotations + and instance-level detection bboxes annotations). All labels and + predictions of a training example are often passed between Dataset, Model, + Visualizer, and Evaluator components. In order to simplify the interface + between components, we can treat them as a large data element and + encapsulate them. Such data elements are generally called XXDataSample in + the OpenMMLab. Therefore, Similar to `nn.Module`, the `BaseDataElement` + allows `BaseDataElement` as its attribute. Such a class generally + encapsulates all the data of a example in the algorithm library, and its + attributes generally are various types of data elements. For example, + MMDetection is assigned by the BaseDataElement to encapsulate all the data + elements of the example labeling and prediction of a example in the + algorithm library. + + The attributes in ``BaseDataElement`` are divided into two parts, + the ``metainfo`` and the ``data`` respectively. + + - ``metainfo``: Usually contains the + information about the image such as filename, + image_shape, pad_shape, etc. The attributes can be accessed or + modified by dict-like or object-like operations, such as + ``.`` (for data access and modification), ``in``, ``del``, + ``pop(str)``, ``get(str)``, ``metainfo_keys()``, + ``metainfo_values()``, ``metainfo_items()``, ``set_metainfo()`` (for + set or change key-value pairs in metainfo). + + - ``data``: Annotations or model predictions are + stored. The attributes can be accessed or modified by + dict-like or object-like operations, such as + ``.``, ``in``, ``del``, ``pop(str)``, ``get(str)``, ``keys()``, + ``values()``, ``items()``. Users can also apply tensor-like + methods to all :obj:`torch.Tensor` in the ``data_fields``, + such as ``.cuda()``, ``.cpu()``, ``.numpy()``, ``.to()``, + ``to_tensor()``, ``.detach()``. + + Args: + metainfo (dict, optional): A dict contains the meta information + of single image, such as ``dict(img_shape=(512, 512, 3), + scale_factor=(1, 1, 1, 1))``. Defaults to None. + kwargs (dict, optional): A dict contains annotations of single image or + model predictions. Defaults to None. + + Examples: + >>> import torch + >>> from mmengine.structures import BaseDataElement + >>> gt_instances = BaseDataElement() + >>> bboxes = torch.rand((5, 4)) + >>> scores = torch.rand((5,)) + >>> img_id = 0 + >>> img_shape = (800, 1333) + >>> gt_instances = BaseDataElement( + ... metainfo=dict(img_id=img_id, img_shape=img_shape), + ... bboxes=bboxes, scores=scores) + >>> gt_instances = BaseDataElement( + ... metainfo=dict(img_id=img_id, img_shape=(640, 640))) + + >>> # new + >>> gt_instances1 = gt_instances.new( + ... metainfo=dict(img_id=1, img_shape=(640, 640)), + ... bboxes=torch.rand((5, 4)), + ... scores=torch.rand((5,))) + >>> gt_instances2 = gt_instances1.new() + + >>> # add and process property + >>> gt_instances = BaseDataElement() + >>> gt_instances.set_metainfo(dict(img_id=9, img_shape=(100, 100))) + >>> assert 'img_shape' in gt_instances.metainfo_keys() + >>> assert 'img_shape' in gt_instances + >>> assert 'img_shape' not in gt_instances.keys() + >>> assert 'img_shape' in gt_instances.all_keys() + >>> print(gt_instances.img_shape) + (100, 100) + >>> gt_instances.scores = torch.rand((5,)) + >>> assert 'scores' in gt_instances.keys() + >>> assert 'scores' in gt_instances + >>> assert 'scores' in gt_instances.all_keys() + >>> assert 'scores' not in gt_instances.metainfo_keys() + >>> print(gt_instances.scores) + tensor([0.5230, 0.7885, 0.2426, 0.3911, 0.4876]) + >>> gt_instances.bboxes = torch.rand((5, 4)) + >>> assert 'bboxes' in gt_instances.keys() + >>> assert 'bboxes' in gt_instances + >>> assert 'bboxes' in gt_instances.all_keys() + >>> assert 'bboxes' not in gt_instances.metainfo_keys() + >>> print(gt_instances.bboxes) + tensor([[0.0900, 0.0424, 0.1755, 0.4469], + [0.8648, 0.0592, 0.3484, 0.0913], + [0.5808, 0.1909, 0.6165, 0.7088], + [0.5490, 0.4209, 0.9416, 0.2374], + [0.3652, 0.1218, 0.8805, 0.7523]]) + + >>> # delete and change property + >>> gt_instances = BaseDataElement( + ... metainfo=dict(img_id=0, img_shape=(640, 640)), + ... bboxes=torch.rand((6, 4)), scores=torch.rand((6,))) + >>> gt_instances.set_metainfo(dict(img_shape=(1280, 1280))) + >>> gt_instances.img_shape # (1280, 1280) + >>> gt_instances.bboxes = gt_instances.bboxes * 2 + >>> gt_instances.get('img_shape', None) # (1280, 1280) + >>> gt_instances.get('bboxes', None) # 6x4 tensor + >>> del gt_instances.img_shape + >>> del gt_instances.bboxes + >>> assert 'img_shape' not in gt_instances + >>> assert 'bboxes' not in gt_instances + >>> gt_instances.pop('img_shape', None) # None + >>> gt_instances.pop('bboxes', None) # None + + >>> # Tensor-like + >>> cuda_instances = gt_instances.cuda() + >>> cuda_instances = gt_instances.to('cuda:0') + >>> cpu_instances = cuda_instances.cpu() + >>> cpu_instances = cuda_instances.to('cpu') + >>> fp16_instances = cuda_instances.to( + ... device=None, dtype=torch.float16, non_blocking=False, + ... copy=False, memory_format=torch.preserve_format) + >>> cpu_instances = cuda_instances.detach() + >>> np_instances = cpu_instances.numpy() + + >>> # print + >>> metainfo = dict(img_shape=(800, 1196, 3)) + >>> gt_instances = BaseDataElement( + ... metainfo=metainfo, det_labels=torch.LongTensor([0, 1, 2, 3])) + >>> example = BaseDataElement(metainfo=metainfo, + ... gt_instances=gt_instances) + >>> print(example) + + ) at 0x7f0fea49e130> + + >>> # inheritance + >>> class DetDataSample(BaseDataElement): + ... @property + ... def proposals(self): + ... return self._proposals + ... @proposals.setter + ... def proposals(self, value): + ... self.set_field(value, '_proposals', dtype=BaseDataElement) + ... @proposals.deleter + ... def proposals(self): + ... del self._proposals + ... @property + ... def gt_instances(self): + ... return self._gt_instances + ... @gt_instances.setter + ... def gt_instances(self, value): + ... self.set_field(value, '_gt_instances', + ... dtype=BaseDataElement) + ... @gt_instances.deleter + ... def gt_instances(self): + ... del self._gt_instances + ... @property + ... def pred_instances(self): + ... return self._pred_instances + ... @pred_instances.setter + ... def pred_instances(self, value): + ... self.set_field(value, '_pred_instances', + ... dtype=BaseDataElement) + ... @pred_instances.deleter + ... def pred_instances(self): + ... del self._pred_instances + >>> det_example = DetDataSample() + >>> proposals = BaseDataElement(bboxes=torch.rand((5, 4))) + >>> det_example.proposals = proposals + >>> assert 'proposals' in det_example + >>> assert det_example.proposals == proposals + >>> del det_example.proposals + >>> assert 'proposals' not in det_example + >>> with self.assertRaises(AssertionError): + ... det_example.proposals = torch.rand((5, 4)) + """ + + def __init__(self, *, metainfo: Optional[dict] = None, **kwargs) -> None: + self._metainfo_fields: set = set() + self._data_fields: set = set() + + if metainfo is not None: + self.set_metainfo(metainfo=metainfo) + if kwargs: + self.set_data(kwargs) + + def set_metainfo(self, metainfo: dict) -> None: + """Set or change key-value pairs in ``metainfo_field`` by parameter + ``metainfo``. + + Args: + metainfo (dict): A dict contains the meta information + of image, such as ``img_shape``, ``scale_factor``, etc. + """ + assert isinstance(metainfo, dict), f"metainfo should be a ``dict`` but got {type(metainfo)}" + meta = copy.deepcopy(metainfo) + for k, v in meta.items(): + self.set_field(name=k, value=v, field_type="metainfo", dtype=None) + + def set_data(self, data: dict) -> None: + """Set or change key-value pairs in ``data_field`` by parameter + ``data``. + + Args: + data (dict): A dict contains annotations of image or + model predictions. + """ + assert isinstance(data, dict), f"data should be a `dict` but got {data}" + for k, v in data.items(): + # Use `setattr()` rather than `self.set_field` to allow `set_data` + # to set property method. + setattr(self, k, v) + + def update(self, instance: "BaseDataElement") -> None: + """The update() method updates the BaseDataElement with the elements + from another BaseDataElement object. + + Args: + instance (BaseDataElement): Another BaseDataElement object for + update the current object. + """ + assert isinstance( + instance, BaseDataElement + ), f"instance should be a `BaseDataElement` but got {type(instance)}" + self.set_metainfo(dict(instance.metainfo_items())) + self.set_data(dict(instance.items())) + + def new(self, *, metainfo: Optional[dict] = None, **kwargs) -> "BaseDataElement": + """Return a new data element with same type. If ``metainfo`` and + ``data`` are None, the new data element will have same metainfo and + data. If metainfo or data is not None, the new result will overwrite it + with the input value. + + Args: + metainfo (dict, optional): A dict contains the meta information + of image, such as ``img_shape``, ``scale_factor``, etc. + Defaults to None. + kwargs (dict): A dict contains annotations of image or + model predictions. + + Returns: + BaseDataElement: A new data element with same type. + """ + new_data = self.__class__() + + if metainfo is not None: + new_data.set_metainfo(metainfo) + else: + new_data.set_metainfo(dict(self.metainfo_items())) + if kwargs: + new_data.set_data(kwargs) + else: + new_data.set_data(dict(self.items())) + return new_data + + def clone(self): + """Deep copy the current data element. + + Returns: + BaseDataElement: The copy of current data element. + """ + clone_data = self.__class__() + clone_data.set_metainfo(dict(self.metainfo_items())) + clone_data.set_data(dict(self.items())) + return clone_data + + def keys(self) -> list: + """ + Returns: + list: Contains all keys in data_fields. + """ + # We assume that the name of the attribute related to property is + # '_' + the name of the property. We use this rule to filter out + # private keys. + # TODO: Use a more robust way to solve this problem + private_keys = { + "_" + key + for key in self._data_fields + if isinstance(getattr(type(self), key, None), property) + } + return list(self._data_fields - private_keys) + + def metainfo_keys(self) -> list: + """ + Returns: + list: Contains all keys in metainfo_fields. + """ + return list(self._metainfo_fields) + + def values(self) -> list: + """ + Returns: + list: Contains all values in data. + """ + return [getattr(self, k) for k in self.keys()] + + def metainfo_values(self) -> list: + """ + Returns: + list: Contains all values in metainfo. + """ + return [getattr(self, k) for k in self.metainfo_keys()] + + def all_keys(self) -> list: + """ + Returns: + list: Contains all keys in metainfo and data. + """ + return self.metainfo_keys() + self.keys() + + def all_values(self) -> list: + """ + Returns: + list: Contains all values in metainfo and data. + """ + return self.metainfo_values() + self.values() + + def all_items(self) -> Iterator[Tuple[str, Any]]: + """ + Returns: + iterator: An iterator object whose element is (key, value) tuple + pairs for ``metainfo`` and ``data``. + """ + for k in self.all_keys(): + yield (k, getattr(self, k)) + + def items(self) -> Iterator[Tuple[str, Any]]: + """ + Returns: + iterator: An iterator object whose element is (key, value) tuple + pairs for ``data``. + """ + for k in self.keys(): + yield (k, getattr(self, k)) + + def metainfo_items(self) -> Iterator[Tuple[str, Any]]: + """ + Returns: + iterator: An iterator object whose element is (key, value) tuple + pairs for ``metainfo``. + """ + for k in self.metainfo_keys(): + yield (k, getattr(self, k)) + + @property + def metainfo(self) -> dict: + """dict: A dict contains metainfo of current data element.""" + return dict(self.metainfo_items()) + + def __setattr__(self, name: str, value: Any): + """setattr is only used to set data.""" + if name in ("_metainfo_fields", "_data_fields"): + if not hasattr(self, name): + super().__setattr__(name, value) + else: + raise AttributeError( + f"{name} has been used as a " "private attribute, which is immutable." + ) + else: + self.set_field(name=name, value=value, field_type="data", dtype=None) + + def __delattr__(self, item: str): + """Delete the item in dataelement. + + Args: + item (str): The key to delete. + """ + if item in ("_metainfo_fields", "_data_fields"): + raise AttributeError( + f"{item} has been used as a " "private attribute, which is immutable." + ) + super().__delattr__(item) + if item in self._metainfo_fields: + self._metainfo_fields.remove(item) + elif item in self._data_fields: + self._data_fields.remove(item) + + # dict-like methods + __delitem__ = __delattr__ + + def get(self, key, default=None) -> Any: + """Get property in data and metainfo as the same as python.""" + # Use `getattr()` rather than `self.__dict__.get()` to allow getting + # properties. + return getattr(self, key, default) + + def pop(self, *args) -> Any: + """Pop property in data and metainfo as the same as python.""" + assert len(args) < 3, "``pop`` get more than 2 arguments" + name = args[0] + if name in self._metainfo_fields: + self._metainfo_fields.remove(args[0]) + return self.__dict__.pop(*args) + + elif name in self._data_fields: + self._data_fields.remove(args[0]) + return self.__dict__.pop(*args) + + # with default value + elif len(args) == 2: + return args[1] + else: + # don't just use 'self.__dict__.pop(*args)' for only popping key in + # metainfo or data + raise KeyError(f"{args[0]} is not contained in metainfo or data") + + def __contains__(self, item: str) -> bool: + """Whether the item is in dataelement. + + Args: + item (str): The key to inquire. + """ + return item in self._data_fields or item in self._metainfo_fields + + def set_field( + self, + value: Any, + name: str, + dtype: Optional[Union[Type, Tuple[Type, ...]]] = None, + field_type: str = "data", + ) -> None: + """Special method for set union field, used as property.setter + functions.""" + assert field_type in ["metainfo", "data"] + if dtype is not None: + assert isinstance(value, dtype), f"{value} should be a {dtype} but got {type(value)}" + + if field_type == "metainfo": + if name in self._data_fields: + raise AttributeError( + f"Cannot set {name} to be a field of metainfo " + f"because {name} is already a data field" + ) + self._metainfo_fields.add(name) + else: + if name in self._metainfo_fields: + raise AttributeError( + f"Cannot set {name} to be a field of data " + f"because {name} is already a metainfo field" + ) + self._data_fields.add(name) + super().__setattr__(name, value) + + # Tensor-like methods + def to(self, *args, **kwargs) -> "BaseDataElement": + """Apply same name function to all tensors in data_fields.""" + new_data = self.new() + for k, v in self.items(): + if hasattr(v, "to"): + v = v.to(*args, **kwargs) + data = {k: v} + new_data.set_data(data) + return new_data + + # Tensor-like methods + def cpu(self) -> "BaseDataElement": + """Convert all tensors to CPU in data.""" + new_data = self.new() + for k, v in self.items(): + if isinstance(v, (torch.Tensor, BaseDataElement)): + v = v.cpu() + data = {k: v} + new_data.set_data(data) + return new_data + + # Tensor-like methods + def cuda(self) -> "BaseDataElement": + """Convert all tensors to GPU in data.""" + new_data = self.new() + for k, v in self.items(): + if isinstance(v, (torch.Tensor, BaseDataElement)): + v = v.cuda() + data = {k: v} + new_data.set_data(data) + return new_data + + # Tensor-like methods + def npu(self) -> "BaseDataElement": + """Convert all tensors to NPU in data.""" + new_data = self.new() + for k, v in self.items(): + if isinstance(v, (torch.Tensor, BaseDataElement)): + v = v.npu() + data = {k: v} + new_data.set_data(data) + return new_data + + def mlu(self) -> "BaseDataElement": + """Convert all tensors to MLU in data.""" + new_data = self.new() + for k, v in self.items(): + if isinstance(v, (torch.Tensor, BaseDataElement)): + v = v.mlu() + data = {k: v} + new_data.set_data(data) + return new_data + + # Tensor-like methods + def detach(self) -> "BaseDataElement": + """Detach all tensors in data.""" + new_data = self.new() + for k, v in self.items(): + if isinstance(v, (torch.Tensor, BaseDataElement)): + v = v.detach() + data = {k: v} + new_data.set_data(data) + return new_data + + # Tensor-like methods + def numpy(self) -> "BaseDataElement": + """Convert all tensors to np.ndarray in data.""" + new_data = self.new() + for k, v in self.items(): + if isinstance(v, (torch.Tensor, BaseDataElement)): + v = v.detach().cpu().numpy() + data = {k: v} + new_data.set_data(data) + return new_data + + def to_tensor(self) -> "BaseDataElement": + """Convert all np.ndarray to tensor in data.""" + new_data = self.new() + for k, v in self.items(): + data = {} + if isinstance(v, np.ndarray): + v = torch.from_numpy(v) + data[k] = v + elif isinstance(v, BaseDataElement): + v = v.to_tensor() + data[k] = v + new_data.set_data(data) + return new_data + + def to_dict(self) -> dict: + """Convert BaseDataElement to dict.""" + return { + k: v.to_dict() if isinstance(v, BaseDataElement) else v for k, v in self.all_items() + } + + def __repr__(self) -> str: + """Represent the object.""" + + def _addindent(s_: str, num_spaces: int) -> str: + """This func is modified from `pytorch` https://github.com/pytorch/ + pytorch/blob/b17b2b1cc7b017c3daaeff8cc7ec0f514d42ec37/torch/nn/modu + les/module.py#L29. + + Args: + s_ (str): The string to add spaces. + num_spaces (int): The num of space to add. + + Returns: + str: The string after add indent. + """ + s = s_.split("\n") + # don't do anything for single-line stuff + if len(s) == 1: + return s_ + first = s.pop(0) + s = [(num_spaces * " ") + line for line in s] + s = "\n".join(s) # type: ignore + s = first + "\n" + s # type: ignore + return s # type: ignore + + def dump(obj: Any) -> str: + """Represent the object. + + Args: + obj (Any): The obj to represent. + + Returns: + str: The represented str. + """ + _repr = "" + if isinstance(obj, dict): + for k, v in obj.items(): + _repr += f"\n{k}: {_addindent(dump(v), 4)}" + elif isinstance(obj, BaseDataElement): + _repr += "\n\n META INFORMATION" + metainfo_items = dict(obj.metainfo_items()) + _repr += _addindent(dump(metainfo_items), 4) + _repr += "\n\n DATA FIELDS" + items = dict(obj.items()) + _repr += _addindent(dump(items), 4) + classname = obj.__class__.__name__ + _repr = f"<{classname}({_repr}\n) at {hex(id(obj))}>" + else: + _repr += repr(obj) + return _repr + + return dump(self) diff --git a/abl/data/structures/list_data.py b/abl/data/structures/list_data.py new file mode 100644 index 0000000..e3c6fa1 --- /dev/null +++ b/abl/data/structures/list_data.py @@ -0,0 +1,257 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from typing import List, Union + +import numpy as np +import torch + +from ...utils import flatten as flatten_list +from ...utils import to_hashable +from .base_data_element import BaseDataElement + +BoolTypeTensor = Union[torch.BoolTensor, torch.cuda.BoolTensor] +LongTypeTensor = Union[torch.LongTensor, torch.cuda.LongTensor] + +IndexType = Union[str, slice, int, list, LongTypeTensor, BoolTypeTensor, np.ndarray] + + +# Modified from +# https://github.com/open-mmlab/mmdetection/blob/master/mmdet/core/data_structures/instance_data.py # noqa +class ListData(BaseDataElement): + """ + Abstract Data Interface used throughout the ABL-Package. + + ``ListData`` is the underlying data structure used in the ABL-Package, + designed to manage diverse forms of data dynamically generated throughout the + Abductive Learning (ABL) framework. This includes handling raw data, predicted + pseudo-labels, abduced pseudo-labels, pseudo-label indices, etc. + + As a fundamental data structure in ABL, ``ListData`` is essential for the smooth + transfer and manipulation of data across various components of the ABL framework, + such as prediction, abductive reasoning, and training phases. It provides a + unified data format across these stages, ensuring compatibility and flexibility + in handling diverse data forms in the ABL framework. + + The attributes in ``ListData`` are divided into two parts, + the ``metainfo`` and the ``data`` respectively. + + - ``metainfo``: Usually used to store basic information about data examples, + such as symbol number, image size, etc. The attributes can be accessed or + modified by dict-like or object-like operations, such as ``.`` (for data + access and modification), ``in``, ``del``, ``pop(str)``, ``get(str)``, + ``metainfo_keys()``, ``metainfo_values()``, ``metainfo_items()``, + ``set_metainfo()`` (for set or change key-value pairs in metainfo). + + - ``data``: raw data, labels, predictions, and abduced results are stored. + The attributes can be accessed or modified by dict-like or object-like operations, + such as ``.``, ``in``, ``del``, ``pop(str)``, ``get(str)``, ``keys()``, + ``values()``, ``items()``. Users can also apply tensor-like + methods to all :obj:`torch.Tensor` in the ``data_fields``, such as ``.cuda()``, + ``.cpu()``, ``.numpy()``, ``.to()``, ``to_tensor()``, ``.detach()``. + + ListData supports ``index`` and ``slice`` for data field. The type of value in + data field can be either ``None`` or ``list`` of base data structures such as + ``torch.Tensor``, ``numpy.ndarray``, ``list``, ``str`` and ``tuple``. + + This design is inspired by and extends the functionalities of the ``BaseDataElement`` + class implemented in `MMEngine `_. # noqa: E501 + + Examples: + >>> from abl.data.structures import ListData + >>> import numpy as np + >>> import torch + >>> data_examples = ListData() + >>> data_examples.X = [list(torch.randn(2)) for _ in range(3)] + >>> data_examples.Y = [1, 2, 3] + >>> data_examples.gt_pseudo_label = [[1, 2], [3, 4], [5, 6]] + >>> len(data_examples) + 3 + >>> print(data_examples) + + >>> print(data_examples[:1]) + + >>> print(data_examples.elements_num("X")) + 6 + >>> print(data_examples.flatten("gt_pseudo_label")) + [1, 2, 3, 4, 5, 6] + >>> print(data_examples.to_tuple("Y")) + (1, 2, 3) + """ + + def __setattr__(self, name: str, value: list): + """setattr is only used to set data. + + The value must have the attribute of `__len__` and have the same length + of `ListData`. + """ + if name in ("_metainfo_fields", "_data_fields"): + if not hasattr(self, name): + super().__setattr__(name, value) + else: + raise AttributeError( + f"{name} has been used as a " "private attribute, which is immutable." + ) + + else: + # assert isinstance(value, list), "value must be of type `list`" + + # if len(self) > 0: + # assert len(value) == len(self), ( + # "The length of " + # f"values {len(value)} is " + # "not consistent with " + # "the length of this " + # ":obj:`ListData` " + # f"{len(self)}" + # ) + super().__setattr__(name, value) + + __setitem__ = __setattr__ + + def __getitem__(self, item: IndexType) -> "ListData": + """ + Args: + item (str, int, list, :obj:`slice`, :obj:`numpy.ndarray`, + :obj:`torch.LongTensor`, :obj:`torch.BoolTensor`): + Get the corresponding values according to item. + + Returns: + :obj:`ListData`: Corresponding values. + """ + assert isinstance(item, IndexType.__args__) + if isinstance(item, list): + item = np.array(item) + if isinstance(item, np.ndarray): + # The default int type of numpy is platform dependent, int32 for + # windows and int64 for linux. `torch.Tensor` requires the index + # should be int64, therefore we simply convert it to int64 here. + # More details in https://github.com/numpy/numpy/issues/9464 + item = item.astype(np.int64) if item.dtype == np.int32 else item + item = torch.from_numpy(item) + + if isinstance(item, str): + return getattr(self, item) + + new_data = self.__class__(metainfo=self.metainfo) + + if isinstance(item, torch.Tensor): + assert item.dim() == 1, "Only support to get the" " values along the first dimension." + + for k, v in self.items(): + if v is None: + new_data[k] = None + elif isinstance(v, torch.Tensor): + new_data[k] = v[item] + elif isinstance(v, np.ndarray): + new_data[k] = v[item.cpu().numpy()] + elif isinstance(v, (str, list, tuple)) or ( + hasattr(v, "__getitem__") and hasattr(v, "cat") + ): + # convert to indexes from BoolTensor + if isinstance(item, BoolTypeTensor.__args__): + indexes = torch.nonzero(item).view(-1).cpu().numpy().tolist() + else: + indexes = item.cpu().numpy().tolist() + slice_list = [] + if indexes: + for index in indexes: + slice_list.append(slice(index, None, len(v))) + else: + slice_list.append(slice(None, 0, None)) + r_list = [v[s] for s in slice_list] + if isinstance(v, (str, list, tuple)): + new_value = r_list[0] + for r in r_list[1:]: + new_value = new_value + r + else: + new_value = v.cat(r_list) + new_data[k] = new_value + else: + raise ValueError( + f"The type of `{k}` is `{type(v)}`, which has no " + "attribute of `cat`, so it does not " + "support slice with `bool`" + ) + + else: + # item is a slice or int + for k, v in self.items(): + if v is None: + new_data[k] = None + else: + new_data[k] = v[item] + return new_data # type:ignore + + def flatten(self, item: str) -> List: + """ + Flatten the list of the attribute specified by ``item``. + + Parameters + ---------- + item + Name of the attribute to be flattened. + + Returns + ------- + list + The flattened list of the attribute specified by ``item``. + """ + return flatten_list(self[item]) + + def elements_num(self, item: str) -> int: + """ + Return the number of elements in the attribute specified by ``item``. + + Parameters + ---------- + item : str + Name of the attribute for which the number of elements is to be determined. + + Returns + ------- + int + The number of elements in the attribute specified by ``item``. + """ + return len(self.flatten(item)) + + def to_tuple(self, item: str) -> tuple: + """ + Convert the attribute specified by ``item`` to a tuple. + + Parameters + ---------- + item : str + Name of the attribute to be converted. + + Returns + ------- + tuple + The attribute after conversion to a tuple. + """ + return to_hashable(self[item]) + + def __len__(self) -> int: + """int: The length of ListData.""" + iterator = iter(self._data_fields) + data = next(iterator) + + while getattr(self, data) is None: + try: + data = next(iterator) + except StopIteration: + break + if getattr(self, data) is None: + raise ValueError("All data fields are None.") + else: + return len(getattr(self, data)) diff --git a/abl/learning/__init__.py b/abl/learning/__init__.py new file mode 100644 index 0000000..ad016a6 --- /dev/null +++ b/abl/learning/__init__.py @@ -0,0 +1,5 @@ +from .abl_model import ABLModel +from .basic_nn import BasicNN +from .torch_dataset import ClassificationDataset, PredictionDataset, RegressionDataset + +__all__ = ["ABLModel", "BasicNN", "ClassificationDataset", "PredictionDataset", "RegressionDataset"] diff --git a/abl/learning/abl_model.py b/abl/learning/abl_model.py new file mode 100644 index 0000000..6ceb798 --- /dev/null +++ b/abl/learning/abl_model.py @@ -0,0 +1,134 @@ +import pickle +from typing import Any, Dict + +from ..data.structures import ListData +from ..utils import reform_list + + +class ABLModel: + """ + Serialize data and provide a unified interface for different machine learning models. + + Parameters + ---------- + base_model : Machine Learning Model + The machine learning base model used for training and prediction. This model should + implement the ``fit`` and ``predict`` methods. It's recommended, but not required, for the + model to also implement the ``predict_proba`` method for generating + predictions on the probabilities. + """ + + def __init__(self, base_model: Any) -> None: + if not (hasattr(base_model, "fit") and hasattr(base_model, "predict")): + raise NotImplementedError("The base_model should implement fit and predict methods.") + + self.base_model = base_model + + def predict(self, data_examples: ListData) -> Dict: + """ + Predict the labels and probabilities for the given data. + + Parameters + ---------- + data_examples : ListData + A batch of data to predict on. + + Returns + ------- + dict + A dictionary containing the predicted labels and probabilities. + """ + model = self.base_model + data_X = data_examples.flatten("X") + if hasattr(model, "predict_proba"): + prob = model.predict_proba(X=data_X) + label = prob.argmax(axis=1) + prob = reform_list(prob, data_examples.X) + else: + prob = None + label = model.predict(X=data_X) + label = reform_list(label, data_examples.X) + + data_examples.pred_idx = label + data_examples.pred_prob = prob + + return {"label": label, "prob": prob} + + def train(self, data_examples: ListData) -> float: + """ + Train the model on the given data. + + Parameters + ---------- + data_examples : ListData + A batch of data to train on, which typically contains the data, ``X``, and the + corresponding labels, ``abduced_idx``. + + Returns + ------- + float + The loss value of the trained model. + """ + data_X = data_examples.flatten("X") + data_y = data_examples.flatten("abduced_idx") + return self.base_model.fit(X=data_X, y=data_y) + + def valid(self, data_examples: ListData) -> float: + """ + Validate the model on the given data. + + Parameters + ---------- + data_examples : ListData + A batch of data to train on, which typically contains the data, ``X``, + and the corresponding labels, ``abduced_idx``. + + Returns + ------- + float + The accuracy the trained model. + """ + data_X = data_examples.flatten("X") + data_y = data_examples.flatten("abduced_idx") + score = self.base_model.score(X=data_X, y=data_y) + return score + + def _model_operation(self, operation: str, *args, **kwargs): + model = self.base_model + if hasattr(model, operation): + method = getattr(model, operation) + method(*args, **kwargs) + else: + if f"{operation}_path" not in kwargs.keys(): + raise ValueError(f"'{operation}_path' should not be None") + else: + try: + if operation == "save": + with open(kwargs["save_path"], "wb") as file: + pickle.dump(model, file, protocol=pickle.HIGHEST_PROTOCOL) + elif operation == "load": + with open(kwargs["load_path"], "rb") as file: + self.base_model = pickle.load(file) + except (OSError, pickle.PickleError): + raise NotImplementedError( + f"{type(model).__name__} object doesn't have the {operation} method \ + and the default pickle-based {operation} method failed." + ) + + def save(self, *args, **kwargs) -> None: + """ + Save the model to a file. + + This method delegates to the ``save`` method of self.base_model. The arguments passed to + this method should match those expected by the ``save`` method of self.base_model. + """ + self._model_operation("save", *args, **kwargs) + + def load(self, *args, **kwargs) -> None: + """ + Load the model from a file. + + This method delegates to the ``load`` method of self.base_model. The arguments passed to + this method should match those expected by the ``load`` method of self.base_model. + """ + self._model_operation("load", *args, **kwargs) diff --git a/abl/learning/basic_nn.py b/abl/learning/basic_nn.py new file mode 100644 index 0000000..e35328a --- /dev/null +++ b/abl/learning/basic_nn.py @@ -0,0 +1,546 @@ +from __future__ import annotations + +import logging +import os +from typing import Any, Callable, List, Optional, Tuple, Union + +import numpy +import torch +from torch.utils.data import DataLoader + +from ..utils.logger import print_log +from .torch_dataset import ClassificationDataset, PredictionDataset + + +class BasicNN: + """ + Wrap NN models into the form of an sklearn estimator. + + Parameters + ---------- + model : torch.nn.Module + The PyTorch model to be trained or used for prediction. + loss_fn : torch.nn.Module + The loss function used for training. + optimizer : torch.optim.Optimizer + The optimizer used for training. + scheduler : Callable[..., Any], optional + The learning rate scheduler used for training, which will be called + at the end of each run of the ``fit`` method. It should implement the + ``step`` method, by default None. + device : Union[torch.device, str] + The device on which the model will be trained or used for prediction, + by default torch.device("cpu"). + batch_size : int, optional + The batch size used for training, by default 32. + num_epochs : int, optional + The number of epochs used for training, by default 1. + stop_loss : float, optional + The loss value at which to stop training, by default 0.0001. + num_workers : int + The number of workers used for loading data, by default 0. + save_interval : int, optional + The model will be saved every ``save_interval`` epochs during training, by default None. + save_dir : str, optional + The directory in which to save the model during training, by default None. + train_transform : Callable[..., Any], optional + A function/transform that takes an object and returns a transformed version used + in the ``fit`` and ``train_epoch`` methods, by default None. + test_transform : Callable[..., Any], optional + A function/transform that takes an object and returns a transformed version in the + ``predict``, ``predict_proba`` and ``score`` methods, , by default None. + collate_fn : Callable[[List[T]], Any], optional + The function used to collate data, by default None. + """ + + def __init__( + self, + model: torch.nn.Module, + loss_fn: torch.nn.Module, + optimizer: torch.optim.Optimizer, + scheduler: Optional[Callable[..., Any]] = None, + device: Union[torch.device, str] = torch.device("cpu"), + batch_size: int = 32, + num_epochs: int = 1, + stop_loss: Optional[float] = 0.0001, + num_workers: int = 0, + save_interval: Optional[int] = None, + save_dir: Optional[str] = None, + train_transform: Optional[Callable[..., Any]] = None, + test_transform: Optional[Callable[..., Any]] = None, + collate_fn: Optional[Callable[[List[Any]], Any]] = None, + ) -> None: + if not isinstance(model, torch.nn.Module): + raise TypeError("model must be an instance of torch.nn.Module") + if not isinstance(loss_fn, torch.nn.Module): + raise TypeError("loss_fn must be an instance of torch.nn.Module") + if not isinstance(optimizer, torch.optim.Optimizer): + raise TypeError("optimizer must be an instance of torch.optim.Optimizer") + if scheduler is not None and not hasattr(scheduler, "step"): + raise NotImplementedError("scheduler should implement the ``step`` method") + if not isinstance(device, torch.device): + if not isinstance(device, str): + raise TypeError( + "device must be an instance of torch.device or a str indicating " + + "the target device" + ) + else: + device = torch.device(device) + if not isinstance(batch_size, int): + raise TypeError("batch_size must be an integer") + if not isinstance(num_epochs, int): + raise TypeError("num_epochs must be an integer") + if stop_loss is not None and not isinstance(stop_loss, float): + raise TypeError("stop_loss must be a float") + if not isinstance(num_workers, int): + raise TypeError("num_workers must be an integer") + if save_interval is not None and not isinstance(save_interval, int): + raise TypeError("save_interval must be an integer") + if save_dir is not None and not isinstance(save_dir, str): + raise TypeError("save_dir must be a string") + if train_transform is not None and not callable(train_transform): + raise TypeError("train_transform must be callable") + if test_transform is not None and not callable(test_transform): + raise TypeError("test_transform must be callable") + if collate_fn is not None and not callable(collate_fn): + raise TypeError("collate_fn must be callable") + + self.model = model.to(device) + self.loss_fn = loss_fn + self.optimizer = optimizer + self.scheduler = scheduler + self.device = device + self.batch_size = batch_size + self.num_epochs = num_epochs + self.stop_loss = stop_loss + self.num_workers = num_workers + self.save_interval = save_interval + self.save_dir = save_dir + self.train_transform = train_transform + self.test_transform = test_transform + self.collate_fn = collate_fn + + if self.save_interval is not None and self.save_dir is None: + raise ValueError("save_dir should not be None if save_interval is not None.") + + if self.train_transform is not None and self.test_transform is None: + print_log( + "Transform used in the training phase will be used in prediction.", + logger="current", + level=logging.WARNING, + ) + self.test_transform = self.train_transform + + def _fit(self, data_loader: DataLoader) -> BasicNN: + """ + Internal method to fit the model on data for ``self.num_epochs`` times, + with early stopping. + + Parameters + ---------- + data_loader : DataLoader + Data loader providing training samples. + + Returns + ------- + BasicNN + The model itself after training. + """ + if not isinstance(data_loader, DataLoader): + raise TypeError( + f"data_loader must be an instance of torch.utils.data.DataLoader, " + f"but got {type(data_loader)}" + ) + + for epoch in range(self.num_epochs): + loss_value = self.train_epoch(data_loader) + if self.save_interval is not None and (epoch + 1) % self.save_interval == 0: + self.save(epoch + 1) + if self.stop_loss is not None and loss_value < self.stop_loss: + break + if self.scheduler is not None: + self.scheduler.step() + print_log(f"model loss: {loss_value:.5f}", logger="current") + return self + + def fit( + self, + data_loader: Optional[DataLoader] = None, + X: Optional[List[Any]] = None, + y: Optional[List[int]] = None, + ) -> BasicNN: + """ + Train the model for self.num_epochs times or until the average loss on one epoch + is less than self.stop_loss. It supports training with either a DataLoader + object (data_loader) or a pair of input data (X) and target labels (y). If both + data_loader and (X, y) are provided, the method will prioritize using the data_loader. + + Parameters + ---------- + data_loader : DataLoader, optional + The data loader used for training, by default None. + X : List[Any], optional + The input data, by default None. + y : List[int], optional + The target data, by default None. + + Returns + ------- + BasicNN + The model itself after training. + """ + if data_loader is not None and X is not None: + print_log( + "data_loader will be used to train the model instead of X and y.", + logger="current", + level=logging.WARNING, + ) + if data_loader is None: + if X is None: + raise ValueError("data_loader and X can not be None simultaneously.") + else: + data_loader = self._data_loader(X, y) + return self._fit(data_loader) + + def train_epoch(self, data_loader: DataLoader) -> float: + """ + Train the model with an instance of DataLoader (data_loader) for one epoch. + + Parameters + ---------- + data_loader : DataLoader + The data loader used for training. + + Returns + ------- + float + The average loss on one epoch. + """ + model = self.model + loss_fn = self.loss_fn + optimizer = self.optimizer + device = self.device + + model.train() + + total_loss, total_num = 0.0, 0 + for data, target in data_loader: + data, target = data.to(device), target.to(device) + out = model(data) + loss = loss_fn(out, target) + + optimizer.zero_grad() + loss.backward() + optimizer.step() + + total_loss += loss.item() * data.size(0) + total_num += data.size(0) + + return total_loss / total_num + + def _predict(self, data_loader: DataLoader) -> torch.Tensor: + """ + Internal method to predict the outputs given a DataLoader. + + Parameters + ---------- + data_loader : DataLoader + The DataLoader providing input samples. + + Returns + ------- + torch.Tensor + Raw output from the model. + """ + if not isinstance(data_loader, DataLoader): + raise TypeError( + f"data_loader must be an instance of torch.utils.data.DataLoader, " + f"but got {type(data_loader)}" + ) + model = self.model + device = self.device + + model.eval() + + with torch.no_grad(): + results = [] + for data in data_loader: + data = data.to(device) + out = model(data) + results.append(out) + + return torch.cat(results, axis=0) + + def predict( + self, + data_loader: Optional[DataLoader] = None, + X: Optional[List[Any]] = None, + ) -> numpy.ndarray: + """ + Predict the class of the input data. This method supports prediction with either + a DataLoader object (data_loader) or a list of input data (X). If both data_loader + and X are provided, the method will predict the input data in data_loader + instead of X. + + Parameters + ---------- + data_loader : DataLoader, optional + The data loader used for prediction, by default None. + X : List[Any], optional + The input data, by default None. + + Returns + ------- + numpy.ndarray + The predicted class of the input data. + """ + + if data_loader is not None and X is not None: + print_log( + "Predict the class of input data in data_loader instead of X.", + logger="current", + level=logging.WARNING, + ) + + if data_loader is None: + dataset = PredictionDataset(X, self.test_transform) + data_loader = DataLoader( + dataset, + batch_size=self.batch_size, + num_workers=int(self.num_workers), + collate_fn=self.collate_fn, + ) + return self._predict(data_loader).argmax(axis=1).cpu().numpy() + + def predict_proba( + self, + data_loader: Optional[DataLoader] = None, + X: Optional[List[Any]] = None, + ) -> numpy.ndarray: + """ + Predict the probability of each class for the input data. This method supports + prediction with either a DataLoader object (data_loader) or a list of input data (X). + If both data_loader and X are provided, the method will predict the input data in + data_loader instead of X. + + Parameters + ---------- + data_loader : DataLoader, optional + The data loader used for prediction, by default None. + X : List[Any], optional + The input data, by default None. + + Returns + ------- + numpy.ndarray + The predicted probability of each class for the input data. + """ + + if data_loader is not None and X is not None: + print_log( + "Predict the class probability of input data in data_loader instead of X.", + logger="current", + level=logging.WARNING, + ) + + if data_loader is None: + dataset = PredictionDataset(X, self.test_transform) + data_loader = DataLoader( + dataset, + batch_size=self.batch_size, + num_workers=int(self.num_workers), + collate_fn=self.collate_fn, + ) + return self._predict(data_loader).softmax(axis=1).cpu().numpy() + + def _score(self, data_loader: DataLoader) -> Tuple[float, float]: + """ + Internal method to compute loss and accuracy for the data provided through a DataLoader. + + Parameters + ---------- + data_loader : DataLoader + Data loader to use for evaluation. + + Returns + ------- + Tuple[float, float] + mean_loss: float, The mean loss of the model on the provided data. + accuracy: float, The accuracy of the model on the provided data. + """ + if not isinstance(data_loader, DataLoader): + raise TypeError( + f"data_loader must be an instance of torch.utils.data.DataLoader, " + f"but got {type(data_loader)}" + ) + + model = self.model + loss_fn = self.loss_fn + device = self.device + + model.eval() + + total_correct_num, total_num, total_loss = 0, 0, 0.0 + + with torch.no_grad(): + for data, target in data_loader: + data, target = data.to(device), target.to(device) + + out = model(data) + + if len(out.shape) > 1: + correct_num = (target == out.argmax(axis=1)).sum().item() + else: + correct_num = (target == (out > 0.5)).sum().item() + loss = loss_fn(out, target) + total_loss += loss.item() * data.size(0) + + total_correct_num += correct_num + total_num += data.size(0) + + mean_loss = total_loss / total_num + accuracy = total_correct_num / total_num + + return mean_loss, accuracy + + def score( + self, + data_loader: Optional[DataLoader] = None, + X: Optional[List[Any]] = None, + y: Optional[List[int]] = None, + ) -> float: + """ + Validate the model. It supports validation with either a DataLoader object (data_loader) + or a pair of input data (X) and ground truth labels (y). If both data_loader and + (X, y) are provided, the method will prioritize using the data_loader. + + Parameters + ---------- + data_loader : DataLoader, optional + The data loader used for scoring, by default None. + X : List[Any], optional + The input data, by default None. + y : List[int], optional + The target data, by default None. + + Returns + ------- + float + The accuracy of the model. + """ + print_log("Start machine learning model validation", logger="current") + + if data_loader is not None and X is not None: + print_log( + "data_loader will be used to validate the model instead of X and y.", + logger="current", + level=logging.WARNING, + ) + + if data_loader is None: + if X is None or y is None: + raise ValueError("data_loader and (X, y) can not be None simultaneously.") + else: + data_loader = self._data_loader(X, y) + mean_loss, accuracy = self._score(data_loader) + print_log(f"mean loss: {mean_loss:.3f}, accuray: {accuracy:.3f}", logger="current") + return accuracy + + def _data_loader( + self, + X: Optional[List[Any]], + y: Optional[List[int]] = None, + shuffle: Optional[bool] = True, + ) -> DataLoader: + """ + Generate a DataLoader for user-provided input data and target labels. + + Parameters + ---------- + X : List[Any] + Input samples. + y : List[int], optional + Target labels. If None, dummy labels are created, by default None. + shuffle : bool, optional + Whether to shuffle the data, by default True. + + Returns + ------- + DataLoader + A DataLoader providing batches of (X, y) pairs. + """ + + if X is None: + raise ValueError("X should not be None.") + if y is None: + y = [0] * len(X) + if not (len(y) == len(X)): + raise ValueError("X and y should have equal length.") + + dataset = ClassificationDataset(X, y, transform=self.train_transform) + data_loader = DataLoader( + dataset, + batch_size=self.batch_size, + shuffle=shuffle, + num_workers=int(self.num_workers), + collate_fn=self.collate_fn, + ) + return data_loader + + def save(self, epoch_id: int = 0, save_path: Optional[str] = None) -> None: + """ + Save the model and the optimizer. User can either provide a save_path or specify + the epoch_id at which the model and optimizer is saved. if both save_path and + epoch_id are provided, save_path will be used. If only epoch_id is specified, + model and optimizer will be saved to the path f"model_checkpoint_epoch_{epoch_id}.pth" + under ``self.save_dir``. save_path and epoch_id can not be None simultaneously. + + Parameters + ---------- + epoch_id : int + The epoch id. + save_path : str, optional + The path to save the model, by default None. + """ + if self.save_dir is None and save_path is None: + raise ValueError("'save_dir' and 'save_path' should not be None simultaneously.") + + if save_path is not None: + if not os.path.exists(os.path.dirname(save_path)): + os.makedirs(os.path.dirname(save_path)) + else: + save_path = os.path.join(self.save_dir, f"model_checkpoint_epoch_{epoch_id}.pth") + if not os.path.exists(self.save_dir): + os.makedirs(self.save_dir) + + print_log(f"Checkpoints will be saved to {save_path}", logger="current") + + save_parma_dic = { + "model": self.model.state_dict(), + "optimizer": self.optimizer.state_dict(), + } + + torch.save(save_parma_dic, save_path) + + def load(self, load_path: str) -> None: + """ + Load the model and the optimizer. + + Parameters + ---------- + load_path : str + The directory to load the model, by default "". + """ + + if load_path is None: + raise ValueError("Load path should not be None.") + + print_log( + f"Loads checkpoint by local backend from path: {load_path}", + logger="current", + ) + + param_dic = torch.load(load_path) + self.model.load_state_dict(param_dic["model"]) + if "optimizer" in param_dic.keys(): + self.optimizer.load_state_dict(param_dic["optimizer"]) diff --git a/abl/learning/model_converter.py b/abl/learning/model_converter.py new file mode 100644 index 0000000..0f79ce9 --- /dev/null +++ b/abl/learning/model_converter.py @@ -0,0 +1,211 @@ +import torch +import copy +from typing import Any, Callable, List, Optional + +from .abl_model import ABLModel +from .basic_nn import BasicNN +from lambdaLearn.Base.DeepModelMixin import DeepModelMixin + + +class ModelConverter: + """ + This class provides functionality to convert LambdaLearn models to ABL-Package models. + """ + + def __init__(self) -> None: + pass + + def convert_lambdalearn_to_ablmodel( + self, + lambdalearn_model, + loss_fn: torch.nn.Module, + optimizer_dict: dict, + scheduler_dict: Optional[dict] = None, + device: Optional[torch.device] = None, + batch_size: int = 32, + num_epochs: int = 1, + stop_loss: Optional[float] = 0.0001, + num_workers: int = 0, + save_interval: Optional[int] = None, + save_dir: Optional[str] = None, + train_transform: Callable[..., Any] = None, + test_transform: Callable[..., Any] = None, + collate_fn: Callable[[List[Any]], Any] = None, + ): + """ + Convert a lambdalearn model to an ABLModel. If the lambdalearn model is an instance of + DeepModelMixin, its network will be used as the model of BasicNN. Otherwise, the lambdalearn + model should implement ``fit`` and ``predict`` methods. + + Parameters + ---------- + lambdalearn_model : Union[DeepModelMixin, Any] + The LambdaLearn model to be converted. + loss_fn : torch.nn.Module + The loss function used for training. + optimizer_dict : dict + The dict contains necessary parameters to construct a optimizer used for training. + The optimizer class is specified by the ``optimizer`` key. + scheduler_dict : dict, optional + The dict contains necessary parameters to construct a learning rate scheduler used + for training, which will be called at the end of each run of the ``fit`` method. + The scheduler class is specified by the ``scheduler`` key. It should implement the + ``step`` method, by default None. + device : torch.device, optional + The device on which the model will be trained or used for prediction, + by default torch.device("cpu"). + batch_size : int, optional + The batch size used for training, by default 32. + num_epochs : int, optional + The number of epochs used for training, by default 1. + stop_loss : float, optional + The loss value at which to stop training, by default 0.0001. + num_workers : int + The number of workers used for loading data, by default 0. + save_interval : int, optional + The model will be saved every ``save_interval`` epochs during training, by default None. + save_dir : str, optional + The directory in which to save the model during training, by default None. + train_transform : Callable[..., Any], optional + A function/transform that takes an object and returns a transformed version used + in the `fit` and `train_epoch` methods, by default None. + test_transform : Callable[..., Any], optional + A function/transform that takes an object and returns a transformed version in the + `predict`, `predict_proba` and `score` methods, , by default None. + collate_fn : Callable[[List[T]], Any], optional + The function used to collate data, by default None. + + Returns + ------- + ABLModel + The converted ABLModel instance. + """ + if isinstance(lambdalearn_model, DeepModelMixin): + base_model = self.convert_lambdalearn_to_basicnn( + lambdalearn_model, + loss_fn, + optimizer_dict, + scheduler_dict, + device, + batch_size, + num_epochs, + stop_loss, + num_workers, + save_interval, + save_dir, + train_transform, + test_transform, + collate_fn, + ) + return ABLModel(base_model) + + if not (hasattr(lambdalearn_model, "fit") and hasattr(lambdalearn_model, "predict")): + raise NotImplementedError( + "The lambdalearn_model should be an instance of DeepModelMixin, or implement " + + "fit and predict methods." + ) + + return ABLModel(lambdalearn_model) + + def convert_lambdalearn_to_basicnn( + self, + lambdalearn_model: DeepModelMixin, + loss_fn: torch.nn.Module, + optimizer_dict: dict, + scheduler_dict: Optional[dict] = None, + device: Optional[torch.device] = None, + batch_size: int = 32, + num_epochs: int = 1, + stop_loss: Optional[float] = 0.0001, + num_workers: int = 0, + save_interval: Optional[int] = None, + save_dir: Optional[str] = None, + train_transform: Callable[..., Any] = None, + test_transform: Callable[..., Any] = None, + collate_fn: Callable[[List[Any]], Any] = None, + ): + """ + Convert a lambdalearn model to a BasicNN. If the lambdalearn model is an instance of + DeepModelMixin, its network will be used as the model of BasicNN. + + Parameters + ---------- + lambdalearn_model : Union[DeepModelMixin, Any] + The LambdaLearn model to be converted. + loss_fn : torch.nn.Module + The loss function used for training. + optimizer_dict : dict + The dict contains necessary parameters to construct a optimizer used for training. + scheduler_dict : dict, optional + The dict contains necessary parameters to construct a learning rate scheduler used + for training, which will be called at the end of each run of the ``fit`` method. + The scheduler class is specified by the ``scheduler`` key. It should implement the + ``step`` method, by default None. + device : torch.device, optional + The device on which the model will be trained or used for prediction, + by default torch.device("cpu"). + batch_size : int, optional + The batch size used for training, by default 32. + num_epochs : int, optional + The number of epochs used for training, by default 1. + stop_loss : float, optional + The loss value at which to stop training, by default 0.0001. + num_workers : int + The number of workers used for loading data, by default 0. + save_interval : int, optional + The model will be saved every ``save_interval`` epochs during training, by default None. + save_dir : str, optional + The directory in which to save the model during training, by default None. + train_transform : Callable[..., Any], optional + A function/transform that takes an object and returns a transformed version used + in the `fit` and `train_epoch` methods, by default None. + test_transform : Callable[..., Any], optional + A function/transform that takes an object and returns a transformed version in the + `predict`, `predict_proba` and `score` methods, , by default None. + collate_fn : Callable[[List[T]], Any], optional + The function used to collate data, by default None. + + Returns + ------- + BasicNN + The converted BasicNN instance. + """ + if isinstance(lambdalearn_model, DeepModelMixin): + if not isinstance(lambdalearn_model.network, torch.nn.Module): + raise NotImplementedError( + "Expected lambdalearn_model.network to be a torch.nn.Module, " + + f"but got {type(lambdalearn_model.network)}" + ) + # Only use the network part and device of the lambdalearn model + network = copy.deepcopy(lambdalearn_model.network) + optimizer_class = optimizer_dict["optimizer"] + optimizer_dict.pop("optimizer") + optimizer = optimizer_class(network.parameters(), **optimizer_dict) + if scheduler_dict is not None: + scheduler_class = scheduler_dict["scheduler"] + scheduler_dict.pop("scheduler") + scheduler = scheduler_class(optimizer, **scheduler_dict) + else: + scheduler = None + device = lambdalearn_model.device if device is None else device + base_model = BasicNN( + model=network, + loss_fn=loss_fn, + optimizer=optimizer, + scheduler=scheduler, + device=device, + batch_size=batch_size, + num_epochs=num_epochs, + stop_loss=stop_loss, + num_workers=num_workers, + save_interval=save_interval, + save_dir=save_dir, + train_transform=train_transform, + test_transform=test_transform, + collate_fn=collate_fn, + ) + return base_model + else: + raise NotImplementedError( + "The lambdalearn_model should be an instance of DeepModelMixin." + ) diff --git a/abl/learning/torch_dataset/__init__.py b/abl/learning/torch_dataset/__init__.py new file mode 100644 index 0000000..b8237a2 --- /dev/null +++ b/abl/learning/torch_dataset/__init__.py @@ -0,0 +1,9 @@ +from .classification_dataset import ClassificationDataset +from .prediction_dataset import PredictionDataset +from .regression_dataset import RegressionDataset + +__all__ = [ + "ClassificationDataset", + "PredictionDataset", + "RegressionDataset", +] diff --git a/abl/learning/torch_dataset/classification_dataset.py b/abl/learning/torch_dataset/classification_dataset.py new file mode 100644 index 0000000..a45acd3 --- /dev/null +++ b/abl/learning/torch_dataset/classification_dataset.py @@ -0,0 +1,66 @@ +from typing import Any, Callable, List, Tuple, Optional + +import torch +from torch.utils.data import Dataset + + +class ClassificationDataset(Dataset): + """ + Dataset used for classification task. + + Parameters + ---------- + X : List[Any] + The input data. + Y : List[int] + The target data. + transform : Callable[..., Any], optional + A function/transform that takes an object and returns a transformed version. + Defaults to None. + """ + + def __init__(self, X: List[Any], Y: List[int], transform: Optional[Callable[..., Any]] = None): + if (not isinstance(X, list)) or (not isinstance(Y, list)): + raise ValueError("X and Y should be of type list.") + if len(X) != len(Y): + raise ValueError("Length of X and Y must be equal.") + + self.X = X + self.Y = torch.LongTensor(Y) + self.transform = transform + + def __len__(self) -> int: + """ + Return the length of the dataset. + + Returns + ------- + int + The length of the dataset. + """ + return len(self.X) + + def __getitem__(self, index: int) -> Tuple[Any, torch.Tensor]: + """ + Get the item at the given index. + + Parameters + ---------- + index : int + The index of the item to get. + + Returns + ------- + Tuple[Any, torch.Tensor] + A tuple containing the object and its label. + """ + if index >= len(self): + raise ValueError("index range error") + + x = self.X[index] + if self.transform is not None: + x = self.transform(x) + + y = self.Y[index] + + return x, y diff --git a/abl/learning/torch_dataset/prediction_dataset.py b/abl/learning/torch_dataset/prediction_dataset.py new file mode 100644 index 0000000..1abf12e --- /dev/null +++ b/abl/learning/torch_dataset/prediction_dataset.py @@ -0,0 +1,58 @@ +from typing import Any, Callable, List, Tuple, Optional + +import torch +from torch.utils.data import Dataset + + +class PredictionDataset(Dataset): + """ + Dataset used for prediction. + + Parameters + ---------- + X : List[Any] + The input data. + transform : Callable[..., Any], optional + A function/transform that takes an object and returns a transformed version. + Defaults to None. + """ + + def __init__(self, X: List[Any], transform: Optional[Callable[..., Any]] = None): + if not isinstance(X, list): + raise ValueError("X should be of type list.") + + self.X = X + self.transform = transform + + def __len__(self) -> int: + """ + Return the length of the dataset. + + Returns + ------- + int + The length of the dataset. + """ + return len(self.X) + + def __getitem__(self, index: int) -> Tuple[Any, torch.Tensor]: + """ + Get the item at the given index. + + Parameters + ---------- + index : int + The index of the item to get. + + Returns + ------- + Tuple[Any, torch.Tensor] + A tuple containing the object and its label. + """ + if index >= len(self): + raise ValueError("index range error") + + x = self.X[index] + if self.transform is not None: + x = self.transform(x) + return x diff --git a/abl/learning/torch_dataset/regression_dataset.py b/abl/learning/torch_dataset/regression_dataset.py new file mode 100644 index 0000000..956b38c --- /dev/null +++ b/abl/learning/torch_dataset/regression_dataset.py @@ -0,0 +1,56 @@ +from typing import Any, List, Tuple + +from torch.utils.data import Dataset + + +class RegressionDataset(Dataset): + """ + Dataset used for regression task. + + Parameters + ---------- + X : List[Any] + A list of objects representing the input data. + Y : List[Any] + A list of objects representing the output data. + """ + + def __init__(self, X: List[Any], Y: List[Any]): + if (not isinstance(X, list)) or (not isinstance(Y, list)): + raise ValueError("X and Y should be of type list.") + if len(X) != len(Y): + raise ValueError("Length of X and Y must be equal.") + + self.X = X + self.Y = Y + + def __len__(self): + """Return the length of the dataset. + + Returns + ------- + int + The length of the dataset. + """ + return len(self.X) + + def __getitem__(self, index: int) -> Tuple[Any, Any]: + """Get an item from the dataset. + + Parameters + ---------- + index : int + The index of the item to retrieve. + + Returns + ------- + Tuple[Any, Any] + A tuple containing the input and output data at the specified index. + """ + if index >= len(self): + raise ValueError("index range error") + + x = self.X[index] + y = self.Y[index] + + return x, y diff --git a/abl/reasoning/__init__.py b/abl/reasoning/__init__.py new file mode 100644 index 0000000..9d5a219 --- /dev/null +++ b/abl/reasoning/__init__.py @@ -0,0 +1,4 @@ +from .kb import GroundKB, KBBase, PrologKB +from .reasoner import Reasoner + +__all__ = ["KBBase", "GroundKB", "PrologKB", "Reasoner"] diff --git a/abl/reasoning/kb.py b/abl/reasoning/kb.py new file mode 100644 index 0000000..8e31ab0 --- /dev/null +++ b/abl/reasoning/kb.py @@ -0,0 +1,622 @@ +import bisect +import inspect +import logging +import os +from abc import ABC, abstractmethod +from collections import defaultdict +from itertools import combinations, product +from multiprocessing import Pool +from typing import Any, Callable, List, Optional + +import numpy as np + +from ..utils.cache import abl_cache +from ..utils.logger import print_log +from ..utils.utils import flatten, hamming_dist, reform_list, to_hashable + + +class KBBase(ABC): + """ + Base class for knowledge base. + + Parameters + ---------- + pseudo_label_list : List[Any] + List of possible pseudo-labels. It's recommended to arrange the pseudo-labels in this + list so that each aligns with its corresponding index in the base model: the first with + the 0th index, the second with the 1st, and so forth. + max_err : float, optional + The upper tolerance limit when comparing the similarity between the reasoning result of + pseudo-labels and the ground truth. This is only applicable when the reasoning + result is of a numerical type. This is particularly relevant for regression problems where + exact matches might not be feasible. Defaults to 1e-10. + use_cache : bool, optional + Whether to use abl_cache for previously abduced candidates to speed up subsequent + operations. Defaults to True. + key_func : Callable, optional + A function employed for hashing in abl_cache. This is only operational when use_cache + is set to True. Defaults to ``to_hashable``. + cache_size: int, optional + The cache size in abl_cache. This is only operational when use_cache is set to + True. Defaults to 4096. + + Notes + ----- + Users should derive from this base class to build their own knowledge base. For the + user-build KB (a derived subclass), it's only required for the user to provide the + ``pseudo_label_list`` and override the ``logic_forward`` function (specifying how to + perform logical reasoning). After that, other operations (e.g. how to perform abductive + reasoning) will be automatically set up. + """ + + def __init__( + self, + pseudo_label_list: List[Any], + max_err: float = 1e-10, + use_cache: bool = True, + key_func: Callable = to_hashable, + cache_size: int = 4096, + ): + if not isinstance(pseudo_label_list, list): + raise TypeError(f"pseudo_label_list should be list, got {type(pseudo_label_list)}") + self.pseudo_label_list = pseudo_label_list + self.max_err = max_err + + self.use_cache = use_cache + self.key_func = key_func + self.cache_size = cache_size + + argspec = inspect.getfullargspec(self.logic_forward) + self._num_args = len(argspec.args) - 1 + if ( + self._num_args == 2 and self.use_cache + ): # If the logic_forward function has 2 arguments, then disable cache + self.use_cache = False + print_log( + "The logic_forward function has 2 arguments, so the cache is disabled. ", + logger="current", + level=logging.WARNING, + ) + # TODO 添加半监督 + # TODO 添加consistency measure+max_err容忍错误 + + @abstractmethod + def logic_forward(self, pseudo_label: List[Any], x: Optional[List[Any]] = None) -> Any: + """ + How to perform (deductive) logical reasoning, i.e. matching pseudo-labels to + their reasoning result. Users are required to provide this. + + Parameters + ---------- + pseudo_label : List[Any] + Pseudo-labels of an example. + x : List[Any], optional + The example. If deductive logical reasoning does not require any + information from the example, the overridden function provided by the user can omit + this parameter. + + Returns + ------- + Any + The reasoning result. + """ + + def abduce_candidates( + self, + pseudo_label: List[Any], + y: Any, + x: List[Any], + max_revision_num: int, + require_more_revision: int, + ) -> List[List[Any]]: + """ + Perform abductive reasoning to get a candidate compatible with the knowledge base. + + Parameters + ---------- + pseudo_label : List[Any] + Pseudo-labels of an example (to be revised by abductive reasoning). + y : Any + Ground truth of the reasoning result for the example. + x : List[Any] + The example. If the information from the example + is not required in the reasoning process, then this parameter will not have + any effect. + max_revision_num : int + The upper limit on the number of revised labels for each example. + require_more_revision : int + Specifies additional number of revisions permitted beyond the minimum required. + + Returns + ------- + Tuple[List[List[Any]], List[Any]] + A tuple of two element. The first element is a list of candidate revisions, i.e. revised + pseudo-labels of the example. that are compatible with the knowledge base. The second + element is a list of reasoning results corresponding to each candidate, i.e., the + outcome of the ``logic_forward`` function. + """ + return self._abduce_by_search(pseudo_label, y, x, max_revision_num, require_more_revision) + + def _check_equal(self, reasoning_result: Any, y: Any) -> bool: + """ + Check whether the reasoning result of a pseduo label example is equal to the ground truth + (or, within the maximum error allowed for numerical results). + + Returns + ------- + bool + The result of the check. + """ + if reasoning_result is None: + return False + + if isinstance(reasoning_result, (int, float)) and isinstance(y, (int, float)): + return abs(reasoning_result - y) <= self.max_err + else: + return reasoning_result == y + + def revise_at_idx( + self, + pseudo_label: List[Any], + y: Any, + x: List[Any], + revision_idx: List[int], + ) -> List[List[Any]]: + """ + Revise the pseudo-labels at specified index positions. + + Parameters + ---------- + pseudo_label : List[Any] + Pseudo-labels of an example (to be revised). + y : Any + Ground truth of the reasoning result for the example. + x : List[Any] + The example. If the information from the example + is not required in the reasoning process, then this parameter will not have + any effect. + revision_idx : List[int] + A list specifying indices of where revisions should be made to the pseudo-labels. + + Returns + ------- + Tuple[List[List[Any]], List[Any]] + A tuple of two element. The first element is a list of candidate revisions, i.e. revised + pseudo-labels of the example that are compatible with the knowledge base. The second + element is a list of reasoning results corresponding to each candidate, i.e., the + outcome of the ``logic_forward`` function. + """ + candidates, reasoning_results = [], [] + abduce_c = product(self.pseudo_label_list, repeat=len(revision_idx)) + for c in abduce_c: + candidate = pseudo_label.copy() + for i, idx in enumerate(revision_idx): + candidate[idx] = c[i] + reasoning_result = self.logic_forward(candidate, *(x,) if self._num_args == 2 else ()) + if self._check_equal(reasoning_result, y): + candidates.append(candidate) + reasoning_results.append(reasoning_result) + return candidates, reasoning_results + + def _revision( + self, + revision_num: int, + pseudo_label: List[Any], + y: Any, + x: List[Any], + ) -> List[List[Any]]: + """ + For a specified number of labels in a pseudo-labels to revise, iterate through + all possible indices to find any candidates that are compatible with the knowledge base. + """ + new_candidates, new_reasoning_results = [], [] + revision_idx_list = combinations(range(len(pseudo_label)), revision_num) + for revision_idx in revision_idx_list: + candidates, reasoning_results = self.revise_at_idx(pseudo_label, y, x, revision_idx) + new_candidates.extend(candidates) + new_reasoning_results.extend(reasoning_results) + return new_candidates, new_reasoning_results + + @abl_cache() + def _abduce_by_search( + self, + pseudo_label: List[Any], + y: Any, + x: List[Any], + max_revision_num: int, + require_more_revision: int, + ) -> List[List[Any]]: + """ + Perform abductive reasoning by exhastive search. Specifically, begin with 0 and + continuously increase the number of labels to revise, until + candidates that are compatible with the knowledge base are found. + + Parameters + ---------- + pseudo_label : List[Any] + Pseudo-labels of an example (to be revised). + y : Any + Ground truth of the reasoning result for the example. + x : List[Any] + The example. If the information from the example + is not required in the reasoning process, then this parameter will not have + any effect. + max_revision_num : int + The upper limit on the number of revisions. + require_more_revision : int + If larger than 0, then after having found any candidates compatible with the + knowledge base, continue to increase the number of labels to + revise to get more possible compatible candidates. + + Returns + ------- + Tuple[List[List[Any]], List[Any]] + A tuple of two element. The first element is a list of candidate revisions, i.e. revised + pseudo-labels of the example that are compatible with the knowledge base. The second + element is a list of reasoning results corresponding to each candidate, i.e., the + outcome of the ``logic_forward`` function. + """ + candidates, reasoning_results = [], [] + for revision_num in range(len(pseudo_label) + 1): + new_candidates, new_reasoning_results = self._revision(revision_num, pseudo_label, y, x) + candidates.extend(new_candidates) + reasoning_results.extend(new_reasoning_results) + if len(candidates) > 0: + min_revision_num = revision_num + break + if revision_num >= max_revision_num: + return [], [] + + for revision_num in range( + min_revision_num + 1, min_revision_num + require_more_revision + 1 + ): + if revision_num > max_revision_num: + return candidates, reasoning_results + new_candidates, new_reasoning_results = self._revision(revision_num, pseudo_label, y, x) + candidates.extend(new_candidates) + reasoning_results.extend(new_reasoning_results) + return candidates, reasoning_results + + def __repr__(self): + return ( + f"{self.__class__.__name__} is a KB with " + f"pseudo_label_list={self.pseudo_label_list!r}, " + f"max_err={self.max_err!r}, " + f"use_cache={self.use_cache!r}." + ) + + +class GroundKB(KBBase): + """ + Knowledge base with a ground KB (GKB). Ground KB is a knowledge base prebuilt upon + class initialization, storing all potential candidates along with their respective + reasoning result. Ground KB can accelerate abductive reasoning in ``abduce_candidates``. + + Parameters + ---------- + pseudo_label_list : List[Any] + Refer to class ``KBBase``. + GKB_len_list : List[int] + List of possible lengths for pseudo-labels of an example. + max_err : float, optional + Refer to class ``KBBase``. + + Notes + ----- + Users can also inherit from this class to build their own knowledge base. Similar + to ``KBBase``, users are only required to provide the ``pseudo_label_list`` and override + the ``logic_forward`` function. Additionally, users should provide the ``GKB_len_list``. + After that, other operations (e.g. auto-construction of GKB, and how to perform + abductive reasoning) will be automatically set up. + """ + + def __init__( + self, + pseudo_label_list: List[Any], + GKB_len_list: List[int], + max_err: float = 1e-10, + ): + super().__init__(pseudo_label_list, max_err) + if not isinstance(GKB_len_list, list): + raise TypeError("GKB_len_list should be list, but got {type(GKB_len_list)}") + if self._num_args == 2: + raise NotImplementedError( + "GroundKB only supports 1-argument logic_forward, but got " + + f"{self._num_args}-argument logic_forward" + ) + self.GKB_len_list = GKB_len_list + self.GKB = {} + X, Y = self._get_GKB() + for x, y in zip(X, Y): + self.GKB.setdefault(len(x), defaultdict(list))[y].append(x) + + def _get_XY_list(self, args): + pre_x, post_x_it = args[0], args[1] + XY_list = [] + for post_x in post_x_it: + x = (pre_x,) + post_x + y = self.logic_forward(x) + if y is not None: + XY_list.append((x, y)) + return XY_list + + def _get_GKB(self): + """ + Prebuild the GKB according to ``pseudo_label_list`` and ``GKB_len_list``. + """ + X, Y = [], [] + for length in self.GKB_len_list: + arg_list = [] + for pre_x in self.pseudo_label_list: + post_x_it = product(self.pseudo_label_list, repeat=length - 1) + arg_list.append((pre_x, post_x_it)) + with Pool(processes=len(arg_list)) as pool: + ret_list = pool.map(self._get_XY_list, arg_list) + for XY_list in ret_list: + if len(XY_list) == 0: + continue + part_X, part_Y = zip(*XY_list) + X.extend(part_X) + Y.extend(part_Y) + if Y and isinstance(Y[0], (int, float)): + X, Y = zip(*sorted(zip(X, Y), key=lambda pair: pair[1])) + return X, Y + + def abduce_candidates( + self, + pseudo_label: List[Any], + y: Any, + x: List[Any], + max_revision_num: int, + require_more_revision: int, + ) -> List[List[Any]]: + """ + Perform abductive reasoning by directly retrieving compatible candidates from + the prebuilt GKB. In this way, the time-consuming exhaustive search can be + avoided. + + Parameters + ---------- + pseudo_label : List[Any] + Pseudo-labels of an example (to be revised by abductive reasoning). + y : Any + Ground truth of the reasoning result for the example. + x : List[Any] + The example (unused in GroundKB). + max_revision_num : int + The upper limit on the number of revised labels for each example. + require_more_revision : int + Specifies additional number of revisions permitted beyond the minimum required. + + Returns + ------- + Tuple[List[List[Any]], List[Any]] + A tuple of two element. The first element is a list of candidate revisions, i.e. revised + pseudo-labels of the example that are compatible with the knowledge base. The second + element is a list of reasoning results corresponding to each candidate, i.e., the + outcome of the ``logic_forward`` function. + """ + if self.GKB == {} or len(pseudo_label) not in self.GKB_len_list: + return [], [] + + all_candidates, all_reasoning_results = self._find_candidate_GKB(pseudo_label, y) + if len(all_candidates) == 0: + return [], [] + + cost_list = hamming_dist(pseudo_label, all_candidates) + min_revision_num = np.min(cost_list) + revision_num = min(max_revision_num, min_revision_num + require_more_revision) + idxs = np.where(cost_list <= revision_num)[0] + candidates = [all_candidates[idx] for idx in idxs] + reasoning_results = [all_reasoning_results[idx] for idx in idxs] + return candidates, reasoning_results + + def _find_candidate_GKB(self, pseudo_label: List[Any], y: Any) -> List[List[Any]]: + """ + Retrieve compatible candidates from the prebuilt GKB. For numerical reasoning results, + return all candidates and their corresponding reasoning results which fall within the + [y - max_err, y + max_err] range. + """ + if isinstance(y, (int, float)): + potential_candidates = self.GKB[len(pseudo_label)] + key_list = list(potential_candidates.keys()) + + low_key = bisect.bisect_left(key_list, y - self.max_err) + high_key = bisect.bisect_right(key_list, y + self.max_err) + + all_candidates, all_reasoning_results = [], [] + for key in key_list[low_key:high_key]: + for candidate in potential_candidates[key]: + all_candidates.append(candidate) + all_reasoning_results.append(key) + else: + all_candidates = self.GKB[len(pseudo_label)][y] + all_reasoning_results = [y] * len(all_candidates) + return all_candidates, all_reasoning_results + + def __repr__(self): + GKB_info_parts = [] + for i in self.GKB_len_list: + num_candidates = len(self.GKB[i]) if i in self.GKB else 0 + GKB_info_parts.append(f"{num_candidates} candidates of length {i}") + GKB_info = ", ".join(GKB_info_parts) + + return ( + f"{self.__class__.__name__} is a KB with " + f"pseudo_label_list={self.pseudo_label_list!r}, " + f"max_err={self.max_err!r}, " + f"use_cache={self.use_cache!r}. " + f"It has a prebuilt GKB with " + f"GKB_len_list={self.GKB_len_list!r}, " + f"and there are " + f"{GKB_info}" + f" in the GKB." + ) + + +class PrologKB(KBBase): + """ + Knowledge base provided by a Prolog (.pl) file. + + Parameters + ---------- + pseudo_label_list : List[Any] + Refer to class ``KBBase``. + pl_file : str + Prolog file containing the KB. + + Notes + ----- + Users can instantiate this class to build their own knowledge base. During the + instantiation, users are only required to provide the ``pseudo_label_list`` and ``pl_file``. + To use the default logic forward and abductive reasoning methods in this class, in the + Prolog (.pl) file, there needs to be a rule which is strictly formatted as + ``logic_forward(Pseudo_labels, Res).``, e.g., ``logic_forward([A,B], C) :- C is A+B``. + For specifics, refer to the ``logic_forward`` and ``get_query_string`` functions in this + class. Users are also welcome to override related functions for more flexible support. + """ + + def __init__(self, pseudo_label_list: List[Any], pl_file: str): + super().__init__(pseudo_label_list) + + try: + import pyswip + except (IndexError, ImportError): + print( + "A Prolog-based knowledge base is in use. Please install Swi-Prolog using the" + + "command 'sudo apt-get install swi-prolog' for Linux users, or download it " + + "following the guide in https://github.com/yuce/pyswip/blob/master/INSTALL.md " + + "for Windows and Mac users." + ) + + self.prolog = pyswip.Prolog() + self.pl_file = pl_file + if not os.path.exists(self.pl_file): + raise FileNotFoundError(f"The Prolog file {self.pl_file} does not exist.") + self.prolog.consult(self.pl_file) + + def logic_forward(self, pseudo_label: List[Any]) -> Any: + """ + Consult prolog with the query ``logic_forward(pseudo_labels, Res).``, and set the + returned ``Res`` as the reasoning results. To use this default function, there must be + a ``logic_forward`` method in the pl file to perform reasoning. + Otherwise, users would override this function. + + Parameters + ---------- + pseudo_label : List[Any] + Pseudo-labels of an example. + """ + result = list(self.prolog.query("logic_forward(%s, Res)." % pseudo_label))[0]["Res"] + if result == "true": + return True + elif result == "false": + return False + return result + + def _revision_pseudo_label( + self, + pseudo_label: List[Any], + revision_idx: List[int], + ) -> List[Any]: + import re + + revision_pseudo_label = pseudo_label.copy() + revision_pseudo_label = flatten(revision_pseudo_label) + + for idx in revision_idx: + revision_pseudo_label[idx] = "P" + str(idx) + revision_pseudo_label = reform_list(revision_pseudo_label, pseudo_label) + + regex = r"'P\d+'" + return re.sub(regex, lambda x: x.group().replace("'", ""), str(revision_pseudo_label)) + + def get_query_string( + self, + pseudo_label: List[Any], + y: Any, + x: List[Any], + revision_idx: List[int], + ) -> str: + """ + Get the query to be used for consulting Prolog. + This is a default function for demo, users would override this function to adapt to + their own Prolog file. In this demo function, return query + ``logic_forward([kept_labels, Revise_labels], Res).``. + + Parameters + ---------- + pseudo_label : List[Any] + Pseudo-labels of an example (to be revised by abductive reasoning). + y : Any + Ground truth of the reasoning result for the example. + x : List[Any] + The corresponding input example. If the information from the input + is not required in the reasoning process, then this parameter will not have + any effect. + revision_idx : List[int] + A list specifying indices of where revisions should be made to the pseudo-labels. + + Returns + ------- + str + A string of the query. + """ + query_string = "logic_forward(" + query_string += self._revision_pseudo_label(pseudo_label, revision_idx) + key_is_none_flag = y is None or (isinstance(y, list) and y[0] is None) + query_string += ",%s)." % y if not key_is_none_flag else ")." + return query_string + + def revise_at_idx( + self, + pseudo_label: List[Any], + y: Any, + x: List[Any], + revision_idx: List[int], + ) -> List[List[Any]]: + """ + Revise the pseudo-labels at specified index positions by querying Prolog. + + Parameters + ---------- + pseudo_label : List[Any] + Pseudo-labels of an example (to be revised). + y : Any + Ground truth of the reasoning result for the example. + x : List[Any] + The corresponding input example. If the information from the input + is not required in the reasoning process, then this parameter will not have + any effect. + revision_idx : List[int] + A list specifying indices of where revisions should be made to the pseudo-labels. + + Returns + ------- + Tuple[List[List[Any]], List[Any]] + A tuple of two element. The first element is a list of candidate revisions, i.e. revised + pseudo-labels of the example that are compatible with the knowledge base. The second + element is a list of reasoning results corresponding to each candidate, i.e., the + outcome of the ``logic_forward`` function. + """ + candidates, reasoning_results = [], [] + query_string = self.get_query_string(pseudo_label, y, x, revision_idx) + save_pseudo_label = pseudo_label + pseudo_label = flatten(pseudo_label) + abduce_c = [list(z.values()) for z in self.prolog.query(query_string)] + for c in abduce_c: + candidate = pseudo_label.copy() + for i, idx in enumerate(revision_idx): + candidate[idx] = c[i] + candidate = reform_list(candidate, save_pseudo_label) + candidates.append(candidate) + reasoning_results.append(y) + return candidates, reasoning_results + + def __repr__(self): + return ( + f"{self.__class__.__name__} is a KB with " + f"pseudo_label_list={self.pseudo_label_list!r}, " + f"defined by " + f"Prolog file {self.pl_file!r}." + ) diff --git a/abl/reasoning/reasoner.py b/abl/reasoning/reasoner.py new file mode 100644 index 0000000..22d1690 --- /dev/null +++ b/abl/reasoning/reasoner.py @@ -0,0 +1,351 @@ +import inspect +from typing import Any, Callable, List, Optional, Union + +import numpy as np +from zoopt import Dimension, Objective, Opt, Parameter, Solution + +from ..data.structures import ListData +from ..reasoning import KBBase +from ..utils.utils import confidence_dist, hamming_dist + + +class Reasoner: + """ + Reasoner for minimizing the inconsistency between the knowledge base and learning models. + + Parameters + ---------- + kb : class KBBase + The knowledge base to be used for reasoning. + dist_func : Union[str, Callable], optional + The distance function used to determine the cost list between each + candidate and the given prediction. The cost is also referred to as a consistency + measure, wherein the candidate with lowest cost is selected as the final + abduced label. It can be either a string representing a predefined distance + function or a callable function. The available predefined distance functions: + 'hamming' | 'confidence'. 'hamming': directly calculates the Hamming + distance between the predicted pseudo-label in the data example and each + candidate, 'confidence': calculates the distance between the prediction + and each candidate based on confidence derived from the predicted probability + in the data example. The callable function should have the signature + dist_func(data_example, candidates, candidate_idxs, reasoning_results) and must + return a cost list. Each element in this cost list should be a numerical value + representing the cost for each candidate, and the list should have the same length + as candidates. Defaults to 'confidence'. + idx_to_label : dict, optional + A mapping from index in the base model to label. If not provided, a default + order-based index to label mapping is created. Defaults to None. + max_revision : Union[int, float], optional + The upper limit on the number of revisions for each data example when + performing abductive reasoning. If float, denotes the fraction of the total + length that can be revised. A value of -1 implies no restriction on the + number of revisions. Defaults to -1. + require_more_revision : int, optional + Specifies additional number of revisions permitted beyond the minimum required + when performing abductive reasoning. Defaults to 0. + use_zoopt : bool, optional + Whether to use ZOOpt library during abductive reasoning. Defaults to False. + """ + + def __init__( + self, + kb: KBBase, + dist_func: Union[str, Callable] = "confidence", + idx_to_label: Optional[dict] = None, + max_revision: Union[int, float] = -1, + require_more_revision: int = 0, + use_zoopt: bool = False, + ): + self.kb = kb + self._check_valid_dist(dist_func) + self.dist_func = dist_func + self.use_zoopt = use_zoopt + self.max_revision = max_revision + self.require_more_revision = require_more_revision + + if idx_to_label is None: + self.idx_to_label = { + index: label for index, label in enumerate(self.kb.pseudo_label_list) + } + else: + self._check_valid_idx_to_label(idx_to_label) + self.idx_to_label = idx_to_label + self.label_to_idx = dict(zip(self.idx_to_label.values(), self.idx_to_label.keys())) + + def _check_valid_dist(self, dist_func): + if isinstance(dist_func, str): + if dist_func not in ["hamming", "confidence"]: + raise NotImplementedError( + 'Valid options for predefined dist_func include "hamming" ' + + f'and "confidence", but got {dist_func}.' + ) + return + elif callable(dist_func): + params = inspect.signature(dist_func).parameters.values() + if len(params) != 4: + raise ValueError( + "User-defined dist_func must have exactly four parameters, " + + f"but got {len(params)}." + ) + return + else: + raise TypeError( + f"dist_func must be a string or a callable function, but got {type(dist_func)}." + ) + + def _check_valid_idx_to_label(self, idx_to_label): + if not isinstance(idx_to_label, dict): + raise TypeError(f"idx_to_label should be dict, but got {type(idx_to_label)}.") + for key, value in idx_to_label.items(): + if not isinstance(key, int): + raise ValueError(f"All keys in the idx_to_label must be integers, but got {key}.") + if value not in self.kb.pseudo_label_list: + raise ValueError( + "All values in the idx_to_label must be in the pseudo_label_list, " + + f"but got {value}." + ) + + def _get_one_candidate( + self, + data_example: ListData, + candidates: List[List[Any]], + reasoning_results: List[Any], + ) -> List[Any]: + """ + Due to the nondeterminism of abductive reasoning, there could be multiple candidates + satisfying the knowledge base. When this happens, return one candidate that has the + minimum cost. If no candidates are provided, an empty list is returned. + + Parameters + ---------- + data_example : ListData + Data example. + candidates : List[List[Any]] + Multiple possible candidates. + reasoning_results : List[Any] + Corresponding reasoning results of the candidates. + + Returns + ------- + List[Any] + A selected candidate. + """ + if len(candidates) == 0: + return [] + elif len(candidates) == 1: + return candidates[0] + else: + cost_array = self._get_cost_list(data_example, candidates, reasoning_results) + candidate = candidates[np.argmin(cost_array)] + return candidate + + def _get_cost_list( + self, + data_example: ListData, + candidates: List[List[Any]], + reasoning_results: List[Any], + ) -> Union[List[Union[int, float]], np.ndarray]: + """ + Get the list of costs between each candidate and the given data example. + + Parameters + ---------- + data_example : ListData + Data example. + candidates : List[List[Any]] + Multiple possible candidates. + reasoning_results : List[Any] + Corresponding reasoning results of the candidates. + + Returns + ------- + Union[List[Union[int, float]], np.ndarray] + The list of costs. + """ + if self.dist_func == "hamming": + return hamming_dist(data_example.pred_pseudo_label, candidates) + elif self.dist_func == "confidence": + candidates_idxs = [[self.label_to_idx[x] for x in c] for c in candidates] + return confidence_dist(data_example.pred_prob, candidates_idxs) + else: + candidate_idxs = [[self.label_to_idx[x] for x in c] for c in candidates] + cost_list = self.dist_func(data_example, candidates, candidate_idxs, reasoning_results) + if len(cost_list) != len(candidates): + raise ValueError( + "The length of the array returned by dist_func must be equal to the number " + + f"of candidates. Expected length {len(candidates)}, but got {len(cost_list)}." + ) + return cost_list + + def _zoopt_get_solution( + self, + symbol_num: int, + data_example: ListData, + max_revision_num: int, + ) -> Solution: + """ + Get the optimal solution using ZOOpt library. From the solution, we can get a list of + boolean values, where '1' (True) indicates the indices chosen to be revised. + + Parameters + ---------- + symbol_num : int + Number of total symbols. + data_example : ListData + Data example. + max_revision_num : int + Specifies the maximum number of revisions allowed. + + Returns + ------- + Solution + The solution for ZOOpt library. + """ + dimension = Dimension(size=symbol_num, regs=[[0, 1]] * symbol_num, tys=[False] * symbol_num) + objective = Objective( + lambda sol: self.zoopt_score(symbol_num, data_example, sol), + dim=dimension, + constraint=lambda sol: self._constrain_revision_num(sol, max_revision_num), + ) + parameter = Parameter( + budget=self.zoopt_budget(symbol_num), intermediate_result=False, autoset=True + ) + solution = Opt.min(objective, parameter) + return solution + + def zoopt_score( + self, + symbol_num: int, + data_example: ListData, + sol: Solution, + ) -> int: + """ + Set the score for a solution. A lower score suggests that ZOOpt library + has a higher preference for this solution. + + Parameters + ---------- + symbol_num : int + Number of total symbols. + data_example : ListData + Data example. + sol: Solution + The solution for ZOOpt library. + + Returns + ------- + int + The score for the solution. + """ + revision_idx = np.where(sol.get_x() != 0)[0] + candidates, reasoning_results = self.kb.revise_at_idx( + data_example.pred_pseudo_label, data_example.Y, data_example.X, revision_idx + ) + if len(candidates) > 0: + return np.min(self._get_cost_list(data_example, candidates, reasoning_results)) + else: + return symbol_num + + def zoopt_budget(self, symbol_num: int) -> int: + """ + Set the budget for ZOOpt optimization. The function, in its default implementation, + returns a fixed budget value of 100. However, it can be adjusted to return other fixed + values, or a dynamic budget based on the number of symbols, if desired. For example, + one might choose to set the budget as 100 times ``symbol_num``. + + Parameters + ---------- + symbol_num : int + The number of symbols to be considered in the ZOOpt optimization process. Although this + parameter can be used to compute a dynamic optimization budget, by default it is not + utilized in the calculation. + + Returns + ------- + int + The budget for ZOOpt optimization. By default, this is a fixed value of 100, + irrespective of the symbol_num value. + """ + return 100 + + def _constrain_revision_num(self, solution: Solution, max_revision_num: int) -> int: + """ + Constrain that the total number of revisions chosen by the solution does not exceed + maximum number of revisions allowed. + """ + x = solution.get_x() + return max_revision_num - x.sum() + + def _get_max_revision_num(self, max_revision: Union[int, float], symbol_num: int) -> int: + """ + Get the maximum revision number according to input ``max_revision``. + """ + if not isinstance(max_revision, (int, float)): + raise TypeError(f"Parameter must be of type int or float, but got {type(max_revision)}") + + if max_revision == -1: + return symbol_num + elif isinstance(max_revision, float): + if not (0 <= max_revision <= 1): + raise ValueError( + "If max_revision is a float, it must be between 0 and 1, " + + f"but got {max_revision}" + ) + return round(symbol_num * max_revision) + else: + if max_revision < 0: + raise ValueError( + f"If max_revision is an int, it must be non-negative, but got {max_revision}" + ) + return max_revision + + def abduce(self, data_example: ListData) -> List[Any]: + """ + Perform abductive reasoning on the given data example. + + Parameters + ---------- + data_example : ListData + Data example. + + Returns + ------- + List[Any] + A revised pseudo-labels of the example through abductive reasoning, which is compatible + with the knowledge base. + """ + symbol_num = data_example.elements_num("pred_pseudo_label") + max_revision_num = self._get_max_revision_num(self.max_revision, symbol_num) + + if self.use_zoopt: + solution = self._zoopt_get_solution(symbol_num, data_example, max_revision_num) + revision_idx = np.where(solution.get_x() != 0)[0] + candidates, reasoning_results = self.kb.revise_at_idx( + pseudo_label=data_example.pred_pseudo_label, + y=data_example.Y, + x=data_example.X, + revision_idx=revision_idx, + ) + else: + candidates, reasoning_results = self.kb.abduce_candidates( + pseudo_label=data_example.pred_pseudo_label, + y=data_example.Y, + x=data_example.X, + max_revision_num=max_revision_num, + require_more_revision=self.require_more_revision, + ) + + candidate = self._get_one_candidate(data_example, candidates, reasoning_results) + return candidate + + def batch_abduce(self, data_examples: ListData) -> List[List[Any]]: + """ + Perform abductive reasoning on the given prediction data examples. + For detailed information, refer to ``abduce``. + """ + abduced_pseudo_label = [self.abduce(data_example) for data_example in data_examples] + data_examples.abduced_pseudo_label = abduced_pseudo_label + return abduced_pseudo_label + + def __call__(self, data_examples: ListData) -> List[List[Any]]: + return self.batch_abduce(data_examples) diff --git a/abl/utils/__init__.py b/abl/utils/__init__.py new file mode 100644 index 0000000..65c5337 --- /dev/null +++ b/abl/utils/__init__.py @@ -0,0 +1,23 @@ +from .cache import Cache, abl_cache +from .logger import ABLLogger, print_log +from .utils import ( + confidence_dist, + flatten, + hamming_dist, + reform_list, + to_hashable, + tab_data_to_tuple, +) + +__all__ = [ + "Cache", + "ABLLogger", + "print_log", + "confidence_dist", + "flatten", + "hamming_dist", + "reform_list", + "to_hashable", + "abl_cache", + "tab_data_to_tuple", +] diff --git a/abl/utils/cache.py b/abl/utils/cache.py new file mode 100644 index 0000000..3687982 --- /dev/null +++ b/abl/utils/cache.py @@ -0,0 +1,99 @@ +from typing import Callable, Generic, TypeVar + +K = TypeVar("K") +T = TypeVar("T") +PREV, NEXT, KEY, RESULT = 0, 1, 2, 3 # names for the link fields + + +class Cache(Generic[K, T]): + def __init__(self, func: Callable[[K], T]): + """Create cache + + :param func: Function this cache evaluates + :param cache: If true, do in memory caching. + :param cache_root: If not None, cache to files at the provided path. + :param key_func: Convert the key into a hashable object if needed + """ + self.func = func + self.has_init = False + + def __getitem__(self, obj, *args) -> T: + return self.get_from_dict(obj, *args) + + def clear_cache(self): + """Invalidate entire cache.""" + self.cache_dict.clear() + + def _init_cache(self, obj): + if self.has_init: + return + + self.cache = True + self.cache_dict = dict() + self.key_func = obj.key_func + self.max_size = obj.cache_size + + self.hits, self.misses = 0, 0 + self.full = False + self.root = [] # root of the circular doubly linked list + self.root[:] = [self.root, self.root, None, None] + + self.has_init = True + + def get_from_dict(self, obj, *args) -> T: + """Implements dict based cache.""" + # x is not used in cache key + pred_pseudo_label, y, x, *res_args = args + cache_key = (self.key_func(pred_pseudo_label), self.key_func(y), *res_args) + link = self.cache_dict.get(cache_key) + if link is not None: + # Move the link to the front of the circular queue + link_prev, link_next, _key, result = link + link_prev[NEXT] = link_next + link_next[PREV] = link_prev + last = self.root[PREV] + last[NEXT] = self.root[PREV] = link + link[PREV] = last + link[NEXT] = self.root + self.hits += 1 + return result + self.misses += 1 + + result = self.func(obj, *args) + + if self.full: + # Use the old root to store the new key and result. + oldroot = self.root + oldroot[KEY] = cache_key + oldroot[RESULT] = result + # Empty the oldest link and make it the new root. + self.root = oldroot[NEXT] + oldkey = self.root[KEY] + self.root[KEY] = self.root[RESULT] = None + # Now update the cache dictionary. + del self.cache_dict[oldkey] + self.cache_dict[cache_key] = oldroot + else: + # Put result in a new link at the front of the queue. + last = self.root[PREV] + link = [last, self.root, cache_key, result] + last[NEXT] = self.root[PREV] = self.cache_dict[cache_key] = link + if isinstance(self.max_size, int): + self.full = len(self.cache_dict) >= self.max_size + return result + + +def abl_cache(): + def decorator(func): + cache_instance = Cache(func) + + def wrapper(obj, *args): + if obj.use_cache: + cache_instance._init_cache(obj) + return cache_instance.get_from_dict(obj, *args) + else: + return func(obj, *args) + + return wrapper + + return decorator diff --git a/abl/utils/logger.py b/abl/utils/logger.py new file mode 100644 index 0000000..298e9f6 --- /dev/null +++ b/abl/utils/logger.py @@ -0,0 +1,344 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import logging +import os +import os.path as osp +import sys +from logging import Logger, LogRecord +from typing import Optional, Union + +from termcolor import colored + +from .manager import ManagerMixin, _accquire_lock, _release_lock + + +class FilterDuplicateWarning(logging.Filter): + """ + Filter for eliminating repeated warning messages in logging. + + This filter checks for duplicate warning messages and allows only the first occurrence of + each message to be logged, filtering out subsequent duplicates. + + Parameters + ---------- + name : str, optional + The name of the filter, by default "abl". + """ + + def __init__(self, name: Optional[str] = "abl"): + super().__init__(name) + self.seen: set = set() + + def filter(self, record: LogRecord) -> bool: + """Filter the repeated warning message. + + Args: + record (LogRecord): The log record. + + Returns: + bool: Whether to output the log record. + """ + if record.levelno != logging.WARNING: + return True + + if record.msg not in self.seen: + self.seen.add(record.msg) + return True + return False + + +class ABLFormatter(logging.Formatter): + """ + Colorful format for ABLLogger. If the log level is error, the logger will + additionally output the location of the code. + + Parameters + ---------- + color : bool, optional + Whether to use colorful format. filehandler is not + allowed to use color format, otherwise it will be garbled. + Defaults to True. + blink : bool, optional + Whether to blink the ``INFO`` and ``DEBUG`` logging + level. Defaults to False. + kwargs : dict + Keyword arguments passed to + :meth:``logging.Formatter.__init__``. + """ + + _color_mapping: dict = dict(ERROR="red", WARNING="yellow", INFO="white", DEBUG="green") + + def __init__(self, color: bool = True, blink: bool = False, **kwargs): + super().__init__(**kwargs) + assert not (not color and blink), "blink should only be available when color is True" + # Get prefix format according to color. + error_prefix = self._get_prefix("ERROR", color, blink=True) + warn_prefix = self._get_prefix("WARNING", color, blink=True) + info_prefix = self._get_prefix("INFO", color, blink) + debug_prefix = self._get_prefix("DEBUG", color, blink) + + # Config output format. + self.err_format = ( + f"%(asctime)s - %(name)s - {error_prefix} - " + "%(pathname)s - %(funcName)s - %(lineno)d - " + "%(message)s" + ) + self.warn_format = f"%(asctime)s - %(name)s - {warn_prefix} - %(" "message)s" + self.info_format = f"%(asctime)s - %(name)s - {info_prefix} - %(" "message)s" + self.debug_format = f"%(asctime)s - %(name)s - {debug_prefix} - %(" "message)s" + + def _get_prefix(self, level: str, color: bool, blink: bool = False) -> str: + """ + Get the prefix of the target log level. + + Parameters + ---------- + level : str + Log level. + color : bool + Whether to get a colorful prefix. + blink : bool, optional + Whether the prefix will blink. Defaults to False. + + Returns + ------- + str + The plain or colorful prefix. + """ + if color: + attrs = ["underline"] + if blink: + attrs.append("blink") + prefix = colored(level, self._color_mapping[level], attrs=attrs) + else: + prefix = level + return prefix + + def format(self, record: LogRecord) -> str: + """ + Override the ``logging.Formatter.format`` method. Output the + message according to the specified log level. + + Parameters + ---------- + record : LogRecord + A LogRecord instance representing an event being logged. + + Returns + ------- + str + Formatted result. + """ + if record.levelno == logging.ERROR: + self._style._fmt = self.err_format + elif record.levelno == logging.WARNING: + self._style._fmt = self.warn_format + elif record.levelno == logging.INFO: + self._style._fmt = self.info_format + elif record.levelno == logging.DEBUG: + self._style._fmt = self.debug_format + + result = logging.Formatter.format(self, record) + return result + + +class ABLLogger(Logger, ManagerMixin): + """ + Formatted logger used to record messages with different log levels and features. + + ``ABLLogger`` provides a formatted logger that can log messages with different + log levels. It allows the creation of logger instances in a similar manner to ``ManagerMixin``. + The logger has features like distributed log storage and colored terminal output for different + log levels. + + Parameters + ---------- + name : str + Global instance name. + logger_name : str, optional + ``name`` attribute of ``logging.Logger`` instance. Defaults to 'abl'. + log_file : str, optional + The log filename. If specified, a ``FileHandler`` will be added to the logger. + Defaults to None. + log_level : Union[int, str], optional + The log level of the handler. Defaults to 'INFO'. + If log level is 'DEBUG', distributed logs will be saved during distributed training. + file_mode : str, optional + The file mode used to open log file. Defaults to 'w'. + + Notes + ----- + - The ``name`` of the logger and the ``instance_name`` of ``ABLLogger`` could be different. + ``ABLLogger`` instances are retrieved using ``ABLLogger.get_instance``, not + ``logging.getLogger``. This ensures ``ABLLogger`` is not influenced by third-party logging + configurations. + - Unlike ``logging.Logger``, ``ABLLogger`` will not log warning or error messages without + ``Handler``. + + Examples + -------- + >>> logger = ABLLogger.get_instance(name='ABLLogger', logger_name='Logger') + >>> # Although logger has a name attribute like ``logging.Logger`` + >>> # We cannot get logger instance by ``logging.getLogger``. + >>> assert logger.name == 'Logger' + >>> assert logger.instance_name == 'ABLLogger' + >>> assert id(logger) != id(logging.getLogger('Logger')) + >>> # Get logger that does not store logs. + >>> logger1 = ABLLogger.get_instance('logger1') + >>> # Get logger only save rank0 logs. + >>> logger2 = ABLLogger.get_instance('logger2', log_file='out.log') + >>> # Get logger only save multiple ranks logs. + >>> logger3 = ABLLogger.get_instance('logger3', log_file='out.log', distributed=True) + """ + + def __init__( + self, + name: str, + logger_name="abl", + log_file: Optional[str] = None, + log_level: Union[int, str] = "INFO", + file_mode: str = "w", + ): + Logger.__init__(self, logger_name) + ManagerMixin.__init__(self, name) + if isinstance(log_level, str): + log_level = logging._nameToLevel[log_level] + + stream_handler = logging.StreamHandler(stream=sys.stdout) + # ``StreamHandler`` record month, day, hour, minute, and second + # timestamp. + stream_handler.setFormatter(ABLFormatter(color=True, datefmt="%m/%d %H:%M:%S")) + stream_handler.setLevel(log_level) + stream_handler.addFilter(FilterDuplicateWarning(logger_name)) + self.handlers.append(stream_handler) + + if log_file is None: + import time + + local_time = time.strftime("%Y%m%d_%H_%M_%S", time.localtime()) + + _log_dir = os.path.join("results", local_time) + self._log_dir = _log_dir + if not os.path.exists(_log_dir): + os.makedirs(_log_dir) + log_file = osp.join(_log_dir, local_time + ".log") + + file_handler = logging.FileHandler(log_file, file_mode) + file_handler.setFormatter(ABLFormatter(color=False, datefmt="%Y/%m/%d %H:%M:%S")) + file_handler.setLevel(log_level) + file_handler.addFilter(FilterDuplicateWarning(logger_name)) + self.handlers.append(file_handler) + self._log_file = log_file + + @property + def log_file(self): + return self._log_file + + @property + def log_dir(self): + return self._log_dir + + @classmethod + def get_current_instance(cls) -> "ABLLogger": + """ + Get the latest created ``ABLLogger`` instance. + + Returns + ------- + ABLLogger + The latest created ``ABLLogger`` instance. If no instance has been created, + returns a logger with the instance name "abl". + """ + if not cls._instance_dict: + cls.get_instance("abl") + return super().get_current_instance() + + def callHandlers(self, record: LogRecord) -> None: + """ + Pass a record to all relevant handlers. + + Override the ``callHandlers`` method in ``logging.Logger`` to avoid + multiple warning messages in DDP mode. This method loops through all + handlers of the logger instance and its parents in the logger hierarchy. + + Parameters + ---------- + record : LogRecord + A ``LogRecord`` instance containing the logged message. + """ + for handler in self.handlers: + if record.levelno >= handler.level: + handler.handle(record) + + def setLevel(self, level): + """ + Set the logging level of this logger. + + Override the ``setLevel`` method to clear caches of all ``ABLLogger`` instances + managed by ``ManagerMixin``. The level must be an int or a str. + + Parameters + ---------- + level : Union[int, str] + The logging level to set. + """ + self.level = logging._checkLevel(level) + _accquire_lock() + # The same logic as ``logging.Manager._clear_cache``. + for logger in ABLLogger._instance_dict.values(): + logger._cache.clear() + _release_lock() + + +def print_log( + msg, + logger: Optional[Union[Logger, str]] = None, + level: Optional[int] = logging.INFO, +) -> None: + """ + Print a log message using the specified logger or a default method. + + This function logs a message with a given logger, if provided, or prints it using + the standard ``print`` function. It supports special logger types such as 'silent' + and 'current'. + + Parameters + ---------- + msg : str + The message to be logged. + logger : Union[Logger, str], optional + The logger to use for logging the message. It can be a ``logging.Logger`` instance, a string + specifying the logger name, 'silent', 'current', or None. If None, the ``print`` + method is used. + - 'silent': No message will be printed. + - 'current': Use the latest created logger to log the message. + - other str: The instance name of the logger. A ``ValueError`` is raised if the logger has + not been created. + - None: The ``print()`` method is used for logging. + level : int, optional + The logging level. This is only applicable when ``logger`` is a Logger object, 'current', + or a named logger instance. The default is ``logging.INFO``. + """ + if logger is None: + print(msg) + elif isinstance(logger, logging.Logger): + logger.log(level, msg) + elif logger == "silent": + pass + elif logger == "current": + logger_instance = ABLLogger.get_current_instance() + logger_instance.log(level, msg) + elif isinstance(logger, str): + # If the type of ``logger`` is ``str``, but not with value of ``current`` or + # ``silent``, we assume it indicates the name of the logger. If the + # corresponding logger has not been created, ``print_log`` will raise + # a ``ValueError``. + if ABLLogger.check_instance_created(logger): + logger_instance = ABLLogger.get_instance(logger) + logger_instance.log(level, msg) + else: + raise ValueError(f"ABLLogger: {logger} has not been created!") + else: + raise TypeError( + "``logger`` should be either a logging.Logger object, str, " + f'"silent", "current" or None, but got {type(logger)}' + ) diff --git a/abl/utils/manager.py b/abl/utils/manager.py new file mode 100644 index 0000000..d93b784 --- /dev/null +++ b/abl/utils/manager.py @@ -0,0 +1,169 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import inspect +import threading +import warnings +from collections import OrderedDict +from typing import Type, TypeVar + +_lock = threading.RLock() +T = TypeVar("T") + + +def _accquire_lock() -> None: + """Acquire the module-level lock for serializing access to shared data. + + This should be released with _release_lock(). + """ + if _lock: + _lock.acquire() + + +def _release_lock() -> None: + """Release the module-level lock acquired by calling _accquire_lock().""" + if _lock: + _lock.release() + + +class ManagerMeta(type): + """The metaclass for global accessible class. + + The subclasses inheriting from ``ManagerMeta`` will manage their + own ``_instance_dict`` and root instances. The constructors of subclasses + must contain the ``name`` argument. + + Examples: + >>> class SubClass1(metaclass=ManagerMeta): + >>> def __init__(self, *args, **kwargs): + >>> pass + AssertionError: .__init__ must have the + name argument. + >>> class SubClass2(metaclass=ManagerMeta): + >>> def __init__(self, name): + >>> pass + >>> # valid format. + """ + + def __init__(cls, *args): + cls._instance_dict = OrderedDict() + params = inspect.getfullargspec(cls) + params_names = params[0] if params[0] else [] + assert "name" in params_names, f"{cls} must have the `name` argument" + super().__init__(*args) + + +class ManagerMixin(metaclass=ManagerMeta): + """``ManagerMixin`` is the base class for classes that have global access + requirements. + + The subclasses inheriting from ``ManagerMixin`` can get their + global instances. + + Examples: + >>> class GlobalAccessible(ManagerMixin): + >>> def __init__(self, name=''): + >>> super().__init__(name) + >>> + >>> GlobalAccessible.get_instance('name') + >>> instance_1 = GlobalAccessible.get_instance('name') + >>> instance_2 = GlobalAccessible.get_instance('name') + >>> assert id(instance_1) == id(instance_2) + + Args: + name (str): Name of the instance. Defaults to ''. + """ + + def __init__(self, name: str = "", **kwargs): + assert isinstance(name, str) and name, "name argument must be an non-empty string." + self._instance_name = name + + @classmethod + def get_instance(cls: Type[T], name: str, **kwargs) -> T: + """Get subclass instance by name if the name exists. + + If corresponding name instance has not been created, ``get_instance`` + will create an instance, otherwise ``get_instance`` will return the + corresponding instance. + + Examples + >>> instance1 = GlobalAccessible.get_instance('name1') + >>> # Create name1 instance. + >>> instance.instance_name + name1 + >>> instance2 = GlobalAccessible.get_instance('name1') + >>> # Get name1 instance. + >>> assert id(instance1) == id(instance2) + + Args: + name (str): Name of instance. Defaults to ''. + + Returns: + object: Corresponding name instance, the latest instance, or root + instance. + """ + _accquire_lock() + assert isinstance(name, str), f"type of name should be str, but got {type(cls)}" + instance_dict = cls._instance_dict # type: ignore + # Get the instance by name. + if name not in instance_dict: + instance = cls(name=name, **kwargs) # type: ignore + instance_dict[name] = instance # type: ignore + elif kwargs: + warnings.warn( + f"{cls} instance named of {name} has been created, " + "the method `get_instance` should not accept any other " + "arguments" + ) + # Get latest instantiated instance or root instance. + _release_lock() + return instance_dict[name] + + @classmethod + def get_current_instance(cls): + """Get latest created instance. + + Before calling ``get_current_instance``, The subclass must have called + ``get_instance(xxx)`` at least once. + + Examples + >>> instance = GlobalAccessible.get_current_instance() + AssertionError: At least one of name and current needs to be set + >>> instance = GlobalAccessible.get_instance('name1') + >>> instance.instance_name + name1 + >>> instance = GlobalAccessible.get_current_instance() + >>> instance.instance_name + name1 + + Returns: + object: Latest created instance. + """ + _accquire_lock() + if not cls._instance_dict: + raise RuntimeError( + f"Before calling {cls.__name__}.get_current_instance(), you " + "should call get_instance(name=xxx) at least once." + ) + name = next(iter(reversed(cls._instance_dict))) + _release_lock() + return cls._instance_dict[name] + + @classmethod + def check_instance_created(cls, name: str) -> bool: + """Check whether the name corresponding instance exists. + + Args: + name (str): Name of instance. + + Returns: + bool: Whether the name corresponding instance exists. + """ + return name in cls._instance_dict + + @property + def instance_name(self) -> str: + """Get the name of instance. + + Returns: + str: Name of instance. + """ + return self._instance_name diff --git a/abl/utils/utils.py b/abl/utils/utils.py new file mode 100644 index 0000000..66e83f8 --- /dev/null +++ b/abl/utils/utils.py @@ -0,0 +1,180 @@ +from typing import List, Any, Union, Tuple, Optional + +import numpy as np + + +def flatten(nested_list: List[Union[Any, List[Any], Tuple[Any, ...]]]) -> List[Any]: + """ + Flattens a nested list at the first level. + + Parameters + ---------- + nested_list : List[Union[Any, List[Any], Tuple[Any, ...]]] + A list which might contain sublists or tuples at the first level. + + Returns + ------- + List[Any] + A flattened version of the input list, where only the first + level of sublists and tuples are reduced. + """ + if not isinstance(nested_list, list): + return nested_list + + flattened_list = [] + for item in nested_list: + if isinstance(item, (list, tuple)): + flattened_list.extend(item) + else: + flattened_list.append(item) + + return flattened_list + + +def reform_list( + flattened_list: List[Any], structured_list: List[Union[Any, List[Any], Tuple[Any, ...]]] +) -> List[List[Any]]: + """ + Reform the list based on the structure of ``structured_list``. + + Parameters + ---------- + flattened_list : List[Any] + A flattened list of elements. + structured_list : List[Union[Any, List[Any], Tuple[Any, ...]]] + A list that reflects the desired structure, which may contain sublists or tuples. + + Returns + ------- + List[List[Any]] + A reformed list that mimics the structure of ``structured_list``. + """ + if not isinstance(structured_list[0], (list, tuple)): + return flattened_list + + reformed_list = [] + idx_start = 0 + for elem in structured_list: + idx_end = idx_start + len(elem) + reformed_list.append(flattened_list[idx_start:idx_end]) + idx_start = idx_end + + return reformed_list + + +def hamming_dist(pred_pseudo_label: List[Any], candidates: List[List[Any]]) -> np.ndarray: + """ + Compute the Hamming distance between two arrays. + + Parameters + ---------- + pred_pseudo_label : List[Any] + Pseudo-labels of an example. + candidates : List[List[Any]] + Multiple possible candidates. + + Returns + ------- + np.ndarray + Hamming distances computed for each candidate. + """ + pred_pseudo_label = np.array(pred_pseudo_label) + candidates = np.array(candidates) + + # Ensuring that pred_pseudo_label is broadcastable to the shape of candidates + pred_pseudo_label = np.expand_dims(pred_pseudo_label, 0) + + return np.sum(pred_pseudo_label != candidates, axis=1) + + +def confidence_dist(pred_prob: List[np.ndarray], candidates_idxs: List[List[Any]]) -> np.ndarray: + """ + Compute the confidence distance between prediction probabilities and candidates. + + Parameters + ---------- + pred_prob : List[np.ndarray] + Prediction probability distributions, each element is an ndarray + representing the probability distribution of a particular prediction. + candidates_idxs : List[List[Any]] + Multiple possible candidates' indices. + + Returns + ------- + np.ndarray + Confidence distances computed for each candidate. + """ + pred_prob = np.clip(pred_prob, 1e-9, 1) + _, cols = np.indices((len(candidates_idxs), len(candidates_idxs[0]))) + return 1 - np.prod(pred_prob[cols, candidates_idxs], axis=1) + + +def to_hashable(x: Union[List[Any], Any]) -> Union[Tuple[Any, ...], Any]: + """ + Convert a nested list to a nested tuple so it is hashable. + + Parameters + ---------- + x : Union[List[Any], Any] + A potentially nested list to convert to a tuple. + + Returns + ------- + Union[Tuple[Any, ...], Any] + The input converted to a tuple if it was a list, + otherwise the original input. + """ + if isinstance(x, list): + return tuple(to_hashable(item) for item in x) + return x + + +def restore_from_hashable(x): + """ + Convert a nested tuple back to a nested list. + + Parameters + ---------- + x : Union[Tuple[Any, ...], Any] + A potentially nested tuple to convert to a list. + + Returns + ------- + Union[List[Any], Any] + The input converted to a list if it was a tuple, + otherwise the original input. + """ + if isinstance(x, tuple): + return [restore_from_hashable(item) for item in x] + return x + + +def tab_data_to_tuple( + X: Union[List[Any], Any], y: Union[List[Any], Any], reasoning_result: Optional[Any] = 0 +) -> Tuple[List[List[Any]], List[List[Any]], List[Any]]: + """ + Convert a tabular data to a tuple by adding a dimension to each element of + X and y. The tuple contains three elements: data, label, and reasoning result. + If X is None, return None. + + Parameters + ---------- + X : Union[List[Any], Any] + The data. + y : Union[List[Any], Any] + The label. + reasoning_result : Any, optional + The reasoning result, by default 0. + + Returns + ------- + Tuple[List[List[Any]], List[List[Any]], List[Any]] + A tuple of (data, label, reasoning_result). + """ + if X is None: + return None + if len(X) != len(y): + raise ValueError( + "The length of X and y should be the same, but got {} and {}.".format(len(X), len(y)) + ) + return ([[x] for x in X], [[y_item] for y_item in y], [reasoning_result] * len(y)) diff --git a/datasets/data_generator.py b/datasets/data_generator.py deleted file mode 100644 index 0d0a5c1..0000000 --- a/datasets/data_generator.py +++ /dev/null @@ -1,186 +0,0 @@ -# coding: utf-8 -#================================================================# -# Copyright (C) 2020 Freecss All rights reserved. -# -# File Name ：data_generator.py -# Author ：freecss -# Email ：karlfreecss@gmail.com -# Created Date ：2020/04/02 -# Description ： -# -#================================================================# - -from itertools import product -import math -import numpy as np -import random -import pickle as pk -import random -from multiprocessing import Pool -import copy - -#def hamming_code_generator(data_len, p_len): -# ret = [] -# for data in product((0, 1), repeat=data_len): -# p_idxs = [2 ** i for i in range(p_len)] -# total_len = data_len + p_len -# data_idx = 0 -# hamming_code = [] -# for idx in range(total_len): -# if idx + 1 in p_idxs: -# hamming_code.append(0) -# else: -# hamming_code.append(data[data_idx]) -# data_idx += 1 -# -# for idx in range(total_len): -# if idx + 1 in p_idxs: -# for i in range(total_len): -# if (i + 1) & (idx + 1) != 0: -# hamming_code[idx] ^= hamming_code[i] -# #hamming_code = "".join([str(x) for x in hamming_code]) -# ret.append(hamming_code) -# return ret - -def code_generator(code_len, code_num, letter_num = 2): - codes = list(product(list(range(letter_num)), repeat = code_len)) - random.shuffle(codes) - return codes[:code_num] - -def hamming_distance_static(codes): - min_dist = len(codes) - avg_dist = 0. - avg_min_dist = 0. - relation_num = 0. - for code1 in codes: - tmp_min_dist = len(codes) - for code2 in codes: - if code1 == code2: - continue - dist = 0 - relation_num += 1 - for c1, c2 in zip(code1, code2): - if c1 != c2: - dist += 1 - avg_dist += dist - if tmp_min_dist > dist: - tmp_min_dist = dist - avg_min_dist += tmp_min_dist - if min_dist > tmp_min_dist: - min_dist = tmp_min_dist - return avg_dist / relation_num, avg_min_dist / len(codes) - -def generate_cosin_data(codes, err, repeat, letter_num): - Y = np.random.random(100000) * letter_num * 3 - 3 - X = np.random.random(100000) * 20 - 10 - data_X = np.concatenate((X.reshape(-1, 1), Y.reshape(-1, 1)), axis = 1) - - samples = {} - all_sign = list(set(sum([[c for c in code] for code in codes], []))) - - for d, sign in enumerate(all_sign): - labels = np.logical_and(Y < np.cos(X) + 2 * d, Y > np.cos(X) + 2 * d - 2) - samples[sign] = data_X[labels] - - data = [] - labels = [] - count = 0 - for _ in range(repeat): - if (count > 100000): - break - for code in codes: - tmp = [] - count += 1 - for d in code: - if random.random() < err: - candidates = copy.deepcopy(all_sign) - candidates.remove(d) - d = candidates[random.randint(0, letter_num - 2)] - idx = random.randint(0, len(samples[d]) - 1) - tmp.append(samples[d][idx]) - data.append(tmp) - labels.append(code) - data = np.array(data) - labels = np.array(labels) - return data, labels - - -#codes = """110011001 -#100011001 -#101101101 -#011111001 -#100100001 -#111111101 -#101110001 -#111100101 -#101000101 -#001001101 -#111110101 -#100101001 -#010010101 -#110100101 -#001111101 -#111111001""" -#codes = codes.split() - -def generate_data_via_codes(codes, err, letter_num): - #codes = code_generator(code_len, code_num) - data, labels = generate_cosin_data(codes, err, 100000, letter_num) - return data, labels - -def generate_data(params): - code_len = params["code_len"] - times = params["times"] - p = params["p"] - code_num = params["code_num"] - - err = p / 20. - codes = code_generator(code_len, code_num) - data, labels = generate_cosin_data(codes, err) - data_name = "code_%d_%d" % (code_len, code_num) - pk.dump((codes, data, labels), open("generated_data/%d_%s_%.2f.pk" % (times, data_name, err), "wb")) - return True - -def generate_multi_data(): - pool = Pool(64) - params_list = [] - #for code_len in [7, 9, 11, 13, 15]: - for code_len in [7, 11, 15]: - for times in range(20): - for p in range(0, 11): - for code_num_power in range(1, code_len): - code_num = 2 ** code_num_power - params_list.append({"code_len" : code_len, "times" : times, "p" : p, "code_num" : code_num}) - return list(pool.map(generate_data, params_list)) - -def read_lexicon(file_path): - ret = [] - with open(file_path) as fin: - ret = [s.strip() for s in fin] - - all_sign = list(set(sum([[c for c in s] for s in ret], []))) - #ret = ["".join(str(all_sign.index(t)) for t in tmp) for tmp in ret] - - return ret, len(all_sign) - -import os - -if __name__ == "__main__": - for root, dirs, files in os.walk("lexicons"): - if root != "lexicons": - continue - for file_name in files: - file_path = os.path.join(root, file_name) - codes, letter_num = read_lexicon(file_path) - data, labels = generate_data_via_codes(codes, 0, letter_num) - - save_path = os.path.join("dataset", file_name.split(".")[0] + ".pk") - pk.dump((data, labels, codes), open(save_path, "wb")) - - - #res = read_lexicon("add2.txt") - #print(res) - exit(0) - - generate_multi_data() - exit() diff --git a/docs/API/abl.bridge.rst b/docs/API/abl.bridge.rst new file mode 100644 index 0000000..c967941 --- /dev/null +++ b/docs/API/abl.bridge.rst @@ -0,0 +1,7 @@ +abl.bridge +================== + +.. automodule:: abl.bridge + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/API/abl.data.rst b/docs/API/abl.data.rst new file mode 100644 index 0000000..fabff2c --- /dev/null +++ b/docs/API/abl.data.rst @@ -0,0 +1,18 @@ +abl.data +=================== + +``structures`` +-------------- + +.. autoclass:: abl.data.structures.ListData + :members: + :undoc-members: + :show-inheritance: + +``evaluation`` +-------------- + +.. automodule:: abl.data.evaluation + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/API/abl.learning.rst b/docs/API/abl.learning.rst new file mode 100644 index 0000000..7bdea1c --- /dev/null +++ b/docs/API/abl.learning.rst @@ -0,0 +1,20 @@ +abl.learning +================== + +.. autoclass:: abl.learning.ABLModel + :members: + :undoc-members: + :show-inheritance: + +.. autoclass:: abl.learning.BasicNN + :members: + :undoc-members: + :show-inheritance: + +``torch_dataset`` +----------------- + +.. automodule:: abl.learning.torch_dataset + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/API/abl.reasoning.rst b/docs/API/abl.reasoning.rst new file mode 100644 index 0000000..fefa64a --- /dev/null +++ b/docs/API/abl.reasoning.rst @@ -0,0 +1,7 @@ +abl.reasoning +================== + +.. automodule:: abl.reasoning + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/API/abl.utils.rst b/docs/API/abl.utils.rst new file mode 100644 index 0000000..0975dae --- /dev/null +++ b/docs/API/abl.utils.rst @@ -0,0 +1,7 @@ +abl.utils +================== + +.. automodule:: abl.utils + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/Examples/HED.rst b/docs/Examples/HED.rst new file mode 100644 index 0000000..829813c --- /dev/null +++ b/docs/Examples/HED.rst @@ -0,0 +1,299 @@ +Handwritten Equation Decipherment (HED) +======================================= + +.. raw:: html + +

For detailed code implementation, please view on GitHub.

+ +Below shows an implementation of `Handwritten Equation +Decipherment `__. +In this task, the handwritten equations are given, which consist of +sequential pictures of characters. The equations are generated with +unknown operation rules from images of symbols (‘0’, ‘1’, ‘+’ and ‘=’), +and each equation is associated with a label indicating whether the +equation is correct (i.e., positive) or not (i.e., negative). Also, we +are given a knowledge base which involves the structure of the equations +and a recursive definition of bit-wise operations. The task is to learn +from a training set of above mentioned equations and then to predict +labels of unseen equations. + +Intuitively, we first use a machine learning model (learning part) to +obtain the pseudo-labels (‘0’, ‘1’, ‘+’ and ‘=’) for the observed +pictures. We then use the knowledge base (reasoning part) to perform +abductive reasoning so as to yield ground hypotheses as possible +explanations to the observed facts, suggesting some pseudo-labels to be +revised. This process enables us to further update the machine learning +model. + +.. code:: ipython3 + + # Import necessary libraries and modules + import os.path as osp + + import matplotlib.pyplot as plt + import torch + import torch.nn as nn + + from abl.learning import ABLModel, BasicNN + from abl.utils import ABLLogger, print_log + + from bridge import HedBridge + from consistency_metric import ConsistencyMetric + from datasets import get_dataset, split_equation + from models.nn import SymbolNet + from reasoning import HedKB, HedReasoner + +Working with Data +----------------- + +First, we get the datasets of handwritten equations: + +.. code:: ipython3 + + total_train_data = get_dataset(train=True) + train_data, val_data = split_equation(total_train_data, 3, 1) + test_data = get_dataset(train=False) + +The dataset are shown below: + +.. code:: ipython3 + + true_train_equation = train_data[1] + false_train_equation = train_data[0] + print(f"Equations in the dataset is organized by equation length, " + + f"from {min(train_data[0].keys())} to {max(train_data[0].keys())}") + print() + + true_train_equation_with_length_5 = true_train_equation[5] + false_train_equation_with_length_5 = false_train_equation[5] + print(f"For each euqation length, there are {len(true_train_equation_with_length_5)} " + + f"true equation and {len(false_train_equation_with_length_5)} false equation " + + f"in the training set") + + true_val_equation = val_data[1] + false_val_equation = val_data[0] + true_val_equation_with_length_5 = true_val_equation[5] + false_val_equation_with_length_5 = false_val_equation[5] + print(f"For each euqation length, there are {len(true_val_equation_with_length_5)} " + + f"true equation and {len(false_val_equation_with_length_5)} false equation " + + f"in the validation set") + + true_test_equation = test_data[1] + false_test_equation = test_data[0] + true_test_equation_with_length_5 = true_test_equation[5] + false_test_equation_with_length_5 = false_test_equation[5] + print(f"For each euqation length, there are {len(true_test_equation_with_length_5)} " + + f"true equation and {len(false_test_equation_with_length_5)} false equation " + + f"in the test set") + + +Out: + .. code:: none + :class: code-out + + Equations in the dataset is organized by equation length, from 5 to 26 + + For each euqation length, there are 225 true equation and 225 false equation in the training set + For each euqation length, there are 75 true equation and 75 false equation in the validation set + For each euqation length, there are 300 true equation and 300 false equation in the test set + + +As illustrations, we show four equations in the training dataset: + +.. code:: ipython3 + + true_train_equation_with_length_5 = true_train_equation[5] + true_train_equation_with_length_8 = true_train_equation[8] + print(f"First true equation with length 5 in the training dataset:") + for i, x in enumerate(true_train_equation_with_length_5[0]): + plt.subplot(1, 5, i+1) + plt.axis('off') + plt.imshow(x.squeeze(), cmap='gray') + plt.show() + print(f"First true equation with length 8 in the training dataset:") + for i, x in enumerate(true_train_equation_with_length_8[0]): + plt.subplot(1, 8, i+1) + plt.axis('off') + plt.imshow(x.squeeze(), cmap='gray') + plt.show() + + false_train_equation_with_length_5 = false_train_equation[5] + false_train_equation_with_length_8 = false_train_equation[8] + print(f"First false equation with length 5 in the training dataset:") + for i, x in enumerate(false_train_equation_with_length_5[0]): + plt.subplot(1, 5, i+1) + plt.axis('off') + plt.imshow(x.squeeze(), cmap='gray') + plt.show() + print(f"First false equation with length 8 in the training dataset:") + for i, x in enumerate(false_train_equation_with_length_8[0]): + plt.subplot(1, 8, i+1) + plt.axis('off') + plt.imshow(x.squeeze(), cmap='gray') + plt.show() + + +Out: + .. code:: none + :class: code-out + + First true equation with length 5 in the training dataset: + + .. image:: ../_static/img/hed_dataset1.png + :width: 300px + + .. code:: none + :class: code-out + + First true equation with length 8 in the training dataset: + + .. image:: ../_static/img/hed_dataset2.png + :width: 480px + + .. code:: none + :class: code-out + + First false equation with length 5 in the training dataset: + + .. image:: ../_static/img/hed_dataset3.png + :width: 300px + + .. code:: none + :class: code-out + + First false equation with length 8 in the training dataset: + + .. image:: ../_static/img/hed_dataset4.png + :width: 480px + + +Building the Learning Part +-------------------------- + +To build the learning part, we need to first build a machine learning +base model. We use SymbolNet, and encapsulate it within a ``BasicNN`` +object to create the base model. ``BasicNN`` is a class that +encapsulates a PyTorch model, transforming it into a base model with an +sklearn-style interface. + +.. code:: ipython3 + + # class of symbol may be one of ['0', '1', '+', '='], total of 4 classes + cls = SymbolNet(num_classes=4) + loss_fn = nn.CrossEntropyLoss() + optimizer = torch.optim.RMSprop(cls.parameters(), lr=0.001, weight_decay=1e-4) + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + base_model = BasicNN( + cls, + loss_fn, + optimizer, + device=device, + batch_size=32, + num_epochs=1, + stop_loss=None, + ) + +However, the base model built above deals with instance-level data +(i.e., individual images), and can not directly deal with example-level +data (i.e., a list of images comprising the equation). Therefore, we +wrap the base model into ``ABLModel``, which enables the learning part +to train, test, and predict on example-level data. + +.. code:: ipython3 + + model = ABLModel(base_model) + +Building the Reasoning Part +--------------------------- + +In the reasoning part, we first build a knowledge base. As mentioned +before, the knowledge base in this task involves the structure of the +equations and a recursive definition of bit-wise operations, which are +defined in Prolog file ``examples/hed/reasoning/BK.pl`` +and ``examples/hed/reasoning/learn_add.pl``, respectively. +Specifically, the knowledge about the structure of equations is a set of DCG +rules recursively define that a digit is a sequence of ‘0’ and ‘1’, and +equations share the structure of X+Y=Z, though the length of X, Y and Z +can be varied. The knowledge about bit-wise operations is a recursive +logic program, which reversely calculates X+Y, i.e., it operates on +X and Y digit-by-digit and from the last digit to the first. + +The knowledge base is already built in ``HedKB``. +``HedKB`` is derived from class ``PrologKB``, and is built upon the aformentioned Prolog +files. + +.. code:: ipython3 + + kb = HedKB() + +.. note:: + + Please notice that, the specific rules for calculating the + operations are undefined in the knowledge base, i.e., results of ‘0+0’, + ‘0+1’ and ‘1+1’ could be ‘0’, ‘1’, ‘00’, ‘01’ or even ‘10’. The missing + calculation rules are required to be learned from the data. Therefore, + ``HedKB`` incorporates methods for abducing rules from data. Users + interested can refer to the specific implementation of ``HedKB`` in + ``examples/hed/reasoning/reasoning.py`` + +Then, we create a reasoner. Due to the indeterminism of abductive +reasoning, there could be multiple candidates compatible to the +knowledge base. When this happens, reasoner can minimize inconsistencies +between the knowledge base and pseudo-labels predicted by the learning +part, and then return only one candidate that has the highest +consistency. + +In this task, we create the reasoner by instantiating the class +``HedReasoner``, which is a reasoner derived from ``Reasoner`` and +tailored specifically for this task. ``HedReasoner`` leverages `ZOOpt +library `__ for acceleration, and has +designed a specific strategy to better harness ZOOpt’s capabilities. +Additionally, methods for abducing rules from data have been +incorporated. Users interested can refer to the specific implementation +of ``HedReasoner`` in ``reasoning/reasoning.py``. + +.. code:: ipython3 + + reasoner = HedReasoner(kb, dist_func="hamming", use_zoopt=True, max_revision=10) + +Building Evaluation Metrics +--------------------------- + +Next, we set up evaluation metrics. These metrics will be used to +evaluate the model performance during training and testing. +Specifically, we use ``SymbolAccuracy`` and ``ReasoningMetric``, which are +used to evaluate the accuracy of the machine learning model’s +predictions and the accuracy of the final reasoning results, +respectively. + +.. code:: ipython3 + + # Set up metrics + metric_list = [SymbolAccuracy(prefix="hed"), ReasoningMetric(kb=kb, prefix="hed")] + +Bridge Learning and Reasoning +----------------------------- + +Now, the last step is to bridge the learning and reasoning part. We +proceed this step by creating an instance of ``HedBridge``, which is +derived from ``SimpleBridge`` and tailored specific for this task. + +.. code:: ipython3 + + bridge = HedBridge(model, reasoner, metric_list) + +Perform pretraining, training and testing by invoking the ``pretrain``, ``train`` and ``test`` methods of ``HedBridge``. + +.. code:: ipython3 + + # Build logger + print_log("Abductive Learning on the HED example.", logger="current") + + # Retrieve the directory of the Log file and define the directory for saving the model weights. + log_dir = ABLLogger.get_current_instance().log_dir + weights_dir = osp.join(log_dir, "weights") + + bridge.pretrain("./weights") + bridge.train(train_data, val_data, save_dir=weights_dir) + bridge.test(test_data) diff --git a/docs/Examples/HWF.rst b/docs/Examples/HWF.rst new file mode 100644 index 0000000..bb26ff9 --- /dev/null +++ b/docs/Examples/HWF.rst @@ -0,0 +1,472 @@ +Handwritten Formula (HWF) +========================= + +.. raw:: html + +

For detailed code implementation, please view on GitHub.

+ +Below shows an implementation of `Handwritten +Formula `__. In this +task, handwritten images of decimal formulas and their computed results +are given, alongwith a domain knowledge base containing information on +how to compute the decimal formula. The task is to recognize the symbols +(which can be digits or operators ‘+’, ‘-’, ‘×’, ‘÷’) of handwritten +images and accurately determine their results. + +Intuitively, we first use a machine learning model (learning part) to +convert the input images to symbols (we call them pseudo-labels), and +then use the knowledge base (reasoning part) to calculate the results of +these symbols. Since we do not have ground-truth of the symbols, in +Abductive Learning, the reasoning part will leverage domain knowledge +and revise the initial symbols yielded by the learning part through +abductive reasoning. This process enables us to further update the +machine learning model. + +.. code:: ipython3 + + # Import necessary libraries and modules + import os.path as osp + + import matplotlib.pyplot as plt + import numpy as np + import torch + import torch.nn as nn + + from abl.bridge import SimpleBridge + from abl.data.evaluation import ReasoningMetric, SymbolAccuracy + from abl.learning import ABLModel, BasicNN + from abl.reasoning import KBBase, Reasoner + from abl.utils import ABLLogger, print_log + + from datasets import get_dataset + from models.nn import SymbolNet + +Working with Data +----------------- + +First, we get the training and testing datasets: + +.. code:: ipython3 + + train_data = get_dataset(train=True, get_pseudo_label=True) + test_data = get_dataset(train=False, get_pseudo_label=True) + +Both ``train_data`` and ``test_data`` have the same structures: tuples +with three components: X (list where each element is a list of images), +gt_pseudo_label (list where each element is a list of symbols, i.e., +pseudo-labels) and Y (list where each element is the computed result). +The length and structures of datasets are illustrated as follows. + +.. note:: + + ``gt_pseudo_label`` is only used to evaluate the performance of + the learning part but not to train the model. + +.. code:: ipython3 + + print(f"Both train_data and test_data consist of 3 components: X, gt_pseudo_label, Y") + print() + train_X, train_gt_pseudo_label, train_Y = train_data + print(f"Length of X, gt_pseudo_label, Y in train_data: " + + f"{len(train_X)}, {len(train_gt_pseudo_label)}, {len(train_Y)}") + test_X, test_gt_pseudo_label, test_Y = test_data + print(f"Length of X, gt_pseudo_label, Y in test_data: " + + f"{len(test_X)}, {len(test_gt_pseudo_label)}, {len(test_Y)}") + print() + + X_0, gt_pseudo_label_0, Y_0 = train_X[0], train_gt_pseudo_label[0], train_Y[0] + print(f"X is a {type(train_X).__name__}, " + + f"with each element being a {type(X_0).__name__} of {type(X_0[0]).__name__}.") + print(f"gt_pseudo_label is a {type(train_gt_pseudo_label).__name__}, " + + f"with each element being a {type(gt_pseudo_label_0).__name__} " + + f"of {type(gt_pseudo_label_0[0]).__name__}.") + print(f"Y is a {type(train_Y).__name__}, " + + f"with each element being a {type(Y_0).__name__}.") + + +Out: + .. code:: none + :class: code-out + + Both train_data and test_data consist of 3 components: X, gt_pseudo_label, Y + + Length of X, gt_pseudo_label, Y in train_data: 10000, 10000, 10000 + Length of X, gt_pseudo_label, Y in test_data: 2000, 2000, 2000 + + X is a list, with each element being a list of Tensor. + gt_pseudo_label is a list, with each element being a list of str. + Y is a list, with each element being a int. + + +The ith element of X, gt_pseudo_label, and Y together constitute the ith +data example. Here we use two of them (the 1001st and the 3001st) as +illstrations: + +.. code:: ipython3 + + X_1000, gt_pseudo_label_1000, Y_1000 = train_X[1000], train_gt_pseudo_label[1000], train_Y[1000] + print(f"X in the 1001st data example (a list of images):") + for i, x in enumerate(X_1000): + plt.subplot(1, len(X_1000), i+1) + plt.axis('off') + plt.imshow(x.squeeze(), cmap='gray') + plt.show() + print(f"gt_pseudo_label in the 1001st data example (a list of ground truth pseudo-labels): {gt_pseudo_label_1000}") + print(f"Y in the 1001st data example (the computed result): {Y_1000}") + print() + X_3000, gt_pseudo_label_3000, Y_3000 = train_X[3000], train_gt_pseudo_label[3000], train_Y[3000] + print(f"X in the 3001st data example (a list of images):") + for i, x in enumerate(X_3000): + plt.subplot(1, len(X_3000), i+1) + plt.axis('off') + plt.imshow(x.squeeze(), cmap='gray') + plt.show() + print(f"gt_pseudo_label in the 3001st data example (a list of ground truth pseudo-labels): {gt_pseudo_label_3000}") + print(f"Y in the 3001st data example (the computed result): {Y_3000}") + + +Out: + .. code:: none + :class: code-out + + X in the 1001st data example (a list of images): + + .. image:: ../_static/img/hwf_dataset1.png + :width: 210px + + .. code:: none + :class: code-out + + gt_pseudo_label in the 1001st data example (a list of pseudo-labels): ['5', '-', '3'] + Y in the 1001st data example (the computed result): 2 + + .. code:: none + :class: code-out + + X in the 3001st data example (a list of images): + + .. image:: ../_static/img/hwf_dataset2.png + :width: 350px + + .. code:: none + :class: code-out + + gt_pseudo_label in the 3001st data example (a list of pseudo-labels): ['4', '/', '6', '*', '5'] + Y in the 3001st data example (the computed result): 3.333333333333333 + + +.. note:: + + The symbols in the HWF dataset can be one of digits or operators + '+', '-', '×', '÷'. + + We may see that, in the 1001st data example, the length of the + formula is 3, while in the 3001st data example, the length of the + formula is 5. In the HWF dataset, the length of the formula varies from + 1 to 7. + +Building the Learning Part +-------------------------- + +To build the learning part, we need to first build a machine learning +base model. We use SymbolNet, and encapsulate it within a ``BasicNN`` +object to create the base model. ``BasicNN`` is a class that +encapsulates a PyTorch model, transforming it into a base model with an +sklearn-style interface. + +.. code:: ipython3 + + # class of symbol may be one of ['1', ..., '9', '+', '-', '*', '/'], total of 14 classes + cls = SymbolNet(num_classes=13, image_size=(45, 45, 1)) + loss_fn = nn.CrossEntropyLoss() + optimizer = torch.optim.Adam(cls.parameters(), lr=0.001, betas=(0.9, 0.99)) + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + + base_model = BasicNN( + model=cls, + loss_fn=loss_fn, + optimizer=optimizer, + device=device, + batch_size=128, + num_epochs=3, + ) + +``BasicNN`` offers methods like ``predict`` and ``predict_prob``, which +are used to predict the class index and the probabilities of each class +for images. As shown below: + +.. code:: ipython3 + + data_instances = [torch.randn(1, 45, 45).to(device) for _ in range(32)] + pred_idx = base_model.predict(X=data_instances) + print(f"Predicted class index for a batch of 32 instances: " + + f"{type(pred_idx).__name__} with shape {pred_idx.shape}") + pred_prob = base_model.predict_proba(X=data_instances) + print(f"Predicted class probabilities for a batch of 32 instances: " + + f"{type(pred_prob).__name__} with shape {pred_prob.shape}") + + +Out: + .. code:: none + :class: code-out + + Predicted class index for a batch of 32 instances: ndarray with shape (32,) + Predicted class probabilities for a batch of 32 instances: ndarray with shape (32, 14) + + +However, the base model built above deals with instance-level data +(i.e., individual images), and can not directly deal with example-level +data (i.e., a list of images comprising the formula). Therefore, we wrap +the base model into ``ABLModel``, which enables the learning part to +train, test, and predict on example-level data. + +.. code:: ipython3 + + model = ABLModel(base_model) + +As an illustration, consider this example of training on example-level +data using the ``predict`` method in ``ABLModel``. In this process, the +method accepts data examples as input and outputs the class labels and +the probabilities of each class for all instances within these data +examples. + +.. code:: ipython3 + + from abl.data.structures import ListData + # ListData is a data structure provided by ABL-Package that can be used to organize data examples + data_examples = ListData() + # We use the first 1001st and 3001st data examples in the training set as an illustration + data_examples.X = [X_1000, X_3000] + data_examples.gt_pseudo_label = [gt_pseudo_label_1000, gt_pseudo_label_3000] + data_examples.Y = [Y_1000, Y_3000] + + # Perform prediction on the two data examples + # Remind that, in the 1001st data example, the length of the formula is 3, + # while in the 3001st data example, the length of the formula is 5. + pred_label, pred_prob = model.predict(data_examples)['label'], model.predict(data_examples)['prob'] + print(f"Predicted class labels for the 100 data examples: a list of length {len(pred_label)}, \n" + + f"the first element is a {type(pred_label[0]).__name__} of shape {pred_label[0].shape}, "+ + f"and the second element is a {type(pred_label[1]).__name__} of shape {pred_label[1].shape}.\n") + print(f"Predicted class probabilities for the 100 data examples: a list of length {len(pred_prob)}, \n" + f"the first element is a {type(pred_prob[0]).__name__} of shape {pred_prob[0].shape}, " + + f"and the second element is a {type(pred_prob[1]).__name__} of shape {pred_prob[1].shape}.") + + +Out: + .. code:: none + :class: code-out + + Predicted class labels for the 100 data examples: a list of length 2, + the first element is a ndarray of shape (3,), and the second element is a ndarray of shape (5,). + + Predicted class probabilities for the 100 data examples: a list of length 2, + the first element is a ndarray of shape (3, 14), and the second element is a ndarray of shape (5, 14). + + +Building the Reasoning Part +--------------------------- + +In the reasoning part, we first build a knowledge base which contain +information on how to perform addition operations. We build it by +creating a subclass of ``KBBase``. In the derived subclass, we +initialize the ``pseudo_label_list`` parameter specifying list of +possible pseudo-labels, and override the ``logic_forward`` function +defining how to perform (deductive) reasoning. + +.. code:: ipython3 + + class HwfKB(KBBase): + def __init__(self, pseudo_label_list=["1", "2", "3", "4", "5", "6", "7", "8", "9", "+", "-", "*", "/"]): + super().__init__(pseudo_label_list) + + def _valid_candidate(self, formula): + if len(formula) % 2 == 0: + return False + for i in range(len(formula)): + if i % 2 == 0 and formula[i] not in ["1", "2", "3", "4", "5", "6", "7", "8", "9"]: + return False + if i % 2 != 0 and formula[i] not in ["+", "-", "*", "/"]: + return False + return True + + # Implement the deduction function + def logic_forward(self, formula): + if not self._valid_candidate(formula): + return np.inf + return eval("".join(formula)) + + kb = HwfKB() + +The knowledge base can perform logical reasoning (both deductive +reasoning and abductive reasoning). Below is an example of performing +(deductive) reasoning, and users can refer to :ref:`Performing abductive +reasoning in the knowledge base ` for details of abductive reasoning. + +.. code:: ipython3 + + pseudo_labels = ["1", "-", "2", "*", "5"] + reasoning_result = kb.logic_forward(pseudo_labels) + print(f"Reasoning result of pseudo-labels {pseudo_labels} is {reasoning_result}.") + + +Out: + .. code:: none + :class: code-out + + Reasoning result of pseudo-labels ['1', '-', '2', '*', '5'] is -9. + + +.. note:: + + In addition to building a knowledge base based on ``KBBase``, we + can also establish a knowledge base with a ground KB using ``GroundKB``. + The corresponding code can be found in the ``examples/hwf/main.py`` file. Those + interested are encouraged to examine it for further insights. + + Also, when building the knowledge base, we can also set the + ``max_err`` parameter during initialization, which is shown in the + ``examples/hwf/main.py`` file. This parameter specifies the upper tolerance limit + when comparing the similarity between the reasoning result of pseudo-labels and + the ground truth during abductive reasoning, with a default + value of 1e-10. + +Then, we create a reasoner by instantiating the class ``Reasoner``. Due +to the indeterminism of abductive reasoning, there could be multiple +candidates compatible to the knowledge base. When this happens, reasoner +can minimize inconsistencies between the knowledge base and +pseudo-labels predicted by the learning part, and then return only one +candidate that has the highest consistency. + +.. code:: ipython3 + + reasoner = Reasoner(kb) + +.. note:: + + During creating reasoner, the definition of “consistency” can be + customized within the ``dist_func`` parameter. In the code above, we + employ a consistency measurement based on confidence, which calculates + the consistency between the data example and candidates based on the + confidence derived from the predicted probability. In ``examples/hwf/main.py``, we + provide options for utilizing other forms of consistency measurement. + + Also, during process of inconsistency minimization, we can + leverage `ZOOpt library `__ for + acceleration. Options for this are also available in ``examples/hwf/main.py``. Those + interested are encouraged to explore these features. + +Building Evaluation Metrics +--------------------------- + +Next, we set up evaluation metrics. These metrics will be used to +evaluate the model performance during training and testing. +Specifically, we use ``SymbolAccuracy`` and ``ReasoningMetric``, which are +used to evaluate the accuracy of the machine learning model’s +predictions and the accuracy of the final reasoning results, +respectively. + +.. code:: ipython3 + + metric_list = [SymbolAccuracy(prefix="hwf"), ReasoningMetric(kb=kb, prefix="hwf")] + +Bridge Learning and Reasoning +----------------------------- + +Now, the last step is to bridge the learning and reasoning part. We +proceed this step by creating an instance of ``SimpleBridge``. + +.. code:: ipython3 + + bridge = SimpleBridge(model, reasoner, metric_list) + +Perform training and testing by invoking the ``train`` and ``test`` +methods of ``SimpleBridge``. + +.. code:: ipython3 + + # Build logger + print_log("Abductive Learning on the HWF example.", logger="current") + log_dir = ABLLogger.get_current_instance().log_dir + weights_dir = osp.join(log_dir, "weights") + + bridge.train(train_data, train_data, loops=3, segment_size=1000, save_dir=weights_dir) + bridge.test(test_data) + +Out: + .. code:: none + :class: code-out + + abl - INFO - Abductive Learning on the HWF example. + abl - INFO - loop(train) [1/3] segment(train) [1/10] + abl - INFO - model loss: 0.00024 + abl - INFO - loop(train) [1/3] segment(train) [2/10] + abl - INFO - model loss: 0.00053 + abl - INFO - loop(train) [1/3] segment(train) [3/10] + abl - INFO - model loss: 0.00260 + abl - INFO - loop(train) [1/3] segment(train) [4/10] + abl - INFO - model loss: 0.00162 + abl - INFO - loop(train) [1/3] segment(train) [5/10] + abl - INFO - model loss: 0.00073 + abl - INFO - loop(train) [1/3] segment(train) [6/10] + abl - INFO - model loss: 0.00055 + abl - INFO - loop(train) [1/3] segment(train) [7/10] + abl - INFO - model loss: 0.00148 + abl - INFO - loop(train) [1/3] segment(train) [8/10] + abl - INFO - model loss: 0.00034 + abl - INFO - loop(train) [1/3] segment(train) [9/10] + abl - INFO - model loss: 0.00167 + abl - INFO - loop(train) [1/3] segment(train) [10/10] + abl - INFO - model loss: 0.00185 + abl - INFO - Evaluation start: loop(val) [1] + abl - INFO - Evaluation ended, hwf/character_accuracy: 1.000 hwf/reasoning_accuracy: 0.999 + abl - INFO - Saving model: loop(save) [1] + abl - INFO - Checkpoints will be saved to weights_dir/model_checkpoint_loop_1.pth + abl - INFO - loop(train) [2/3] segment(train) [1/10] + abl - INFO - model loss: 0.00219 + abl - INFO - loop(train) [2/3] segment(train) [2/10] + abl - INFO - model loss: 0.00069 + abl - INFO - loop(train) [2/3] segment(train) [3/10] + abl - INFO - model loss: 0.00013 + abl - INFO - loop(train) [2/3] segment(train) [4/10] + abl - INFO - model loss: 0.00013 + abl - INFO - loop(train) [2/3] segment(train) [5/10] + abl - INFO - model loss: 0.00248 + abl - INFO - loop(train) [2/3] segment(train) [6/10] + abl - INFO - model loss: 0.00010 + abl - INFO - loop(train) [2/3] segment(train) [7/10] + abl - INFO - model loss: 0.00020 + abl - INFO - loop(train) [2/3] segment(train) [8/10] + abl - INFO - model loss: 0.00076 + abl - INFO - loop(train) [2/3] segment(train) [9/10] + abl - INFO - model loss: 0.00061 + abl - INFO - loop(train) [2/3] segment(train) [10/10] + abl - INFO - model loss: 0.00117 + abl - INFO - Evaluation start: loop(val) [2] + abl - INFO - Evaluation ended, hwf/character_accuracy: 1.000 hwf/reasoning_accuracy: 1.000 + abl - INFO - Saving model: loop(save) [2] + abl - INFO - Checkpoints will be saved to weights_dir/model_checkpoint_loop_2.pth + abl - INFO - loop(train) [3/3] segment(train) [1/10] + abl - INFO - model loss: 0.00120 + abl - INFO - loop(train) [3/3] segment(train) [2/10] + abl - INFO - model loss: 0.00114 + abl - INFO - loop(train) [3/3] segment(train) [3/10] + abl - INFO - model loss: 0.00071 + abl - INFO - loop(train) [3/3] segment(train) [4/10] + abl - INFO - model loss: 0.00027 + abl - INFO - loop(train) [3/3] segment(train) [5/10] + abl - INFO - model loss: 0.00017 + abl - INFO - loop(train) [3/3] segment(train) [6/10] + abl - INFO - model loss: 0.00018 + abl - INFO - loop(train) [3/3] segment(train) [7/10] + abl - INFO - model loss: 0.00141 + abl - INFO - loop(train) [3/3] segment(train) [8/10] + abl - INFO - model loss: 0.00099 + abl - INFO - loop(train) [3/3] segment(train) [9/10] + abl - INFO - model loss: 0.00145 + abl - INFO - loop(train) [3/3] segment(train) [10/10] + abl - INFO - model loss: 0.00215 + abl - INFO - Evaluation start: loop(val) [3] + abl - INFO - Evaluation ended, hwf/character_accuracy: 1.000 hwf/reasoning_accuracy: 1.000 + abl - INFO - Saving model: loop(save) [3] + abl - INFO - Checkpoints will be saved to weights_dir/model_checkpoint_loop_2.pth + abl - INFO - Evaluation ended, hwf/character_accuracy: 0.996 hwf/reasoning_accuracy: 0.977 diff --git a/docs/Examples/MNISTAdd.rst b/docs/Examples/MNISTAdd.rst new file mode 100644 index 0000000..c05fd85 --- /dev/null +++ b/docs/Examples/MNISTAdd.rst @@ -0,0 +1,381 @@ +MNIST Addition +============== + +.. raw:: html + +

For detailed code implementation, please view on GitHub.

+ +Below shows an implementation of `MNIST +Addition `__. In this task, pairs of +MNIST handwritten images and their sums are given, alongwith a domain +knowledge base containing information on how to perform addition +operations. The task is to recognize the digits of handwritten images +and accurately determine their sum. + +Intuitively, we first use a machine learning model (learning part) to +convert the input images to digits (we call them pseudo-labels), and +then use the knowledge base (reasoning part) to calculate the sum of +these digits. Since we do not have ground-truth of the digits, in +Abductive Learning, the reasoning part will leverage domain knowledge +and revise the initial digits yielded by the learning part through +abductive reasoning. This process enables us to further update the +machine learning model. + +.. code:: ipython3 + + # Import necessary libraries and modules + import os.path as osp + + import matplotlib.pyplot as plt + import torch + import torch.nn as nn + from torch.optim import RMSprop, lr_scheduler + + from abl.bridge import SimpleBridge + from abl.data.evaluation import ReasoningMetric, SymbolAccuracy + from abl.learning import ABLModel, BasicNN + from abl.reasoning import KBBase, Reasoner + from abl.utils import ABLLogger, print_log + + from datasets import get_dataset + from models.nn import LeNet5 + +Working with Data +----------------- + +First, we get the training and testing datasets: + +.. code:: ipython3 + + train_data = get_dataset(train=True, get_pseudo_label=True) + test_data = get_dataset(train=False, get_pseudo_label=True) + +``train_data`` and ``test_data`` share identical structures: +tuples with three components: X (list where each element is a +list of two images), gt_pseudo_label (list where each element +is a list of two digits, i.e., pseudo-labels) and Y (list where +each element is the sum of the two digits). The length and structures +of datasets are illustrated as follows. + +.. note:: + + ``gt_pseudo_label`` is only used to evaluate the performance of + the learning part but not to train the model. + +.. code:: ipython3 + + print(f"Both train_data and test_data consist of 3 components: X, gt_pseudo_label, Y") + print("\n") + train_X, train_gt_pseudo_label, train_Y = train_data + print(f"Length of X, gt_pseudo_label, Y in train_data: " + + f"{len(train_X)}, {len(train_gt_pseudo_label)}, {len(train_Y)}") + test_X, test_gt_pseudo_label, test_Y = test_data + print(f"Length of X, gt_pseudo_label, Y in test_data: " + + f"{len(test_X)}, {len(test_gt_pseudo_label)}, {len(test_Y)}") + print("\n") + + X_0, gt_pseudo_label_0, Y_0 = train_X[0], train_gt_pseudo_label[0], train_Y[0] + print(f"X is a {type(train_X).__name__}, " + + f"with each element being a {type(X_0).__name__} " + + f"of {len(X_0)} {type(X_0[0]).__name__}.") + print(f"gt_pseudo_label is a {type(train_gt_pseudo_label).__name__}, " + + f"with each element being a {type(gt_pseudo_label_0).__name__} " + + f"of {len(gt_pseudo_label_0)} {type(gt_pseudo_label_0[0]).__name__}.") + print(f"Y is a {type(train_Y).__name__}, " + + f"with each element being a {type(Y_0).__name__}.") + + +Out: + .. code:: none + :class: code-out + + Both train_data and test_data consist of 3 components: X, gt_pseudo_label, Y + + Length of X, gt_pseudo_label, Y in train_data: 30000, 30000, 30000 + Length of X, gt_pseudo_label, Y in test_data: 5000, 5000, 5000 + + X is a list, with each element being a list of 2 Tensor. + gt_pseudo_label is a list, with each element being a list of 2 int. + Y is a list, with each element being a int. + + +The ith element of X, gt_pseudo_label, and Y together constitute the ith +data example. As an illustration, in the first data example of the +training set, we have: + +.. code:: ipython3 + + X_0, gt_pseudo_label_0, Y_0 = train_X[0], train_gt_pseudo_label[0], train_Y[0] + print(f"X in the first data example (a list of two images):") + plt.subplot(1,2,1) + plt.axis('off') + plt.imshow(X_0[0].squeeze(), cmap='gray') + plt.subplot(1,2,2) + plt.axis('off') + plt.imshow(X_0[1].squeeze(), cmap='gray') + plt.show() + print(f"gt_pseudo_label in the first data example (a list of two ground truth pseudo-labels): {gt_pseudo_label_0}") + print(f"Y in the first data example (their sum result): {Y_0}") + + +Out: + .. code:: none + :class: code-out + + X in the first data example (a list of two images): + + .. image:: ../_static/img/mnist_add_datasets.png + :width: 200px + + + .. code:: none + :class: code-out + + gt_pseudo_label in the first data example (a list of two ground truth pseudo-labels): [7, 5] + Y in the first data example (their sum result): 12 + + +Building the Learning Part +-------------------------- + +To build the learning part, we need to first build a machine learning +base model. We use a simple `LeNet-5 neural +network `__, and encapsulate it +within a ``BasicNN`` object to create the base model. ``BasicNN`` is a +class that encapsulates a PyTorch model, transforming it into a base +model with an sklearn-style interface. + +.. code:: ipython3 + + cls = LeNet5(num_classes=10) + loss_fn = nn.CrossEntropyLoss(label_smoothing=0.1) + optimizer = RMSprop(cls.parameters(), lr=0.001, alpha=0.9) + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + scheduler = lr_scheduler.OneCycleLR(optimizer, max_lr=0.001, pct_start=0.1, total_steps=100) + + base_model = BasicNN( + cls, + loss_fn, + optimizer, + scheduler=scheduler, + device=device, + batch_size=32, + num_epochs=1, + ) + +``BasicNN`` offers methods like ``predict`` and ``predict_prob``, which +are used to predict the class index and the probabilities of each class +for images. As shown below: + +.. code:: ipython3 + + data_instances = [torch.randn(1, 28, 28).to(device) for _ in range(32)] + pred_idx = base_model.predict(X=data_instances) + print(f"Predicted class index for a batch of 32 instances: np.ndarray with shape {pred_idx.shape}") + pred_prob = base_model.predict_proba(X=data_instances) + print(f"Predicted class probabilities for a batch of 32 instances: np.ndarray with shape {pred_prob.shape}") + + +Out: + .. code:: none + :class: code-out + + Predicted class index for a batch of 32 instances: np.ndarray with shape (32,) + Predicted class probabilities for a batch of 32 instances: np.ndarray with shape (32, 10) + + +However, the base model built above deals with instance-level data +(i.e., individual images), and can not directly deal with example-level +data (i.e., a pair of images). Therefore, we wrap the base model into +``ABLModel``, which enables the learning part to train, test, and +predict on example-level data. + +.. code:: ipython3 + + model = ABLModel(base_model) + +As an illustration, consider this example of training on example-level +data using the ``predict`` method in ``ABLModel``. In this process, the +method accepts data examples as input and outputs the class labels and +the probabilities of each class for all instances within these data +examples. + +.. code:: ipython3 + + from abl.data.structures import ListData + # ListData is a data structure provided by ABL-Package that can be used to organize data examples + data_examples = ListData() + # We use the first 100 data examples in the training set as an illustration + data_examples.X = train_X[:100] + data_examples.gt_pseudo_label = train_gt_pseudo_label[:100] + data_examples.Y = train_Y[:100] + + # Perform prediction on the 100 data examples + pred_label, pred_prob = model.predict(data_examples)['label'], model.predict(data_examples)['prob'] + print(f"Predicted class labels for the 100 data examples: \n" + + f"a list of length {len(pred_label)}, and each element is " + + f"a {type(pred_label[0]).__name__} of shape {pred_label[0].shape}.\n") + print(f"Predicted class probabilities for the 100 data examples: \n" + + f"a list of length {len(pred_prob)}, and each element is " + + f"a {type(pred_prob[0]).__name__} of shape {pred_prob[0].shape}.") + + +Out: + .. code:: none + :class: code-out + + Predicted class labels for the 100 data examples: + a list of length 100, and each element is a ndarray of shape (2,). + + Predicted class probabilities for the 100 data examples: + a list of length 100, and each element is a ndarray of shape (2, 10). + + +Building the Reasoning Part +--------------------------- + +In the reasoning part, we first build a knowledge base which contain +information on how to perform addition operations. We build it by +creating a subclass of ``KBBase``. In the derived subclass, we +initialize the ``pseudo_label_list`` parameter specifying list of +possible pseudo-labels, and override the ``logic_forward`` function +defining how to perform (deductive) reasoning. + +.. code:: ipython3 + + class AddKB(KBBase): + def __init__(self, pseudo_label_list=list(range(10))): + super().__init__(pseudo_label_list) + + # Implement the deduction function + def logic_forward(self, nums): + return sum(nums) + + kb = AddKB() + +The knowledge base can perform logical reasoning (both deductive +reasoning and abductive reasoning). Below is an example of performing +(deductive) reasoning, and users can refer to :ref:`Performing abductive +reasoning in the knowledge base ` for details of abductive reasoning. + +.. code:: ipython3 + + pseudo_labels = [1, 2] + reasoning_result = kb.logic_forward(pseudo_labels) + print(f"Reasoning result of pseudo-labels {pseudo_labels} is {reasoning_result}.") + + +Out: + .. code:: none + :class: code-out + + Reasoning result of pseudo-labels [1, 2] is 3. + + +.. note:: + + In addition to building a knowledge base based on ``KBBase``, we + can also establish a knowledge base with a ground KB using ``GroundKB``, + or a knowledge base implemented based on Prolog files using + ``PrologKB``. The corresponding code for these implementations can be + found in the ``main.py`` file. Those interested are encouraged to + examine it for further insights. + +Then, we create a reasoner by instantiating the class ``Reasoner``. Due +to the indeterminism of abductive reasoning, there could be multiple +candidates compatible to the knowledge base. When this happens, reasoner +can minimize inconsistencies between the knowledge base and +pseudo-labels predicted by the learning part, and then return only one +candidate that has the highest consistency. + +.. code:: ipython3 + + reasoner = Reasoner(kb) + +.. note:: + + During creating reasoner, the definition of “consistency” can be + customized within the ``dist_func`` parameter. In the code above, we + employ a consistency measurement based on confidence, which calculates + the consistency between the data example and candidates based on the + confidence derived from the predicted probability. In ``examples/mnist_add/main.py``, we + provide options for utilizing other forms of consistency measurement. + + Also, during process of inconsistency minimization, we can leverage + `ZOOpt library `__ for acceleration. + Options for this are also available in ``examples/mnist_add/main.py``. Those interested are + encouraged to explore these features. + +Building Evaluation Metrics +--------------------------- + +Next, we set up evaluation metrics. These metrics will be used to +evaluate the model performance during training and testing. +Specifically, we use ``SymbolAccuracy`` and ``ReasoningMetric``, which are +used to evaluate the accuracy of the machine learning model’s +predictions and the accuracy of the final reasoning results, +respectively. + +.. code:: ipython3 + + metric_list = [SymbolAccuracy(prefix="mnist_add"), ReasoningMetric(kb=kb, prefix="mnist_add")] + +Bridge Learning and Reasoning +----------------------------- + +Now, the last step is to bridge the learning and reasoning part. We +proceed this step by creating an instance of ``SimpleBridge``. + +.. code:: ipython3 + + bridge = SimpleBridge(model, reasoner, metric_list) + +Perform training and testing by invoking the ``train`` and ``test`` +methods of ``SimpleBridge``. + +.. code:: ipython3 + + # Build logger + print_log("Abductive Learning on the MNIST Addition example.", logger="current") + log_dir = ABLLogger.get_current_instance().log_dir + weights_dir = osp.join(log_dir, "weights") + + bridge.train(train_data, loops=1, segment_size=0.01, save_interval=1, save_dir=weights_dir) + bridge.test(test_data) + +Out: + .. code:: none + :class: code-out + + abl - INFO - Abductive Learning on the MNIST Addition example. + abl - INFO - loop(train) [1/1] segment(train) [1/100] + abl - INFO - model loss: 2.23587 + abl - INFO - loop(train) [1/1] segment(train) [2/100] + abl - INFO - model loss: 2.23756 + abl - INFO - loop(train) [1/1] segment(train) [3/100] + abl - INFO - model loss: 2.04475 + abl - INFO - loop(train) [1/1] segment(train) [4/100] + abl - INFO - model loss: 2.01035 + abl - INFO - loop(train) [1/1] segment(train) [5/100] + abl - INFO - model loss: 1.97584 + abl - INFO - loop(train) [1/1] segment(train) [6/100] + abl - INFO - model loss: 1.91570 + abl - INFO - loop(train) [1/1] segment(train) [7/100] + abl - INFO - model loss: 1.90268 + abl - INFO - loop(train) [1/1] segment(train) [8/100] + abl - INFO - model loss: 1.77436 + abl - INFO - loop(train) [1/1] segment(train) [9/100] + abl - INFO - model loss: 1.73454 + abl - INFO - loop(train) [1/1] segment(train) [10/100] + abl - INFO - model loss: 1.62495 + abl - INFO - loop(train) [1/1] segment(train) [11/100] + abl - INFO - model loss: 1.58456 + abl - INFO - loop(train) [1/1] segment(train) [12/100] + abl - INFO - model loss: 1.62575 + ... + abl - INFO - Eval start: loop(val) [1] + abl - INFO - Evaluation ended, mnist_add/character_accuracy: 0.986 mnist_add/reasoning_accuracy: 0.973 + abl - INFO - Saving model: loop(save) [1] + abl - INFO - Checkpoints will be saved to log_dir/weights/model_checkpoint_loop_1.pth + abl - INFO - Test start: + abl - INFO - Evaluation ended, mnist_add/character_accuracy: 0.983 mnist_add/reasoning_accuracy: 0.967 diff --git a/docs/Examples/Zoo.rst b/docs/Examples/Zoo.rst new file mode 100644 index 0000000..f222d96 --- /dev/null +++ b/docs/Examples/Zoo.rst @@ -0,0 +1,255 @@ +Zoo +=== + +.. raw:: html + +

For detailed code implementation, please view on GitHub.

+ +Below shows an implementation of +`Zoo `__ dataset. In this task, +attributes of animals (such as presence of hair, eggs, etc.) and their +targets (the animal class they belong to) are given, along with a +knowledge base which contain information about the relations between +attributes and targets, e.g., Implies(milk == 1, mammal == 1). + +The goal of this task is to develop a learning model that can predict +the targets of animals based on their attributes. In the initial stages, +when the model is under-trained, it may produce incorrect predictions +that conflict with the relations contained in the knowledge base. When +this happens, abductive reasoning can be employed to adjust these +results and retrain the model accordingly. This process enables us to +further update the learning model. + +.. code:: ipython3 + + # Import necessary libraries and modules + import os.path as osp + + import numpy as np + from sklearn.ensemble import RandomForestClassifier + + from abl.bridge import SimpleBridge + from abl.data.evaluation import ReasoningMetric, SymbolAccuracy + from abl.learning import ABLModel + from abl.reasoning import Reasoner + from abl.utils import ABLLogger, confidence_dist, print_log, tab_data_to_tuple + + from get_dataset import load_and_preprocess_dataset, split_dataset + from kb import ZooKB + +Working with Data +----------------- + +First, we load and preprocess the `Zoo +dataset `__, and split it +into labeled/unlabeled/test data + +.. code:: ipython3 + + X, y = load_and_preprocess_dataset(dataset_id=62) + X_label, y_label, X_unlabel, y_unlabel, X_test, y_test = split_dataset(X, y, test_size=0.3) + +Zoo dataset consist of tabular data. The attributes contains 17 boolean +values (e.g., hair, feathers, eggs, milk, airborne, aquatic, etc.) and +the target is a integer value in range [0,6] representing 7 classes +(e.g., mammal, bird, reptile, fish, amphibian, insect, and other). Below +is an illustration: + +.. code:: ipython3 + + print("Shape of X and y:", X.shape, y.shape) + print("First five elements of X:") + print(X[:5]) + print("First five elements of y:") + print(y[:5]) + +Out: + .. code:: none + :class: code-out + + Shape of X and y: (101, 16) (101,) + First five elements of X: + [[True False False True False False True True True True False False 4 + False False True] + [True False False True False False False True True True False False 4 + True False True] + [False False True False False True True True True False False True 0 + True False False] + [True False False True False False True True True True False False 4 + False False True] + [True False False True False False True True True True False False 4 + True False True]] + First five elements of y: + [0 0 3 0 0] + + +Next, we transform the tabular data to the format required by +ABL-Package, which is a tuple of (X, gt_pseudo_label, Y). In this task, +we treat the attributes as X and the targets as gt_pseudo_label (ground +truth pseudo-labels). Y (reasoning results) are expected to be 0, +indicating no rules are violated. + +.. code:: ipython3 + + label_data = tab_data_to_tuple(X_label, y_label, reasoning_result = 0) + data = tab_data_to_tuple(X_test, y_test, reasoning_result = 0) + train_data = tab_data_to_tuple(X_unlabel, y_unlabel, reasoning_result = 0) + +Building the Learning Part +-------------------------- + +To build the learning part, we need to first build a machine learning +base model. We use a `Random +Forest `__ as the base +model. + +.. code:: ipython3 + + base_model = RandomForestClassifier() + +However, the base model built above deals with instance-level data, and +can not directly deal with example-level data. Therefore, we wrap the +base model into ``ABLModel``, which enables the learning part to train, +test, and predict on example-level data. + +.. code:: ipython3 + + model = ABLModel(base_model) + +Building the Reasoning Part +--------------------------- + +In the reasoning part, we first build a knowledge base which contains +information about the relations between attributes (X) and targets +(pseudo-labels), e.g., Implies(milk == 1, mammal == 1). The knowledge +base is built in the ``ZooKB`` class within file ``examples/zoo/kb.py``, and is +derived from the ``KBBase`` class. + +.. code:: ipython3 + + kb = ZooKB() + +As mentioned, for all attributes and targets in the dataset, the +reasoning results are expected to be 0 since there should be no +violations of the established knowledge in real data. As shown below: + +.. code:: ipython3 + + for idx, (x, y_item) in enumerate(zip(X[:5], y[:5])): + print(f"Example {idx}: the attributes are: {x}, and the target is {y_item}.") + print(f"Reasoning result is {kb.logic_forward([y_item], [x])}.") + print() + +Out: + .. code:: none + :class: code-out + + Example 0: the attributes are: [True False False True False False True True True True False False 4 False + False True], and the target is 0. + Reasoning result is 0. + + Example 1: the attributes are: [True False False True False False False True True True False False 4 True + False True], and the target is 0. + Reasoning result is 0. + + Example 2: the attributes are: [False False True False False True True True True False False True 0 True + False False], and the target is 3. + Reasoning result is 0. + + Example 3: the attributes are: [True False False True False False True True True True False False 4 False + False True], and the target is 0. + Reasoning result is 0. + + Example 4: the attributes are: [True False False True False False True True True True False False 4 True + False True], and the target is 0. + Reasoning result is 0. + + + +Then, we create a reasoner by instantiating the class ``Reasoner``. Due +to the indeterminism of abductive reasoning, there could be multiple +candidates compatible to the knowledge base. When this happens, reasoner +can minimize inconsistencies between the knowledge base and +pseudo-labels predicted by the learning part, and then return only one +candidate that has the highest consistency. + +.. code:: ipython3 + + def consitency(data_example, candidates, candidate_idxs, reasoning_results): + pred_prob = data_example.pred_prob + model_scores = confidence_dist(pred_prob, candidate_idxs) + rule_scores = np.array(reasoning_results) + scores = model_scores + rule_scores + return scores + + reasoner = Reasoner(kb, dist_func=consitency) + +Building Evaluation Metrics +--------------------------- + +Next, we set up evaluation metrics. These metrics will be used to +evaluate the model performance during training and testing. +Specifically, we use ``SymbolAccuracy`` and ``ReasoningMetric``, which +are used to evaluate the accuracy of the machine learning model’s +predictions and the accuracy of the final reasoning results, +respectively. + +.. code:: ipython3 + + metric_list = [SymbolAccuracy(prefix="zoo"), ReasoningMetric(kb=kb, prefix="zoo")] + +Bridging Learning and Reasoning +------------------------------- + +Now, the last step is to bridge the learning and reasoning part. We +proceed this step by creating an instance of ``SimpleBridge``. + +.. code:: ipython3 + + bridge = SimpleBridge(model, reasoner, metric_list) + +Perform training and testing by invoking the ``train`` and ``test`` +methods of ``SimpleBridge``. + +.. code:: ipython3 + + # Build logger + print_log("Abductive Learning on the Zoo example.", logger="current") + log_dir = ABLLogger.get_current_instance().log_dir + weights_dir = osp.join(log_dir, "weights") + + print_log("------- Use labeled data to pretrain the model -----------", logger="current") + base_model.fit(X_label, y_label) + print_log("------- Test the initial model -----------", logger="current") + bridge.test(test_data) + print_log("------- Use ABL to train the model -----------", logger="current") + bridge.train(train_data=train_data, label_data=label_data, loops=3, segment_size=len(X_unlabel), save_dir=weights_dir) + print_log("------- Test the final model -----------", logger="current") + bridge.test(test_data) + + +Out: + .. code:: none + :class: code-out + + abl - INFO - Abductive Learning on the ZOO example. + abl - INFO - ------- Use labeled data to pretrain the model ----------- + abl - INFO - ------- Test the initial model ----------- + abl - INFO - Evaluation ended, zoo/character_accuracy: 0.903 zoo/reasoning_accuracy: 0.903 + abl - INFO - ------- Use ABL to train the model ----------- + abl - INFO - loop(train) [1/3] segment(train) [1/1] + abl - INFO - Evaluation start: loop(val) [1] + abl - INFO - Evaluation ended, zoo/character_accuracy: 1.000 zoo/reasoning_accuracy: 1.000 + abl - INFO - loop(train) [2/3] segment(train) [1/1] + abl - INFO - Evaluation start: loop(val) [2] + abl - INFO - Evaluation ended, zoo/character_accuracy: 1.000 zoo/reasoning_accuracy: 1.000 + abl - INFO - loop(train) [3/3] segment(train) [1/1] + abl - INFO - Evaluation start: loop(val) [3] + abl - INFO - Evaluation ended, zoo/character_accuracy: 1.000 zoo/reasoning_accuracy: 1.000 + abl - INFO - ------- Test the final model ----------- + abl - INFO - Evaluation ended, zoo/character_accuracy: 0.968 zoo/reasoning_accuracy: 0.968 + + +We may see from the results, after undergoing training with ABL, the +model’s accuracy has improved. + diff --git a/docs/Intro/Basics.rst b/docs/Intro/Basics.rst new file mode 100644 index 0000000..a32db34 --- /dev/null +++ b/docs/Intro/Basics.rst @@ -0,0 +1,95 @@ +**Learn the Basics** || +`Quick Start `_ || +`Dataset & Data Structure `_ || +`Learning Part `_ || +`Reasoning Part `_ || +`Evaluation Metrics `_ || +`Bridge `_ + +Learn the Basics +================ + +Modules in ABL-Package +---------------------- + +ABL-Package is an efficient implementation of `Abductive Learning <../Overview/Abductive-Learning.html>`_ (ABL), +a paradigm which integrates machine learning and logical reasoning in a balanced-loop. +The ABL-Package comprises three primary parts: **Data**, **Learning**, and +**Reasoning**, corresponding to the three pivotal components of current +AI: data, models, and knowledge. Below is an overview of the ABL-Package. + +.. image:: ../_static/img/ABL-Package.png + +**Data** part manages the storage, operation, and evaluation of data efficiently. +It includes the ``ListData`` class, which defines the data structures used in +ABL, and comprises common data operations like insertion, deletion, +retrieval, slicing, etc. Additionally, it contains a series of evaluation metrics +such as ``SymbolAccuracy`` and ``ReasoningMetric`` (both specialized metrics +inherited from the ``BaseMetric`` class), for evaluating model quality from a +data perspective. + +:blue-bold:`Learning` part focuses on the construction, training, and +prediction of machine learning models. The ``ABLModel`` class is the +central class that encapsulates the machine learning model. This class is +compatible with various frameworks, including those based on Scikit-learn +or PyTorch neural networks constructed by the ``BasicNN`` class. + +:green-bold:`Reasoning` part concentrates on constructing domain knowledge and +performing reasoning. The ``KBBase`` class allows users to define a +domain knowledge base. For diverse types of knowledge, we also offer +implementations like ``GroundKB`` and ``PrologKB`` (both inherited +from the ``KBBase`` class). The latter, for instance, enables +knowledge bases to be imported in the form of Prolog files. +Upon building the knowledge base, the ``Reasoner`` class is +responsible for minimizing the inconsistency between the knowledge base +and data. + +The integration of these three parts are achieved through the +:yellow-bold:`Bridge` part, which features the ``SimpleBridge`` class (derived +from the ``BaseBridge`` class). The Bridge part synthesizes data, +learning, and reasoning, facilitating the training and testing +of the entire ABL framework. + +Use ABL-Package Step by Step +---------------------------- + +In a typical ABL process, as illustrated below, +data inputs are first predicted by the learning model ``ABLModel.predict``, and the outcomes are pseudo-labels. +These labels then pass through deductive reasoning of the domain knowledge base ``KBBase.logic_forward`` +to obtain the reasoning result. During training, +alongside the aforementioned forward flow (i.e., prediction --> deduction reasoning), +there also exists a reverse flow, which starts from the reasoning result and +involves abductive reasoning ``KBBase.abduce_candidates`` to generate possible revised pseudo-labels. +Subsequently, these pseudo-labels are processed to minimize inconsistencies with the learning part, +which in turn revise the outcomes of the learning model, and then +fed back for further training ``ABLModel.train``. + +.. image:: ../_static/img/usage.png + +To implement this process, the following five steps are necessary: + +1. Prepare **datasets** + + Prepare the data's input, ground truth for pseudo-labels (optional), and ground truth for reasoning results. + +2. :blue:`Build the` :blue-bold:`learning` :blue:`part` + + Build a machine learning base model that can predict inputs to pseudo-labels. + Then, use ``ABLModel`` to encapsulate the base model. + +3. :green:`Build the` :green-bold:`reasoning` :green:`part` + + Define a knowledge base by building a subclass of ``KBBase``, specifying how to + process pseudo-labels to reasoning results. + Also, create a ``Reasoner`` for minimizing inconsistencies + between the knowledge base and data. + +4. Define evaluation metrics + + Define the metrics by building a subclass of ``BaseMetric``. The metrics will + specify how to measure performance during the training and testing of the ABL framework. + +5. :yellow-bold:`Bridge` :yellow:`learning and reasoning` + + Use ``SimpleBridge`` to bridge the learning and reasoning part + for integrated training and testing. diff --git a/docs/Intro/Bridge.rst b/docs/Intro/Bridge.rst new file mode 100644 index 0000000..aa5a51d --- /dev/null +++ b/docs/Intro/Bridge.rst @@ -0,0 +1,95 @@ +`Learn the Basics `_ || +`Quick Start `_ || +`Dataset & Data Structure `_ || +`Learning Part `_ || +`Reasoning Part `_ || +`Evaluation Metrics `_ || +**Bridge** + + +Bridge +====== + +In this section, we will look at how to bridge learning and reasoning parts to train the model, which is the fundamental idea of Abductive Learning. ABL-Package implements a set of bridge classes to achieve this. + +.. code:: python + + from abl.bridge import BaseBridge, SimpleBridge + +``BaseBridge`` is an abstract class with the following initialization parameters: + +- ``model`` is an object of type ``ABLModel``. Learning part are wrapped in this object. +- ``reasoner`` is a object of type ``Reasoner``. Reasoning part are wrapped in this object. + +``BaseBridge`` has the following important methods that need to be overridden in subclasses: + ++---------------------------------------+----------------------------------------------------+ +| Method Signature | Description | ++=======================================+====================================================+ +| ``predict(data_examples)`` | Predicts class probabilities and indices | +| | for the given data examples. | ++---------------------------------------+----------------------------------------------------+ +| ``abduce_pseudo_label(data_examples)``| Abduces pseudo-labels for the given data examples. | ++---------------------------------------+----------------------------------------------------+ +| ``idx_to_pseudo_label(data_examples)``| Converts indices to pseudo-labels using | +| | the provided or default mapping. | ++---------------------------------------+----------------------------------------------------+ +| ``pseudo_label_to_idx(data_examples)``| Converts pseudo-labels to indices | +| | using the provided or default remapping. | ++---------------------------------------+----------------------------------------------------+ +| ``train(train_data)`` | Train the model. | ++---------------------------------------+----------------------------------------------------+ +| ``test(test_data)`` | Test the model. | ++---------------------------------------+----------------------------------------------------+ + +where ``train_data`` and ``test_data`` are both in the form of a tuple or a `ListData <../API/abl.data.html#structures.ListData>`_. Regardless of the form, they all need to include three components: ``X``, ``gt_pseudo_label`` and ``Y``. Since ``ListData`` is the underlying data structure used throughout the ABL-Package, tuple-formed data will be firstly transformed into ``ListData`` in the ``train`` and ``test`` methods, and such ``ListData`` instances are referred to as ``data_examples``. More details can be found in `preparing datasets `_. + +``SimpleBridge`` inherits from ``BaseBridge`` and provides a basic implementation. Besides the ``model`` and ``reasoner``, ``SimpleBridge`` has an extra initialization arguments, ``metric_list``, which will be used to evaluate model performance. Its training process involves several Abductive Learning loops and each loop consists of the following five steps: + + 1. Predict class probabilities and indices for the given data examples. + 2. Transform indices into pseudo-labels. + 3. Revise pseudo-labels based on abdutive reasoning. + 4. Transform the revised pseudo-labels to indices. + 5. Train the model. + +The fundamental part of the ``train`` method is as follows: + +.. code-block:: python + + def train(self, train_data, loops=50, segment_size=10000): + """ + Parameters + ---------- + train_data : Union[ListData, Tuple[List[List[Any]], Optional[List[List[Any]]], List[Any]]] + Training data should be in the form of ``(X, gt_pseudo_label, Y)`` or a ``ListData`` + object with ``X``, ``gt_pseudo_label`` and ``Y`` attributes. + - ``X`` is a list of sublists representing the input data. + - ``gt_pseudo_label`` is only used to evaluate the performance of the ``ABLModel`` but not + to train. ``gt_pseudo_label`` can be ``None``. + - ``Y`` is a list representing the ground truth reasoning result for each sublist in ``X``. + loops : int + Machine Learning part and Reasoning part will be iteratively optimized + for ``loops`` times. + segment_size : Union[int, float] + Data will be split into segments of this size and data in each segment + will be used together to train the model. + """ + if isinstance(train_data, ListData): + data_examples = train_data + else: + data_examples = self.data_preprocess(*train_data) + + if isinstance(segment_size, float): + segment_size = int(segment_size * len(data_examples)) + + for loop in range(loops): + for seg_idx in range((len(data_examples) - 1) // segment_size + 1): + sub_data_examples = data_examples[ + seg_idx * segment_size : (seg_idx + 1) * segment_size + ] + self.predict(sub_data_examples) # 1 + self.idx_to_pseudo_label(sub_data_examples) # 2 + self.abduce_pseudo_label(sub_data_examples) # 3 + self.pseudo_label_to_idx(sub_data_examples) # 4 + loss = self.model.train(sub_data_examples) # 5, self.model is an ABLModel object + diff --git a/docs/Intro/Datasets.rst b/docs/Intro/Datasets.rst new file mode 100644 index 0000000..9aadb99 --- /dev/null +++ b/docs/Intro/Datasets.rst @@ -0,0 +1,89 @@ +`Learn the Basics `_ || +`Quick Start `_ || +**Dataset & Data Structure** || +`Learning Part `_ || +`Reasoning Part `_ || +`Evaluation Metrics `_ || +`Bridge `_ + + +Dataset & Data Structure +======================== + +In this section, we will look at the dataset and data structure in ABL-Package. + +.. code:: python + + import torch + from abl.data.structures import ListData + +Dataset +------- + +ABL-Package requires user data to be either structured as a tuple ``(X, gt_pseudo_label, Y)`` or a ``ListData`` (the underlying data structure utilized in ABL-Package, cf. the next section) object with ``X``, ``gt_pseudo_label`` and ``Y`` attributes. Regardless of the chosen format, the data should encompass three essential components: + +- ``X``: List[List[Any]] + + A list of sublists representing the input data. We refer to each sublist in ``X`` as an **example** and each example may contain several **instances**. + +- ``gt_pseudo_label``: List[List[Any]], optional + + A list of sublists with each sublist representing ground-truth pseudo-labels of an example. Each pseudo-label in the sublist serves as ground-truth for each **instance** within the example. + + .. note:: + + ``gt_pseudo_label`` is only used to evaluate the performance of the learning part but not to train the model. If the pseudo-label of the instances in the datasets are unlabeled, ``gt_pseudo_label`` should be ``None``. + +- ``Y``: List[Any] + + A list representing the ground-truth reasoning result for each **example** in ``X``. + + +.. warning:: + + The length of ``X``, ``gt_pseudo_label`` (if not ``None``) and ``Y`` should be the same. Also, each sublist in ``gt_pseudo_label`` should have the same length as the sublist in ``X``. + +As an illustration, in the MNIST Addition task, the data are organized as follows: + +.. image:: ../_static/img/Datasets_1.png + :width: 350px + :align: center + +.. |data_example| image:: ../_static/img/data_example.png + :alt: alternate text + :scale: 8% + +.. |instance| image:: ../_static/img/instance.png + :alt: alternate text + :scale: 55% + +where each sublist in ``X``, e.g., |data_example|, is a data example and each image in the sublist, e.g., |instance|, is an instance. + +Data Structure +-------------- + +Besides the user-provided dataset, various forms of data are utilized and dynamicly generated throughout the training and testing process of ABL framework. Examples include raw data, predicted pseudo-label, abduced pseudo-label, pseudo-label indices, etc. To manage this diversity and ensure a stable, versatile interface, ABL-Package employs `abstract data interfaces <../API/abl.data.html#structure>`_ to encapsulate different forms of data that will be used in the total learning process. + +``ListData`` is the underlying abstract data interface utilized in ABL-Package. As the fundamental data structure, ``ListData`` implements commonly used data manipulation methods and is responsible for transferring data between various components of ABL, ensuring that stages such as prediction, abductive reasoning, and training can utilize ``ListData`` as a unified input format. Before proceeding to other stages, user-provided datasets will be firstly converted into ``ListData``. + +Besides providing a tuple of ``(X, gt_pseudo_label, Y)``, ABL-Package also allows users to directly supply data in ``ListData`` format, which similarly requires the inclusion of these three attributes. The following code shows the basic usage of ``ListData``. More information can be found in the `API documentation <../API/abl.data.html#structure>`_. + +.. code-block:: python + + # Prepare data + X = [list(torch.randn(3, 28, 28)), list(torch.randn(3, 28, 28))] + gt_pseudo_label = [[1, 2, 3], [4, 5, 6]] + Y = [1, 2] + + # Convert data into ListData + data = ListData(X=X, Y=Y, gt_pseudo_label=gt_pseudo_label) + + # Get data + X = data.X + Y = data.Y + gt_pseudo_label = data.gt_pseudo_label + + # Set data + data.X = X + data.Y = Y + data.gt_pseudo_label = gt_pseudo_label \ No newline at end of file diff --git a/docs/Intro/Evaluation.rst b/docs/Intro/Evaluation.rst new file mode 100644 index 0000000..532edd0 --- /dev/null +++ b/docs/Intro/Evaluation.rst @@ -0,0 +1,52 @@ +`Learn the Basics `_ || +`Quick Start `_ || +`Dataset & Data Structure `_ || +`Learning Part `_ || +`Reasoning Part `_ || +**Evaluation Metrics** || +`Bridge `_ + + +Evaluation Metrics +================== + +In this section, we will look at how to build evaluation metrics. + +.. code:: python + + from abl.data.evaluation import BaseMetric, SymbolAccuracy, ReasoningMetric + +ABL-Package seperates the evaluation process from model training and testing as an independent class, ``BaseMetric``. The training and testing processes are implemented in the ``BaseBridge`` class, so metrics are used by this class and its sub-classes. After building a ``bridge`` with a list of ``BaseMetric`` instances, these metrics will be used by the ``bridge.valid`` method to evaluate the model performance during training and testing. + +To customize our own metrics, we need to inherit from ``BaseMetric`` and implement the ``process`` and ``compute_metrics`` methods. + +- The ``process`` method accepts a batch of model prediction and saves the information to ``self.results`` property after processing this batch. +- The ``compute_metrics`` method uses all the information saved in ``self.results`` to calculate and return a dict that holds the evaluation results. + +Besides, we can assign a ``str`` to the ``prefix`` argument of the ``__init__`` function. This string is automatically prefixed to the output metric names. For example, if we set ``prefix="mnist_add"``, the output metric name will be ``character_accuracy``. +We provide two basic metrics, namely ``SymbolAccuracy`` and ``ReasoningMetric``, which are used to evaluate the accuracy of the machine learning model's predictions and the accuracy of the final reasoning results, respectively. Using ``SymbolAccuracy`` as an example, the following code shows how to implement a custom metrics. + +.. code:: python + + class SymbolAccuracy(BaseMetric): + def __init__(self, prefix: Optional[str] = None) -> None: + # prefix is used to distinguish different metrics + super().__init__(prefix) + + def process(self, data_examples: Sequence[dict]) -> None: + # pred_pseudo_label and gt_pseudo_label are both of type List[List[Any]] + # and have the same length + pred_pseudo_label = data_examples.pred_pseudo_label + gt_pseudo_label = data_examples.gt_pseudo_label + + for pred_z, z in zip(pred_pseudo_label, gt_pseudo_label): + correct_num = 0 + for pred_symbol, symbol in zip(pred_z, z): + if pred_symbol == symbol: + correct_num += 1 + self.results.append(correct_num / len(z)) + + def compute_metrics(self, results: list) -> dict: + metrics = dict() + metrics["character_accuracy"] = sum(results) / len(results) + return metrics \ No newline at end of file diff --git a/docs/Intro/Learning.rst b/docs/Intro/Learning.rst new file mode 100644 index 0000000..dbbea25 --- /dev/null +++ b/docs/Intro/Learning.rst @@ -0,0 +1,86 @@ +`Learn the Basics `_ || +`Quick Start `_ || +`Dataset & Data Structure `_ || +**Learning Part** || +`Reasoning Part `_ || +`Evaluation Metrics `_ || +`Bridge `_ + + +Learning Part +============= + +In this section, we will look at how to build the learning part. + +In ABL-Package, building the learning part involves two steps: + +1. Build a machine learning base model used to make predictions on instance-level data. +2. Instantiate an ``ABLModel`` with the base model, which enables the learning part to process example-level data. + +.. code:: python + + import sklearn + import torchvision + from abl.learning import BasicNN, ABLModel + +Building a base model +--------------------- + +ABL package allows the base model to be one of the following forms: + +1. Any machine learning model conforming to the scikit-learn style, i.e., models which has implemented the ``fit`` and ``predict`` methods; + +2. A PyTorch-based neural network, provided it has defined the architecture and implemented the ``forward`` method. + +For a scikit-learn model, we can directly use the model itself as a base model. For example, we can customize our base model by a KNN classfier: + +.. code:: python + + base_model = sklearn.neighbors.KNeighborsClassifier(n_neighbors=3) + +For a PyTorch-based neural network, we need to encapsulate it within a ``BasicNN`` object to create a base model. For example, we can customize our base model by a pre-trained ResNet-18: + +.. code:: python + + # Load a PyTorch-based neural network + cls = torchvision.models.resnet18(pretrained=True) + + # loss function and optimizer are used for training + loss_fn = torch.nn.CrossEntropyLoss() + optimizer = torch.optim.Adam(cls.parameters()) + + base_model = BasicNN(cls, loss_fn, optimizer) + +BasicNN +^^^^^^^ + +``BasicNN`` is a wrapper class for PyTorch-based neural networks, which enables them to work as scikit-learn models. It encapsulates the neural network, loss function, optimizer, and other elements into a single object, which can be used as a base model. + +Besides the necessary methods required to instantiate an ``ABLModel``, i.e., ``fit`` and ``predict``, ``BasicNN`` also implements the following methods: + ++-------------------------------+------------------------------------------+ +| Method | Function | ++===============================+==========================================+ +| ``train_epoch(data_loader)`` | Train the neural network for one epoch. | ++-------------------------------+------------------------------------------+ +| ``predict_proba(X)`` | Predict the class probabilities of ``X``.| ++-------------------------------+------------------------------------------+ +| ``score(X, y)`` | Calculate the accuracy of the model on | +| | test data. | ++-------------------------------+------------------------------------------+ +| ``save(epoch_id, save_path)`` | Save the model. | ++-------------------------------+------------------------------------------+ +| ``load(load_path)`` | Load the model. | ++-------------------------------+------------------------------------------+ + +Instantiating an ABLModel +------------------------- + +Typically, base model is trained to make predictions on instance-level data, and can not directly process example-level data, which is not suitable for most neural-symbolic tasks. ABL-Package provides the ``ABLModel`` to solve this problem. This class serves as a unified wrapper for all base models, which enables the learning part to train, test, and predict on example-level data. + +Generally, we can simply instantiate an ``ABLModel`` by: + +.. code:: python + + # Instantiate an ABLModel + model = ABLModel(base_model) \ No newline at end of file diff --git a/docs/Intro/Quick-Start.rst b/docs/Intro/Quick-Start.rst new file mode 100644 index 0000000..658e0ba --- /dev/null +++ b/docs/Intro/Quick-Start.rst @@ -0,0 +1,129 @@ +`Learn the Basics `_ || +**Quick Start** || +`Dataset & Data Structure `_ || +`Learning Part `_ || +`Reasoning Part `_ || +`Evaluation Metrics `_ || +`Bridge `_ + +Quick Start +=========== + +We use the MNIST Addition task as a quick start example. In this task, pairs of MNIST handwritten images and their sums are given, alongwith a domain knowledge base which contain information on how to perform addition operations. Our objective is to input a pair of handwritten images and accurately determine their sum. Refer to the links in each section to dive deeper. + +Working with Data +----------------- + +ABL-Package requires data in the format of ``(X, gt_pseudo_label, Y)`` where ``X`` is a list of input examples containing instances, +``gt_pseudo_label`` is the ground-truth label of each example in ``X`` and ``Y`` is the ground-truth reasoning result of each example in ``X``. Note that ``gt_pseudo_label`` is only used to evaluate the machine learning model's performance but not to train it. + +In the MNIST Addition task, the data loading looks like + +.. code:: python + + # The 'datasets' module below is located in 'examples/mnist_add/' + from datasets import get_dataset + + # train_data and test_data are tuples in the format of (X, gt_pseudo_label, Y) + train_data = get_dataset(train=True) + test_data = get_dataset(train=False) + +Read more about `preparing datasets `_. + +Building the Learning Part +-------------------------- + +Learning part is constructed by first defining a base model for machine learning. The ABL-Package offers considerable flexibility, supporting any base model that conforms to the scikit-learn style (which requires the implementation of ``fit`` and ``predict`` methods), or a PyTorch-based neural network (which has defined the architecture and implemented ``forward`` method). +In this example, we build a simple LeNet5 network as the base model. + +.. code:: python + + # The 'models' module below is located in 'examples/mnist_add/' + from models.nn import LeNet5 + + cls = LeNet5(num_classes=10) + +To facilitate uniform processing, ABL-Package provides the ``BasicNN`` class to convert a PyTorch-based neural network into a format compatible with scikit-learn models. To construct a ``BasicNN`` instance, aside from the network itself, we also need to define a loss function, an optimizer, and the computing device. + +.. code:: python + + import torch + from abl.learning import BasicNN + + loss_fn = torch.nn.CrossEntropyLoss() + optimizer = torch.optim.RMSprop(cls.parameters(), lr=0.001) + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + base_model = BasicNN(model=cls, loss_fn=loss_fn, optimizer=optimizer, device=device) + +The base model built above are trained to make predictions on instance-level data (e.g., a single image), while ABL deals with example-level data. To bridge this gap, we wrap the ``base_model`` into an instance of ``ABLModel``. This class serves as a unified wrapper for base models, facilitating the learning part to train, test, and predict on example-level data, (e.g., images that comprise an equation). + +.. code:: python + + from abl.learning import ABLModel + + model = ABLModel(base_model) + +Read more about `building the learning part `_. + +Building the Reasoning Part +--------------------------- + +To build the reasoning part, we first define a knowledge base by creating a subclass of ``KBBase``. In the subclass, we initialize the ``pseudo_label_list`` parameter and override the ``logic_forward`` method, which specifies how to perform (deductive) reasoning that processes pseudo-labels of an example to the corresponding reasoning result. Specifically for the MNIST Addition task, this ``logic_forward`` method is tailored to execute the sum operation. + +.. code:: python + + from abl.reasoning import KBBase + + class AddKB(KBBase): + def __init__(self, pseudo_label_list=list(range(10))): + super().__init__(pseudo_label_list) + + def logic_forward(self, nums): + return sum(nums) + + kb = AddKB() + +Next, we create a reasoner by instantiating the class ``Reasoner``, passing the knowledge base as a parameter. +Due to the indeterminism of abductive reasoning, there could be multiple candidate pseudo-labels compatible to the knowledge base. +In such scenarios, the reasoner can minimize inconsistency and return the pseudo-label with the highest consistency. + +.. code:: python + + from abl.reasoning import Reasoner + + reasoner = Reasoner(kb) + +Read more about `building the reasoning part `_. + +Building Evaluation Metrics +--------------------------- + +ABL-Package provides two basic metrics, namely ``SymbolAccuracy`` and ``ReasoningMetric``, which are used to evaluate the accuracy of the machine learning model's predictions and the accuracy of the ``logic_forward`` results, respectively. + +.. code:: python + + from abl.data.evaluation import ReasoningMetric, SymbolAccuracy + + metric_list = [SymbolAccuracy(), ReasoningMetric(kb=kb)] + +Read more about `building evaluation metrics `_ + +Bridging Learning and Reasoning +--------------------------------------- + +Now, we use ``SimpleBridge`` to combine learning and reasoning in a unified ABL framework. + +.. code:: python + + from abl.bridge import SimpleBridge + + bridge = SimpleBridge(model, reasoner, metric_list) + +Finally, we proceed with training and testing. + +.. code:: python + + bridge.train(train_data, loops=1, segment_size=0.01) + bridge.test(test_data) + +Read more about `bridging machine learning and reasoning `_. diff --git a/docs/Intro/Reasoning.rst b/docs/Intro/Reasoning.rst new file mode 100644 index 0000000..e93bc41 --- /dev/null +++ b/docs/Intro/Reasoning.rst @@ -0,0 +1,381 @@ +`Learn the Basics `_ || +`Quick Start `_ || +`Dataset & Data Structure `_ || +`Learning Part `_ || +**Reasoning Part** || +`Evaluation Metrics `_ || +`Bridge `_ + + +Reasoning part +=============== + +In this section, we will look at how to build the reasoning part, which +leverage domain knowledge and perform deductive or abductive reasoning. +In ABL-Package, building the reasoning part involves two steps: + +1. Build a knowledge base by creating a subclass of ``KBBase``, which + specifies how to process pseudo-label of an example to the reasoning result. +2. Create a reasoner by instantiating the class ``Reasoner`` + to minimize inconsistencies between the knowledge base and pseudo + labels predicted by the learning part. + +.. code:: python + + from abl.reasoning import KBBase, GroundKB, PrologKB, Reasoner + +Building a knowledge base +------------------------- + +Generally, we can create a subclass derived from ``KBBase`` to build our own +knowledge base. In addition, ABL-Package also offers several predefined +subclasses of ``KBBase`` (e.g., ``PrologKB`` and ``GroundKB``), +which we can utilize to build our knowledge base more conveniently. + +Building a knowledge base from ``KBBase`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For the user-built KB from ``KBBase`` (a derived subclass), it's only +required to pass the ``pseudo_label_list`` parameter in the ``__init__`` function +and override the ``logic_forward`` function: + +- ``pseudo_label_list`` is the list of possible pseudo-labels (also, + the output of the machine learning model). +- ``logic_forward`` defines how to perform (deductive) reasoning, + i.e. matching each pseudo-labels to its reasoning result. + +.. note:: + + Generally, the overridden function ``logic_forward`` provided by the user accepts + only one parameter, ``pseudo_label`` (pseudo-labels of an example). However, for certain + scenarios, deductive reasoning in the knowledge base may necessitate information + from the input. In these scenarios, ``logic_forward`` can also accept two parameters: + ``pseudo_label`` and ``x``. See examples in `Zoo <../Examples/Zoo.html>`_. + +After that, other operations, including how to perform abductive +reasoning, will be **automatically** set up. + +MNIST Addition example +^^^^^^^^^^^^^^^^^^^^^^ + +As an example, the ``pseudo_label_list`` passed in MNIST Addition is all the +possible digits, namely, ``[0,1,2,...,9]``, and the ``logic_forward`` +should be: “Add the two pseudo-labels to get the result.”. Therefore, the +construction of the KB (``add_kb``) for MNIST Addition would be: + +.. code:: python + + class AddKB(KBBase): + def __init__(self, pseudo_label_list=list(range(10))): + super().__init__(pseudo_label_list) + + def logic_forward(self, pseudo_labels): + return sum(pseudo_labels) + + add_kb = AddKB() + +and (deductive) reasoning in ``add_kb`` would be: + +.. code:: python + + pseudo_labels = [1, 2] + reasoning_result = add_kb.logic_forward(pseudo_labels) + print(f"Reasoning result of pseudo-labels {pseudo_labels} is {reasoning_result}.") + +Out: + .. code:: none + :class: code-out + + Reasoning result of pseudo-labels [1, 2] is 3 + +.. _other-par: + +Other optional parameters +^^^^^^^^^^^^^^^^^^^^^^^^^ + +We can also pass the following parameters in the ``__init__`` function when building our +knowledge base: + +- ``max_err`` (float, optional), specifying the upper tolerance limit + when comparing the similarity between the reasoning result of pseudo-labels + and the ground truth during abductive reasoning. This is only + applicable when the reasoning result is of a numerical type. This is + particularly relevant for regression problems where exact matches + might not be feasible. Defaults to 1e-10. See :ref:`an example `. +- ``use_cache`` (bool, optional), indicating whether to use cache to store + previous candidates (pseudo-labels generated from abductive reasoning) + to speed up subsequent abductive reasoning operations. Defaults to True. + For more information of abductive reasoning, please refer to :ref:`this `. +- ``cache_size`` (int, optional), specifying the maximum cache + size. This is only operational when ``use_cache`` is set to True. + Defaults to 4096. + +.. _prolog: + +Building a knowledge base from Prolog file +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When aiming to leverage knowledge base from an external Prolog file +(which contains how to perform reasoning), we can directly create an +instance of class ``PrologKB``. Upon instantiation of +``PrologKB``, we are required to pass the ``pseudo_label_list`` (same as ``KBBase``) +and ``pl_file`` (the Prolog file) in the ``__init__`` function. + +.. admonition:: What is a Prolog file? + + A Prolog file (typically have the extension ``.pl``) is a script or source + code file written in the Prolog language. Prolog is a logic programming language + where the logic is represented as facts + (basic assertions about some world) and + rules (logical statements that describe the relationships between facts). + A computation is initiated by running a query over these facts and rules. + See some Prolog examples + in `SWISH `_. + +After the instantiation, other operations, including how to perform +abductive reasoning, will also be **automatically** set up. + +.. warning:: + + Note that to use the default logic forward and abductive reasoning + methods in this class, the Prolog (.pl) file should contain a rule + with a strict format: ``logic_forward(Pseudo_labels, Res).`` + Otherwise, we might have to override ``logic_forward`` and + ``get_query_string`` to allow for more adaptable usage. + +MNIST Addition example (cont.) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +As an example, we can first write a Prolog file for the MNIST Addition +example as the following code, and then save it as ``add.pl``. + +.. code:: prolog + + pseudo_label(N) :- between(0, 9, N). + logic_forward([Z1, Z2], Res) :- pseudo_label(Z1), pseudo_label(Z2), Res is Z1+Z2. + +Afterwards, the construction of knowledge base from Prolog file +(``add_prolog_kb``) would be as follows: + +.. code:: python + + add_prolog_kb = PrologKB(pseudo_label_list=list(range(10)), pl_file="add.pl") + +Building a knowledge base with GKB from ``GroundKB`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +We can also inherit from class ``GroundKB`` to build our own +knowledge base. In this way, the knowledge built will have a Ground KB +(GKB). + +.. admonition:: What is Ground KB? + + `Ground KB `_ is a knowledge base prebuilt upon class initialization, + storing all potential candidates along with their respective reasoning + result. The key advantage of having a Ground KB is that it may + accelerate abductive reasoning. + +``GroundKB`` is a subclass of ``GKBBase``. Similar to ``KBBase``, we +are required to pass the ``pseudo_label_list`` parameter in the ``__init__`` function and +override the ``logic_forward`` function, and are allowed to pass other +:ref:`optional parameters `. Additionally, we are required pass the +``GKB_len_list`` parameter in the ``__init__`` function. + +- ``GKB_len_list`` is the list of possible lengths for pseudo-labels of an example. + +After that, other operations, including auto-construction of GKB, and +how to perform abductive reasoning, will be **automatically** set up. + +MNIST Addition example (cont.) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +As an example, the ``GKB_len_list`` for MNIST Addition should be ``[2]``, +since all pseudo-labels in the example consist of two digits. Therefore, +the construction of KB with GKB (``add_ground_kb``) of MNIST Addition would be +as follows. As mentioned, the difference between this and the previously +built ``add_kb`` lies only in the base class from which it is derived +and whether an extra parameter ``GKB_len_list`` is passed. + +.. code:: python + + class AddGroundKB(GroundKB): + def __init__(self, pseudo_label_list=list(range(10)), + GKB_len_list=[2]): + super().__init__(pseudo_label_list, GKB_len_list) + + def logic_forward(self, nums): + return sum(nums) + + add_ground_kb = AddGroundKB() + +.. _kb-abd: + +Performing abductive reasoning in the knowledge base +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +As mentioned in :ref:`What is Abductive Reasoning? `, abductive reasoning +enables the inference of candidates (i.e., possible pseudo-labels) as potential +explanations for the reasoning result. Also, in Abductive Learning where +an observation (pseudo-labels of an example predicted by the learning part) is +available, we aim to let the candidate do not largely revise the +previously identified pseudo-labels. + +``KBBase`` (also, ``GroundKB`` and ``PrologKB``) implement the method +``abduce_candidates(pseudo_label, y, x, max_revision_num, require_more_revision)`` +for performing abductive reasoning, where the parameters are: + +- ``pseudo_label``, pseudo-labels of an example, usually generated by the learning + part. They are to be revised by abductive reasoning. +- ``y``, the ground truth of the reasoning result for the example. The + returned candidates should be compatible with it. +- ``x``, the corresponding input example. If the information from the input + is not required in the reasoning process, then this parameter will not have + any effect. +- ``max_revision_num``, an int value specifying the upper limit on the + number of revised labels for each example. +- ``require_more_revision``, an int value specifying additional number + of revisions permitted beyond the minimum required. (e.g., If we set + it to 0, even if ``max_revision_num`` is set to a high value, the + method will only output candidates with the minimum possible + revisions.) + +And it return a list of candidates (i.e., revised pseudo-labels of the example) +that are all compatible with ``y``. + +MNIST Addition example (cont.) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +As an example, with MNIST Addition, the candidates returned by +``add_kb.abduce_candidates`` would be as follows: + ++------------------+-------+----------------------+--------------------------+----------------+ +| ``pseudo_label`` | ``y`` | ``max_revision_num`` | ``require_more_address`` | Output | ++==================+=======+======================+==========================+================+ +| [1,1] | 8 | 1 | 0 | [[1,7], [7,1]] | ++------------------+-------+----------------------+--------------------------+----------------+ +| [1,1] | 8 | 1 | 1 | [[1,7], [7,1]] | ++------------------+-------+----------------------+--------------------------+----------------+ +| [1,1] | 8 | 2 | 0 | [[1,7], [7,1]] | ++------------------+-------+----------------------+--------------------------+----------------+ +| [1,1] | 8 | 2 | 1 | [[1,7], | +| | | | | [7,1], [2,6], | +| | | | | [6,2], [3,5], | +| | | | | [5,3], [4,4]] | ++------------------+-------+----------------------+--------------------------+----------------+ +| [1,1] | 11 | 1 | 0 | [] | ++------------------+-------+----------------------+--------------------------+----------------+ + +.. _kb-abd-2: + +As another example, if we set the ``max_err`` of ``AddKB`` to be 1 +instead of the default 1e-10, the tolerance limit for consistency will +be higher, hence the candidates returned would be: + ++------------------+-------+----------------------+--------------------------+----------------+ +| ``pseudo_label`` | ``y`` | ``max_revision_num`` | ``require_more_address`` | Output | ++==================+=======+======================+==========================+================+ +| [1,1] | 8 | 1 | 0 | [[1,7], [7,1], | +| | | | | [1,6], [6,1], | +| | | | | [1,8], [8,1]] | ++------------------+-------+----------------------+--------------------------+----------------+ +| [1,1] | 11 | 1 | 0 | [[1,9], [9,1]] | ++------------------+-------+----------------------+--------------------------+----------------+ + +Creating a reasoner +------------------- + +After building our knowledge base, the next step is creating a +reasoner. Due to the indeterminism of abductive reasoning, there could +be multiple candidates compatible to the knowledge base. When this +happens, reasoner can minimize inconsistencies between the knowledge +base and pseudo-labels predicted by the learning part, and then return **only +one** candidate that has the highest consistency. + +We can create a reasoner simply by instantiating class +``Reasoner`` and passing our knowledge base as an parameter. As an +example for MNIST Addition, the reasoner definition would be: + +.. code:: python + + reasoner_add = Reasoner(kb_add) + +When instantiating, besides the required knowledge base, we may also +specify: + +- ``max_revision`` (int or float, optional), specifies the upper limit + on the number of revisions for each example when performing + :ref:`abductive reasoning in the knowledge base `. If float, denotes the + fraction of the total length that can be revised. A value of -1 + implies no restriction on the number of revisions. Defaults to -1. +- ``require_more_revision`` (int, optional), Specifies additional + number of revisions permitted beyond the minimum required when + performing :ref:`abductive reasoning in the knowledge base `. Defaults to + 0. +- ``use_zoopt`` (bool, optional), indicating whether to use the `ZOOpt library `_, + which is a library for zeroth-order optimization that can be used to + accelerate consistency minimization. Defaults to False. +- ``dist_func`` (str, optional), specifying the distance function to be + used when determining consistency between your prediction and + candidate returned from knowledge base. Valid options include + “confidence” (default) and “hamming”. For “confidence”, it calculates + the distance between the prediction and candidate based on confidence + derived from the predicted probability in the data example. For + “hamming”, it directly calculates the Hamming distance between the + predicted pseudo-label in the data example and candidate. +- ``idx_to_label`` (dict, optional), a mapping from index in the base model to label. + If not provided, a default order-based index to label mapping is created. + Defaults to None. + +The main method implemented by ``Reasoner`` is +``abduce(data_example)``, which obtains the most consistent candidate +based on the distance function defined in ``dist_func``. + +MNIST Addition example (cont.) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +As an example, consider these data examples for MNIST Addition: + +.. code:: python + + # favor "1" for the first label + prob1 = [[0, 0.99, 0, 0, 0, 0, 0, 0.01, 0, 0], + [0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1]] + + # favor "7" for the first label + prob2 = [[0, 0.01, 0, 0, 0, 0, 0, 0.99, 0, 0], + [0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1]] + + example1 = ListData() + example1.pred_pseudo_label = [1, 1] + example1.pred_prob = prob1 + example1.Y = 8 + + example2 = ListData() + example2.pred_pseudo_label = [1, 1] + example2.pred_prob = prob2 + example2.Y = 8 + +The compatible candidates after abductive reasoning for both examples +would be ``[[1,7], [7,1]]``. However, when the reasoner call ``abduce`` +to select only one candidate based on the ``confidence`` distance function, +the output would differ for each example: + +.. code:: python + + reasoner_add = Reasoner(kb_add, dist_func="confidence") + candidate1 = reasoner_add.abduce(example1) + candidate2 = reasoner_add.abduce(example2) + print(f"The outputs for example1 and example2 are {candidate1} and {candidate2}, respectively.") + +Out: + .. code:: none + :class: code-out + + The outputs for example1 and example2 are [1,7] and [7,1], respectively. + +Specifically, as mentioned before, ``confidence`` calculates the distance between the data +example and candidates based on the confidence derived from the predicted probability. +Take ``example1`` as an example, the ``pred_prob`` in it indicates a higher +confidence that the first label should be "1" rather than "7". Therefore, among the +candidates [1,7] and [7,1], it would be closer to [1,7] (as its first label is "1"). + diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..caeb32e --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = ABL-Package +SOURCEDIR = . +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/Overview/Abductive-Learning.rst b/docs/Overview/Abductive-Learning.rst new file mode 100644 index 0000000..9fbe25f --- /dev/null +++ b/docs/Overview/Abductive-Learning.rst @@ -0,0 +1,80 @@ +Abductive Learning +================== + +Traditional supervised machine learning, e.g. classification, is +predominantly data-driven, as shown in the below figure. +Here, a set of data examples is given, including training instances +:math:`\{x_1,\dots,x_m\}` and corresponding ground-truth labels :math:`\{\text{label}(x_1),\dots,\text{label}(x_m)\}`. +These data are then used to train a classifier model :math:`f`, +aiming to accurately predict the unseen data instances. + +.. image:: ../_static/img/ML.png + :align: center + :width: 280px + +In **Abductive Learning (ABL)**, we assume that, in addition to data, +there is also a knowledge base :math:`\mathcal{KB}` containing +domain knowledge at our disposal. We aim for the classifier :math:`f` +to make correct predictions on data instances :math:`\{x_1,\dots,x_m\}`, +and meanwhile, the pseudo-groundings grounded by the prediction +:math:`\left\{f(\boldsymbol{x}_1), \ldots, f(\boldsymbol{x}_m)\right\}` +should be compatible with :math:`\mathcal{KB}`. + +The process of ABL is as follows: + +1. Upon receiving data instances :math:`\left\{x_1,\dots,x_m\right\}` as input, + pseudo-labels + :math:`\left\{f(\boldsymbol{x}_1), \ldots, f(\boldsymbol{x}_m)\right\}` + are predicted by a data-driven classifier model. +2. These pseudo-labels are then converted into pseudo-groundings + :math:`\mathcal{O}` that are acceptable for logical reasoning. +3. Conduct joint reasoning with :math:`\mathcal{KB}` to find any + inconsistencies. If found, the pseudo-groundings that lead to minimal + inconsistency can be identified. +4. Modify the identified facts through **abductive reasoning** (or, **abduction**), + returning revised pseudo-groundings :math:`\Delta(\mathcal{O})` which are + compatible with :math:`\mathcal{KB}`. +5. These revised pseudo-groundings are converted back to the form of + pseudo-labels, and used like ground-truth labels in conventional + supervised learning to train a new classifier. +6. The new classifier will then be adopted to replace the previous one + in the next iteration. + +This above process repeats until the classifier is no longer updated, or +the pseudo-groundings :math:`\mathcal{O}` are compatible with the knowledge +base. + +The following figure illustrates this process: + +.. image:: ../_static/img/ABL.png + :width: 800px + +We can observe that in the above figure, the left half involves machine +learning, while the right half involves logical reasoning. Thus, the +entire Abductive Learning process is a continuous cycle of machine +learning and logical reasoning. This effectively forms a paradigm that +is dual-driven by both data and domain knowledge, integrating and +balancing the use of machine learning and logical reasoning in a unified +model. + +For more information about ABL, please refer to: `Zhou, 2019 `_ +and `Zhou and Huang, 2022 `_. + +.. _abd: + +.. admonition:: What is Abductive Reasoning? + + Abductive reasoning, also known as abduction, refers to the process of + selectively inferring certain facts and hypotheses that explain + phenomena and observations based on background knowledge. Unlike + deductive reasoning, which leads to definitive conclusions, abductive + reasoning may arrive at conclusions that are plausible but not conclusively + proven. + + In ABL, given :math:`\mathcal{KB}` (typically expressed + in first-order logic clauses), one can perform both deductive and + abductive reasoning. Deductive reasoning allows deriving + :math:`b` from :math:`a`, while abductive reasoning allows inferring + :math:`a` as an explanation of :math:`b`. In other words, + deductive reasoning and abductive reasoning differ in which end, + right or left, of the proposition “:math:`a\models b`” serves as conclusion. diff --git a/docs/Overview/Installation.rst b/docs/Overview/Installation.rst new file mode 100644 index 0000000..ecabd0e --- /dev/null +++ b/docs/Overview/Installation.rst @@ -0,0 +1,34 @@ +Installation +================== + +ABL is distributed on `PyPI `__ and can be installed with ``pip``: + +.. code:: console + + # (TODO) + $ pip install abl + +For testing purposes, you can install it using: + +.. code:: console + + $ pip install -i https://test.pypi.org/simple/ --extra-index-url https://mirrors.nju.edu.cn/pypi/web/simple/ abl + +Alternatively, to install ABL by source code, +sequentially run following commands in your terminal/command line. + +.. code:: console + + $ git clone https://github.com/AbductiveLearning/ABL-Package.git + $ cd ABL-Package + $ pip install -v -e . + +(Optional) If the use of a :ref:`Prolog-based knowledge base ` is necessary, the installation of `Swi-Prolog `_ is also required: + +For Linux users: + +.. code:: console + + $ sudo apt-get install swi-prolog + +For Windows and Mac users, please refer to the `Swi-Prolog Install Guide `_. \ No newline at end of file diff --git a/docs/README.rst b/docs/README.rst new file mode 100644 index 0000000..c83deae --- /dev/null +++ b/docs/README.rst @@ -0,0 +1,60 @@ +ABL-Package +=========== + +**ABL-Package** is an open source library for **Abductive Learning (ABL)**. +ABL is a novel paradigm that integrates machine learning and +logical reasoning in a unified framework. It is suitable for tasks +where both data and (logical) domain knowledge are available. + +Key Features of ABL-Package: + +- **Great Flexibility**: Adaptable to various machine learning modules and logical reasoning components. +- **User-Friendly**: Provide data, model, and KB, and get started with just a few lines of code. +- **High-Performance**: Optimization for high accuracy and fast training speed. + +ABL-Package encapsulates advanced ABL techniques, providing users with +an efficient and convenient package to develop dual-driven ABL systems, +which leverage the power of both data and knowledge. + +.. image:: _static/img/ABL.png + +Installation +------------ + +ABL is distributed on `PyPI `__ and can be installed with ``pip``: + +.. code:: console + + # (TODO) + $ pip install abl + +For testing purposes, you can install it using: + +.. code:: console + + $ pip install -i https://test.pypi.org/simple/ --extra-index-url https://mirrors.nju.edu.cn/pypi/web/simple/ abl + +Alternatively, to install ABL by source code, +sequentially run following commands in your terminal/command line. + +.. code:: console + + $ git clone https://github.com/AbductiveLearning/ABL-Package.git + $ cd ABL-Package + $ pip install -v -e . + +(Optional) If the use of a :ref:`Prolog-based knowledge base ` is necessary, the installation of `Swi-Prolog `_ is also required: + +For Linux users: + +.. code:: console + + $ sudo apt-get install swi-prolog + +For Windows and Mac users, please refer to the `Swi-Prolog Install Guide `_. + +References +---------- + +For more information about ABL, please refer to: `Zhou, 2019 `_ +and `Zhou and Huang, 2022 `_. \ No newline at end of file diff --git a/docs/References.rst b/docs/References.rst new file mode 100644 index 0000000..85ad7e5 --- /dev/null +++ b/docs/References.rst @@ -0,0 +1,10 @@ +References +========== + +Zhi-Hua Zhou. `Abductive learning: Towards bridging machine learning and logical reasoning. `_. **Science China Information Sciences**, 2019, 62: 076101. + +Zhi-Hua Zhou and Yu-Xuan Huang. `Abductive learning `_. In P. Hitzler and M. K. Sarker eds., **Neuro-Symbolic Artificial Intelligence: The State of the Art**, IOP Press, Amsterdam, 2022, p.353-379 + + + + diff --git a/docs/_static/custom.css b/docs/_static/custom.css new file mode 100644 index 0000000..de4f232 --- /dev/null +++ b/docs/_static/custom.css @@ -0,0 +1,24 @@ +div.code-out > div.highlight > pre { + background-color: #d3effd !important; +} +.green-bold { + color: green; + font-weight: bold; +} +.blue-bold { + color: blue; + font-weight: bold; +} +.yellow-bold { + color: rgb(255, 192, 0); + font-weight: bold; +} +.green { + color: green; +} +.blue { + color: blue; +} +.yellow { + color: rgb(255, 192, 0); +} \ No newline at end of file diff --git a/docs/_static/img/ABL-Package.png b/docs/_static/img/ABL-Package.png new file mode 100644 index 0000000..0cf0cfb Binary files /dev/null and b/docs/_static/img/ABL-Package.png differ diff --git a/docs/_static/img/ABL.png b/docs/_static/img/ABL.png new file mode 100644 index 0000000..2cedb4a Binary files /dev/null and b/docs/_static/img/ABL.png differ diff --git a/docs/_static/img/Datasets_1.png b/docs/_static/img/Datasets_1.png new file mode 100644 index 0000000..900b2e5 Binary files /dev/null and b/docs/_static/img/Datasets_1.png differ diff --git a/docs/_static/img/ML.png b/docs/_static/img/ML.png new file mode 100644 index 0000000..7faa733 Binary files /dev/null and b/docs/_static/img/ML.png differ diff --git a/docs/_static/img/data_example.png b/docs/_static/img/data_example.png new file mode 100644 index 0000000..a42e542 Binary files /dev/null and b/docs/_static/img/data_example.png differ diff --git a/docs/_static/img/hed_dataset1.png b/docs/_static/img/hed_dataset1.png new file mode 100644 index 0000000..4908bcb Binary files /dev/null and b/docs/_static/img/hed_dataset1.png differ diff --git a/docs/_static/img/hed_dataset2.png b/docs/_static/img/hed_dataset2.png new file mode 100644 index 0000000..46de604 Binary files /dev/null and b/docs/_static/img/hed_dataset2.png differ diff --git a/docs/_static/img/hed_dataset3.png b/docs/_static/img/hed_dataset3.png new file mode 100644 index 0000000..7e46295 Binary files /dev/null and b/docs/_static/img/hed_dataset3.png differ diff --git a/docs/_static/img/hed_dataset4.png b/docs/_static/img/hed_dataset4.png new file mode 100644 index 0000000..39b3dcb Binary files /dev/null and b/docs/_static/img/hed_dataset4.png differ diff --git a/docs/_static/img/hwf_dataset1.png b/docs/_static/img/hwf_dataset1.png new file mode 100644 index 0000000..ddb623d Binary files /dev/null and b/docs/_static/img/hwf_dataset1.png differ diff --git a/docs/_static/img/hwf_dataset2.png b/docs/_static/img/hwf_dataset2.png new file mode 100644 index 0000000..f3b6e93 Binary files /dev/null and b/docs/_static/img/hwf_dataset2.png differ diff --git a/docs/_static/img/instance.png b/docs/_static/img/instance.png new file mode 100644 index 0000000..76a1b3b Binary files /dev/null and b/docs/_static/img/instance.png differ diff --git a/docs/_static/img/mnist_add_datasets.png b/docs/_static/img/mnist_add_datasets.png new file mode 100644 index 0000000..8e8d8b4 Binary files /dev/null and b/docs/_static/img/mnist_add_datasets.png differ diff --git a/docs/_static/img/usage.png b/docs/_static/img/usage.png new file mode 100644 index 0000000..d16e940 Binary files /dev/null and b/docs/_static/img/usage.png differ diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..dcd522d --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,115 @@ +import os +import re +import sys + +from docutils import nodes +from docutils.parsers.rst import roles + +from sphinx.application import Sphinx + + +def remove_noqa(app: Sphinx, what: str, name: str, obj, options, lines): + new_lines = [] + for line in lines: + new_line = re.sub(r"\s*#\s*noqa.*$", "", line) + new_lines.append(new_line) + lines[:] = new_lines + + +def colored_text_role(role, rawtext, text, lineno, inliner, options={}, content=[]): + node = nodes.inline(rawtext, text, classes=[role]) + return [node], [] + + +roles.register_local_role("green-bold", colored_text_role) +roles.register_local_role("blue-bold", colored_text_role) +roles.register_local_role("yellow-bold", colored_text_role) +roles.register_local_role("green", colored_text_role) +roles.register_local_role("blue", colored_text_role) +roles.register_local_role("yellow", colored_text_role) + + +if "READTHEDOCS" not in os.environ: + sys.path.insert(0, os.path.abspath("..")) +sys.path.append(os.path.abspath("./ABL/")) + + +project = "ABL" +slug = re.sub(r"\W+", "-", project.lower()) +project = "ABL-Package" +copyright = "LAMDA, 2024" +author = "Author" + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [ + "sphinx.ext.intersphinx", + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.mathjax", + "sphinx.ext.viewcode", + "sphinx_rtd_theme", + "recommonmark", + "sphinx_markdown_tables", + "sphinx.ext.napoleon", + "sphinx_copybutton", +] + +templates_path = ["_templates"] +source_suffix = [".rst", ".md"] +exclude_patterns = [] +# locale_dirs = ['locale/'] +gettext_compact = False + +master_doc = "index" +suppress_warnings = ["image.nonlocal_uri"] +pygments_style = "default" + +# intersphinx_mapping = { +# 'rtd': ('https://docs.readthedocs.io/en/latest/', None), +# 'sphinx': ('http://www.sphinx-doc.org/en/stable/', None), +# } + +html_theme = "sphinx_rtd_theme" +html_theme_options = {"display_version": True} +html_static_path = ["_static"] +html_css_files = ["custom.css"] +# html_theme_path = ["../.."] +# html_logo = "demo/static/logo-wordmark-light.svg" +# html_show_sourcelink = True + +htmlhelp_basename = slug + +# latex_documents = [ +# ('index', '{0}.tex'.format(slug), project, author, 'manual'), +# ] + +man_pages = [("index", slug, project, [author], 1)] + +texinfo_documents = [ + ("index", slug, project, author, slug, project, "Miscellaneous"), +] + + +# Extensions to theme docs +def setup(app): + from sphinx.domains.python import PyField + from sphinx.util.docfields import Field + + app.connect("autodoc-process-docstring", remove_noqa) + app.add_object_type( + "confval", + "confval", + objname="configuration value", + indextemplate="pair: %s; configuration value", + doc_field_types=[ + PyField("type", label=("Type"), has_arg=False, names=("type",), bodyrolename="class"), + Field( + "default", + label=("Default"), + has_arg=False, + names=("default",), + ), + ], + ) diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..31c9129 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,47 @@ +.. include:: README.rst + +.. toctree:: + :maxdepth: 1 + :caption: Overview + + Overview/Abductive-Learning + Overview/Installation + +.. toctree:: + :maxdepth: 1 + :caption: Introduction to ABL-Package + + Intro/Basics + Intro/Quick-Start + Intro/Datasets + Intro/Learning + Intro/Reasoning + Intro/Evaluation + Intro/Bridge + +.. toctree:: + :maxdepth: 1 + :caption: Examples + + Examples/MNISTAdd + Examples/HWF + Examples/HED + Examples/Zoo + +.. toctree:: + :maxdepth: 1 + :caption: API + + API/abl.data + API/abl.learning + API/abl.reasoning + API/abl.bridge + API/abl.utils + +.. toctree:: + :maxdepth: 1 + :caption: References + + References + + diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..ce105e9 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,38 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=python -msphinx +) +set SPHINXOPTS= +set SPHINXBUILD=sphinx-build +set SOURCEDIR=. +set BUILDDIR=build +set SPHINXPROJ=ReadtheDocsSphinxTheme + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The Sphinx module was not found. Make sure you have Sphinx installed, + echo.then set the SPHINXBUILD environment variable to point to the full + echo.path of the 'sphinx-build' executable. Alternatively you may add the + echo.Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd \ No newline at end of file diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..88bedfb --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,5 @@ +sphinx +sphinx-rtd-theme +recommonmark +sphinx-markdown-tables +sphinx-copybutton \ No newline at end of file diff --git a/examples/hed/README.md b/examples/hed/README.md new file mode 100644 index 0000000..2cebd1c --- /dev/null +++ b/examples/hed/README.md @@ -0,0 +1,39 @@ +# Handwritten Equation Decipherment + +This notebook shows an implementation of [Handwritten Equation Decipherment](https://proceedings.neurips.cc/paper_files/paper/2019/file/9c19a2aa1d84e04b0bd4bc888792bd1e-Paper.pdf). In this task, the handwritten equations are given, which consist of sequential pictures of characters. The equations are generated with unknown operation rules from images of symbols ('0', '1', '+' and '='), and each equation is associated with a label indicating whether the equation is correct (i.e., positive) or not (i.e., negative). Also, we are given a knowledge base which involves the structure of the equations and a recursive definition of bit-wise operations. The task is to learn from a training set of above mentioned equations and then to predict labels of unseen equations. + +## Run + +```bash +pip install -r requirements.txt +python main.py +``` + +## Usage + +```bash +usage: main.py [-h] [--no-cuda] [--epochs EPOCHS] [--lr LR] + [--weight-decay WEIGHT_DECAY] [--batch-size BATCH_SIZE] + [--loops LOOPS] [--segment_size SEGMENT_SIZE] + [--save_interval SAVE_INTERVAL] [--max-revision MAX_REVISION] + [--require-more-revision REQUIRE_MORE_REVISION] + [--ground] [--max-err MAX_ERR] + +Handwritten Equation Decipherment example + +optional arguments: + -h, --help show this help message and exit + --no-cuda disables CUDA training + --epochs EPOCHS number of epochs in each learning loop iteration + (default : 1) + --lr LR base model learning rate (default : 0.001) + --weight-decay WEIGHT_DECAY + weight decay (default : 0.0001) + --batch-size BATCH_SIZE + base model batch size (default : 32) + --save_interval SAVE_INTERVAL + save interval (default : 1) + --max-revision MAX_REVISION + maximum revision in reasoner (default : 10) + +``` diff --git a/examples/hed/bridge.py b/examples/hed/bridge.py new file mode 100644 index 0000000..0706786 --- /dev/null +++ b/examples/hed/bridge.py @@ -0,0 +1,273 @@ +import os +from collections import defaultdict +from typing import Any, List, Optional, Tuple, Union + +import torch + +from abl.bridge import SimpleBridge +from abl.data.evaluation import BaseMetric +from abl.data.structures import ListData +from abl.learning import ABLModel, BasicNN +from abl.learning.torch_dataset import RegressionDataset +from abl.reasoning import Reasoner +from abl.utils import print_log + +from datasets import get_pretrain_data +from models.nn import SymbolNetAutoencoder +from utils import InfiniteSampler, gen_mappings + + +class HedBridge(SimpleBridge): + def __init__( + self, + model: ABLModel, + reasoner: Reasoner, + metric_list: BaseMetric, + ) -> None: + super().__init__(model, reasoner, metric_list) + + def pretrain(self, weights_dir): + if not os.path.exists(os.path.join(weights_dir, "pretrain_weights.pth")): + print_log("Pretrain Start", logger="current") + + cls_autoencoder = SymbolNetAutoencoder( + num_classes=len(self.reasoner.kb.pseudo_label_list) + ) + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + loss_fn = torch.nn.MSELoss() + optimizer = torch.optim.RMSprop( + cls_autoencoder.parameters(), lr=0.001, alpha=0.9, weight_decay=1e-6 + ) + + pretrain_model = BasicNN( + cls_autoencoder, + loss_fn, + optimizer, + device=device, + save_interval=1, + save_dir=weights_dir, + num_epochs=10, + ) + + pretrain_data_X, pretrain_data_Y = get_pretrain_data(["0", "1", "10", "11"]) + pretrain_data = RegressionDataset(pretrain_data_X, pretrain_data_Y) + pretrain_data_loader = torch.utils.data.DataLoader( + pretrain_data, batch_size=64, shuffle=True + ) + + pretrain_model.fit(pretrain_data_loader) + save_parma_dic = { + "model": cls_autoencoder.base_model.state_dict(), + } + + torch.save(save_parma_dic, os.path.join(weights_dir, "pretrain_weights.pth")) + + self.model.load(load_path=os.path.join(weights_dir, "pretrain_weights.pth")) + + def select_mapping_and_abduce(self, data_examples: ListData): + candidate_mappings = gen_mappings([0, 1, 2, 3], ["+", "=", 0, 1]) + mapping_score = [] + abduced_pseudo_label_list = [] + for _mapping in candidate_mappings: + self.reasoner.idx_to_label = _mapping + self.reasoner.label_to_idx = dict(zip(_mapping.values(), _mapping.keys())) + self.idx_to_pseudo_label(data_examples) + abduced_pseudo_label = self.reasoner.abduce(data_examples) + mapping_score.append(len(abduced_pseudo_label) - abduced_pseudo_label.count([])) + abduced_pseudo_label_list.append(abduced_pseudo_label) + + max_revisible_instances = max(mapping_score) + return_idx = mapping_score.index(max_revisible_instances) + self.reasoner.idx_to_label = candidate_mappings[return_idx] + self.reasoner.label_to_idx = dict( + zip(self.reasoner.idx_to_label.values(), self.reasoner.idx_to_label.keys()) + ) + self.idx_to_pseudo_label(data_examples) + data_examples.abduced_pseudo_label = abduced_pseudo_label_list[return_idx] + + return data_examples.abduced_pseudo_label + + def abduce_pseudo_label(self, data_examples: ListData): + self.reasoner.abduce(data_examples) + return data_examples.abduced_pseudo_label + + def check_training_impact(self, filtered_data_examples, data_examples): + character_accuracy = self.model.valid(filtered_data_examples) + revisible_ratio = len(filtered_data_examples.X) / len(data_examples.X) + log_string = ( + f"Revisible ratio is {revisible_ratio:.3f}, Character " + f"accuracy is {character_accuracy:.3f}" + ) + print_log(log_string, logger="current") + + if character_accuracy >= 0.95 and revisible_ratio >= 0.95: + return True + return False + + def check_rule_quality(self, rule, val_data, equation_len): + val_X_true = self.data_preprocess(val_data[1], equation_len) + val_X_false = self.data_preprocess(val_data[0], equation_len) + + true_ratio = self.calc_consistent_ratio(val_X_true, rule) + false_ratio = self.calc_consistent_ratio(val_X_false, rule) + + log_string = ( + f"True consistent ratio is {true_ratio:.3f}, False inconsistent ratio " + f"is {1 - false_ratio:.3f}" + ) + print_log(log_string, logger="current") + + if true_ratio > 0.9 and false_ratio < 0.05: + return True + return False + + def calc_consistent_ratio(self, data_examples, rule): + self.predict(data_examples) + pred_pseudo_label = self.idx_to_pseudo_label(data_examples) + consistent_num = sum( + [self.reasoner.kb.consist_rule(instance, rule) for instance in pred_pseudo_label] + ) + return consistent_num / len(data_examples.X) + + def get_rules_from_data(self, data_examples, samples_per_rule, samples_num): + rules = [] + sampler = InfiniteSampler(len(data_examples), batch_size=samples_per_rule) + + for _ in range(samples_num): + for select_idx in sampler: + sub_data_examples = data_examples[select_idx] + self.predict(sub_data_examples) + pred_pseudo_label = self.idx_to_pseudo_label(sub_data_examples) + consistent_instance = [] + for instance in pred_pseudo_label: + if self.reasoner.kb.logic_forward([instance]): + consistent_instance.append(instance) + + if len(consistent_instance) != 0: + rule = self.reasoner.abduce_rules(consistent_instance) + if rule is not None: + rules.append(rule) + break + + all_rule_dict = defaultdict(int) + for rule in rules: + for r in rule: + all_rule_dict[r] += 1 + rule_dict = {rule: cnt for rule, cnt in all_rule_dict.items() if cnt >= 5} + rules = self.select_rules(rule_dict) + + return rules + + @staticmethod + def filter_empty(data_examples: ListData): + consistent_dix = [ + i + for i in range(len(data_examples.abduced_pseudo_label)) + if len(data_examples.abduced_pseudo_label[i]) > 0 + ] + return data_examples[consistent_dix] + + @staticmethod + def select_rules(rule_dict): + add_nums_dict = {} + for r in list(rule_dict): + add_nums = str(r.split("]")[0].split("[")[1]) + str( + r.split("]")[1].split("[")[1] + ) # r = 'my_op([1], [0], [1, 0])' then add_nums = '10' + if add_nums in add_nums_dict: + old_r = add_nums_dict[add_nums] + if rule_dict[r] >= rule_dict[old_r]: + rule_dict.pop(old_r) + add_nums_dict[add_nums] = r + else: + rule_dict.pop(r) + else: + add_nums_dict[add_nums] = r + return list(rule_dict) + + def data_preprocess(self, data, equation_len) -> ListData: + data_examples = ListData() + data_examples.X = data[equation_len] + data[equation_len + 1] + data_examples.gt_pseudo_label = None + data_examples.Y = [None] * len(data_examples.X) + + return data_examples + + def train(self, train_data, val_data, segment_size=10, min_len=5, max_len=8, save_dir="./"): + for equation_len in range(min_len, max_len): + print_log( + f"============== equation_len: {equation_len}-{equation_len + 1} ================", + logger="current", + ) + + condition_num = 0 + data_examples = self.data_preprocess(train_data[1], equation_len) + sampler = InfiniteSampler(len(data_examples), batch_size=segment_size) + for seg_idx, select_idx in enumerate(sampler): + print_log( + f"Equation Len(train) [{equation_len}] Segment Index [{seg_idx + 1}]", + logger="current", + ) + sub_data_examples = data_examples[select_idx] + self.predict(sub_data_examples) + if equation_len == min_len: + self.select_mapping_and_abduce(sub_data_examples) + else: + self.idx_to_pseudo_label(sub_data_examples) + self.abduce_pseudo_label(sub_data_examples) + filtered_sub_data_examples = self.filter_empty(sub_data_examples) + self.pseudo_label_to_idx(filtered_sub_data_examples) + self.model.train(filtered_sub_data_examples) + + if self.check_training_impact(filtered_sub_data_examples, sub_data_examples): + condition_num += 1 + else: + condition_num = 0 + + if condition_num >= 5: + print_log("Now checking if we can go to next course", logger="current") + rules = self.get_rules_from_data( + data_examples, samples_per_rule=3, samples_num=50 + ) + print_log("Learned rules from data: " + str(rules), logger="current") + + seems_good = self.check_rule_quality(rules, val_data, equation_len) + if seems_good: + self.reasoner.kb.learned_rules.update( + {equation_len: rules, equation_len + 1: rules} + ) + self.model.save( + save_path=os.path.join(save_dir, f"eq_len_{equation_len}.pth") + ) + break + else: + if equation_len == min_len: + print_log( + "Learned mapping is: " + str(self.reasoner.idx_to_label), + logger="current", + ) + self.model.load( + load_path=os.path.join(save_dir, "pretrain_weights.pth") + ) + else: + self.model.load( + load_path=os.path.join(save_dir, f"eq_len_{equation_len - 1}.pth") + ) + condition_num = 0 + print_log("Reload Model and retrain", logger="current") + + def test( + self, + test_data: Union[ + ListData, Tuple[List[List[Any]], Optional[List[List[Any]]], Optional[List[Any]]] + ], + min_len=5, + max_len=8, + ) -> None: + for equation_len in range(min_len, max_len): + test_data_examples = self.data_preprocess(test_data[1], equation_len) + print_log(f"Test on true equations with length {equation_len}", logger="current") + self._valid(test_data_examples) + test_data_examples = self.data_preprocess(test_data[0], equation_len) + print_log(f"Test on false equations with length {equation_len}", logger="current") + self._valid(test_data_examples) diff --git a/examples/hed/consistency_metric.py b/examples/hed/consistency_metric.py new file mode 100644 index 0000000..48f0a5e --- /dev/null +++ b/examples/hed/consistency_metric.py @@ -0,0 +1,28 @@ +from typing import Optional + +from abl.data.evaluation.base_metric import BaseMetric +from abl.data.structures import ListData +from abl.reasoning import KBBase + + +class ConsistencyMetric(BaseMetric): + def __init__(self, kb: KBBase, prefix: Optional[str] = None) -> None: + super().__init__(prefix) + self.kb = kb + + def process(self, data_examples: ListData) -> None: + pred_pseudo_label = data_examples.pred_pseudo_label + learned_rules = self.kb.learned_rules + consistent_num = sum( + [ + self.kb.consist_rule(instance, learned_rules[len(instance)]) + for instance in pred_pseudo_label + ] + ) + self.results.append((consistent_num, len(pred_pseudo_label))) + + def compute_metrics(self) -> dict: + results = self.results + metrics = dict() + metrics["consistency"] = sum(t[0] for t in results) / sum(t[1] for t in results) + return metrics diff --git a/examples/hed/datasets/__init__.py b/examples/hed/datasets/__init__.py new file mode 100644 index 0000000..b07b583 --- /dev/null +++ b/examples/hed/datasets/__init__.py @@ -0,0 +1,3 @@ +from .get_dataset import get_dataset, get_pretrain_data, split_equation + +__all__ = ["get_dataset", "get_pretrain_data", "split_equation"] diff --git a/examples/hed/datasets/get_dataset.py b/examples/hed/datasets/get_dataset.py new file mode 100644 index 0000000..5d02b0c --- /dev/null +++ b/examples/hed/datasets/get_dataset.py @@ -0,0 +1,123 @@ +import os +import os.path as osp +import pickle +import random +import zipfile +from collections import defaultdict +from PIL import Image + +import gdown +import numpy as np +from torchvision.transforms import transforms + +CURRENT_DIR = os.path.abspath(os.path.dirname(__file__)) + + +def download_and_unzip(url, zip_file_name): + try: + gdown.download(url, zip_file_name) + with zipfile.ZipFile(zip_file_name, "r") as zip_ref: + zip_ref.extractall(CURRENT_DIR) + os.remove(zip_file_name) + except Exception as e: + if os.path.exists(zip_file_name): + os.remove(zip_file_name) + raise Exception( + f"An error occurred during download or unzip: {e}. Instead, you can download " + + f"the dataset from {url} and unzip it in 'examples/hed/datasets' folder" + ) + + +def get_pretrain_data(labels, image_size=(28, 28, 1)): + transform = transforms.Compose([transforms.ToTensor()]) + X = [] + img_dir = osp.join(CURRENT_DIR, "mnist_images") + for label in labels: + label_path = osp.join(img_dir, label) + img_path_list = os.listdir(label_path) + for img_path in img_path_list: + with Image.open(osp.join(label_path, img_path)) as img: + img = img.convert("L") + img = img.resize((image_size[1], image_size[0])) + img_array = np.array(img, dtype=np.float32) + normalized_img = (img_array - 127) / 128.0 + X.append(normalized_img) + + Y = [img.copy().reshape(image_size[0] * image_size[1] * image_size[2]) for img in X] + X = [transform(img[:, :, np.newaxis]) for img in X] + return X, Y + + +def divide_equations_by_len(equations, labels): + equations_by_len = {1: defaultdict(list), 0: defaultdict(list)} + for i, equation in enumerate(equations): + equations_by_len[labels[i]][len(equation)].append(equation) + return equations_by_len + + +def split_equation(equations_by_len, prop_train, prop_val): + """ + Split the equations in each length to training and validation data according to the proportion + """ + train_equations_by_len = {1: dict(), 0: dict()} + val_equations_by_len = {1: dict(), 0: dict()} + + for label in range(2): + for equation_len, equations in equations_by_len[label].items(): + random.shuffle(equations) + train_equations_by_len[label][equation_len] = equations[ + : len(equations) // (prop_train + prop_val) * prop_train + ] + val_equations_by_len[label][equation_len] = equations[ + len(equations) // (prop_train + prop_val) * prop_train : + ] + + return train_equations_by_len, val_equations_by_len + + +def get_dataset(dataset="mnist", train=True): + data_dir = CURRENT_DIR + "/mnist_images" + + if not os.path.exists(data_dir): + print("Dataset not exist, downloading it...") + url = "https://drive.google.com/u/0/uc?id=1XoJDjO3cNUdytqVgXUKOBe9dOcUBobom&export=download" + download_and_unzip(url, os.path.join(CURRENT_DIR, "HED.zip")) + print("Download and extraction complete.") + + if train: + file = os.path.join(data_dir, "expr_train.json") + else: + file = os.path.join(data_dir, "expr_test.json") + + if dataset == "mnist": + file = osp.join(CURRENT_DIR, "mnist_equation_data_train_len_26_test_len_26_sys_2_.pk") + elif dataset == "random": + file = osp.join(CURRENT_DIR, "random_equation_data_train_len_26_test_len_26_sys_2_.pk") + else: + raise ValueError("Undefined dataset") + + with open(file, "rb") as f: + img_dataset = pickle.load(f) + + X, Y = [], [] + if train: + positive = img_dataset["train:positive"] + negative = img_dataset["train:negative"] + else: + positive = img_dataset["test:positive"] + negative = img_dataset["test:negative"] + + for equation in positive: + equation = equation.astype(np.float32) + img_list = np.vsplit(equation, equation.shape[0]) + X.append(img_list) + Y.append(1) + + for equation in negative: + equation = equation.astype(np.float32) + img_list = np.vsplit(equation, equation.shape[0]) + X.append(img_list) + Y.append(0) + + equations_by_len = divide_equations_by_len(X, Y) + return equations_by_len diff --git a/examples/hed/hed.ipynb b/examples/hed/hed.ipynb new file mode 100644 index 0000000..10d2549 --- /dev/null +++ b/examples/hed/hed.ipynb @@ -0,0 +1,439 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Handwritten Equation Decipherment (HED)\n", + "\n", + "This notebook shows an implementation of [Handwritten Equation Decipherment](https://proceedings.neurips.cc/paper_files/paper/2019/file/9c19a2aa1d84e04b0bd4bc888792bd1e-Paper.pdf). In this task, the handwritten equations are given, which consist of sequential pictures of characters. The equations are generated with unknown operation rules from images of symbols ('0', '1', '+' and '='), and each equation is associated with a label indicating whether the equation is correct (i.e., positive) or not (i.e., negative). Also, we are given a knowledge base which involves the structure of the equations and a recursive definition of bit-wise operations. The task is to learn from a training set of above mentioned equations and then to predict labels of unseen equations. \n", + "\n", + "Intuitively, we first use a machine learning model (learning part) to obtain the pseudo-labels ('0', '1', '+' and '=') for the observed pictures. We then use the knowledge base (reasoning part) to perform abductive reasoning so as to yield ground hypotheses as possible explanations to the observed facts, suggesting some pseudo-labels to be revised. This process enables us to further update the machine learning model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Import necessary libraries and modules\n", + "import os.path as osp\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import torch\n", + "import torch.nn as nn\n", + "\n", + "from abl.learning import ABLModel, BasicNN\n", + "from abl.utils import ABLLogger, print_log\n", + "\n", + "from bridge import HedBridge\n", + "from consistency_metric import ConsistencyMetric\n", + "from datasets import get_dataset, split_equation\n", + "from models.nn import SymbolNet\n", + "from reasoning import HedKB, HedReasoner" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Working with Data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First, we get the datasets of handwritten equations:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "total_train_data = get_dataset(train=True)\n", + "train_data, val_data = split_equation(total_train_data, 3, 1)\n", + "test_data = get_dataset(train=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The dataset are shown below:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Equations in the dataset is organized by equation length, from 5 to 26\n", + "\n", + "For each euqation length, there are 225 true equation and 225 false equation in the training set\n", + "For each euqation length, there are 75 true equation and 75 false equation in the validation set\n", + "For each euqation length, there are 300 true equation and 300 false equation in the test set\n" + ] + } + ], + "source": [ + "true_train_equation = train_data[1]\n", + "false_train_equation = train_data[0]\n", + "print(\n", + " f\"Equations in the dataset is organized by equation length, \"\n", + " + f\"from {min(train_data[0].keys())} to {max(train_data[0].keys())}\"\n", + ")\n", + "print()\n", + "\n", + "true_train_equation_with_length_5 = true_train_equation[5]\n", + "false_train_equation_with_length_5 = false_train_equation[5]\n", + "print(\n", + " f\"For each euqation length, there are {len(true_train_equation_with_length_5)} \"\n", + " + f\"true equation and {len(false_train_equation_with_length_5)} false equation \"\n", + " + f\"in the training set\"\n", + ")\n", + "\n", + "true_val_equation = val_data[1]\n", + "false_val_equation = val_data[0]\n", + "true_val_equation_with_length_5 = true_val_equation[5]\n", + "false_val_equation_with_length_5 = false_val_equation[5]\n", + "print(\n", + " f\"For each euqation length, there are {len(true_val_equation_with_length_5)} \"\n", + " + f\"true equation and {len(false_val_equation_with_length_5)} false equation \"\n", + " + f\"in the validation set\"\n", + ")\n", + "\n", + "true_test_equation = test_data[1]\n", + "false_test_equation = test_data[0]\n", + "true_test_equation_with_length_5 = true_test_equation[5]\n", + "false_test_equation_with_length_5 = false_test_equation[5]\n", + "print(\n", + " f\"For each euqation length, there are {len(true_test_equation_with_length_5)} \"\n", + " + f\"true equation and {len(false_test_equation_with_length_5)} false equation \"\n", + " + f\"in the test set\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As illustrations, we show four equations in the training dataset:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "First true equation with length 5 in the training dataset:\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAABpCAYAAABF9zs7AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAKUklEQVR4nO3dWWwNfRjH8aF0Xxy0jVrelAvthSYoyoVaopGIKCKu0KIh1C6WC7HF0tQSERo3SCw3GhJLiCUVsaWSRhEau0g0SFQXSpW+V+/jad857Wk7M6fT8/1c/VpzzjyaM/X4/2f+/y4NDQ0NBgAACGhd/V0AAADwPxoCAABAQwAAAGgIAACAQUMAAAAMGgIAAGDQEAAAAIOGAAAAGIbRzdcDu3TpYmcdAcuKdaEqKiosqKR5YWFhks+ePSs5Oztb8rFjxyTPmDFDcm1trc3V2cPj8bTr9Vwz9rDimqmsrLSgEjQVExPTrtdzzdjD12uGEQIAAEBDAAAAaAgAAIBBQwAAAAwaAgAAYNAQAAAAg4YAAAAYNAQAAMCgIQAAAAYNAQAAMGgIAACA0Yq9DBDY9Brj9fX1kuvq6kyPAYBAceDAAcm5ubmSDx8+LHnZsmWO1tQWjBAAAAAaAgAA4PIpgy1btpjm1ho3bpzkoqIi02MCfTj8x48fkjMyMiRnZWVJ/vnzp+SuXek13WrMmDGS09PTJW/YsEFyVFSU5PXr10vOz8+3uTq0RlBQkGlui1+/fkm2Ygtqt9O/48LDwyXrn83o0aMdram9+K0NAABoCAAAQCeaMmgPPWUAc3/+/JEcGxsrOTk5WfLGjRslDx48WPKQIUMk62FHOENP62zfvl1yXl6eZD0NpHP37t1N31MPi+7YscP0XFOnTm30mtevX/tetM3c+jnUP3d9TWoRERGS58yZI7mkpERycHCwT+errq6WrO+knz59uuTa2lqf3quz0T/n7Oxs02MuXbrkVDmWYIQAAADQEAAAAJdPGVhl8+bNpt8fP368w5W4gx6q1EOYX79+lawXLILzevfuLXnp0qWSExISJOshYE0/UXPq1CnJmZmZkvVd1fru9aSkJMnFxcWN3nfx4sWSCwsLm63fboMGDfLr+VtDX2OhoaGS4+PjW3ytvg59nSaAb3z5+Q8cONCBSqzDCAEAAKAhAAAANAQAAMAI4HsIvK1IqN28edP+QjoRPfcc6Cs7+kNaWprk06dPS/7nn39Mjy8tLZX85s0bybdu3ZJ88OBByTNnzpS8atUqyaNGjTJ9f4/H0+jrsLAwr7U7bfjw4f4uwWd6BdCUlBTJBQUFkr9//97i+7T3mtSPagbqo4bavHnzWjzm8ePHDlRiHUYIAAAADQEAAAjgKQNvqxPyqCHcJC4uTrJehVBPE+jh5JcvX0qePHmy5E+fPrV4rjNnzkguLy+XfOHCBcnR0dFeX79kyRLJJ06caPF8drp27Zpfz99Wv3//lqwf82WKznlz585t8ZiOtDqnLxghAAAANAQAACDApgy8bYa0detWyTxZADeZPXu25AkTJkguKyuTvGDBAsmfP3+W7Ms0gTe3b9+W/Pz5c8mpqaleX9O3b982n89qNTU1/i6h3Zgm8K9+/fpJ1qtJvn//XvL169cdram9GCEAAAA0BAAAIACmDPQ0gbdNjJgmgFvExMQ0+nrFihWmx1VVVUm+f/++rTX5Sk9XAJ2VXkhKPwniBowQAAAAGgIAABAAUwbp6emm39cLELl9yqB79+6Oni8oKMjR8+GvadOmNfo6MTHR9LjCwkInyvnfuZp7yuDo0aNOlBPw9PXJtWqt5j7f/3nw4IEDldiDEQIAAEBDAAAAOumUgV4kwhu3TxNoTuy/oLc7nTVrluQePXpI1gul1NfXt5jReqGhoT4dV1JSYnMlf+kFWpqzcuVKyYcOHbKpmtbTvy/0Fr9uWvgnIiJC8po1aySfP39eshPbT7948cL2c/jTpEmTWjzmxo0bDlRiD0YIAAAADQEAAOhEUwbetjPW2Nq47fRQ9ZUrVyTrrXVjY2Mlz58/35nCbPT27Vt/l/A/mzZt8vpnr169kuzktqtpaWk+Hbdt2zabK/FdSEiI5Hv37kmeOHGi5KioKEdraq0/f/5I9ng8kvX0gRPTBIFE/zvTtevf/0/r3xXHjx93riCLMUIAAABoCAAAQCeaMigqKjL9vlVbGzedkuhITyl4+7vbJTIyUnJ+fr7k3bt3Sx46dKhkPYSJ1svNzZXc3BbC7969M81203fje8uGYRg7d+6UfOLECfsLa4Yebtefz5EjR0oODw93tKbW+vbtm+ScnBzJCxculKyfDkLb6MXtxo4dK1l/hvbu3etoTXZhhAAAANAQAAAAF04Z+LKdsWbV0H5HmiJoSi+m4vT59LCZfuJg165dkkeMGCG5rq7O5uo6t+YW3fJlQS476PN6y4bR+Nr1N/0ZTk5Olnz37l3J/vp5+kpPyXjbctfpxZU641MNKSkpkvW+MfrzUVlZ6WhNdmGEAAAA0BAAAAAXThm0dprAqicLOvKUQUehhyf13c01NTWSnZ7eaI/g4GB/l9AqV69edexcAwYMkNyrVy/TY5oOo5aVldlaU1vpaa/q6mo/VmINN+3B4AaZmZmm39eLf/n7qRmrMEIAAABoCAAAgAunDPTQvbf9C6zas4BpArhJRkaG5D179th6ruHDh0tOTEw0PWbdunWNvr5z546tNQFW0dfS6NGjTY/pSHtzWIURAgAAQEMAAABcOGXgbZqA4X3AXn369JG8du1a02OqqqokP3v2zPaaADusX79esrenjU6ePOlUOY5hhAAAANAQAAAAF04ZeKO3OQZgvUmTJklOS0szPUYvPuTGpwr0IkX19fWSWeyn8+vfv7/kpKQk02OOHj3qVDl+wQgBAACgIQAAAC6ZMvC2baqeJuApAwQ6vU2rzo8ePbLk/ePi4lo85vPnz5acy0nh4eGST58+LTkrK0tydHS0kyW5lpu3Ac7JyZEcHx9veszHjx+dKscvGCEAAAA0BAAAgIYAAAAYLrmHID093fT73DcA/BUbGyt54MCBkttzD0FCQoLk1atXmx7z9OlTyYsWLWrzufxFP2rYs2dPyampqZKjoqIcrQnO0BvhNd2M6z+lpaWS9+/fb3tN/sQIAQAAoCEAAAAdeMpAP2qoNzTS0wRMGSAQ6M9508e6YmJiTF9TUFAg+cOHD5KLi4tbPF9ISIhkvfJgRESEZL2K3+HDhyWXl5e3+P4dTW1trWQ9hDxlyhTJDQ0NjtYEZ4SGhkru1s38n8O8vDzJX758sb0mf2KEAAAA0BAAAIAOPGXgjR7SAwLBkydPJJ87d67Rn+nV9DS9quDFixclZ2RkSH748KHkyZMnS9Z7wetpAm358uWSjxw54qVyd9AbF/3+/VtydXW1P8pxNW9TWB3V5cuXJXubMggkjBAAAAAaAgAAYBhdGny8fZb9wO1hxd3LFRUVFlTiu8jISMn79u2TrDebunLliuRhw4ZJ/vXrl83VWcfj8bTr9Vwz9rDimnHzJjwdWXunDLhm7OHrNcMIAQAAoCEAAAA0BAAAwKAhAAAABg0BAAAwaAgAAIBBQwAAAAwaAgAAYLRiYSIAANB5MUIAAABoCAAAAA0BAAAwaAgAAIBBQwAAAAwaAgAAYNAQAAAAg4YAAAAYNAQAAMAwjH8BPPq6zcjr8zUAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "First true equation with length 8 in the training dataset:\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAABICAYAAACJB+2oAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAnoUlEQVR4nO2dWXCbV/33v3q075K1WJtlybvrpEnabE46KW3S0gVaBmiHMsAwA9PpDZQ7ZrjhhmUYYOCiZbgsMIX5E5YuQNN9S5rQLHbsOJbjRbZlSZatfd+f9yLvOchO3DiJbMnO+cx42rqPpfNs53zPbxXwPM+DwWAwGAzGHQ3X6AEwGAwGg8FoPEwQMBgMBoPBYIKAwWAwGAwGEwQMBoPBYDDABAGDwWAwGAwwQcBgMBgMBgNMEDAYDAaDwQATBAwGg8FgMACI1nugQCDYyHE0hPXWZAoGg3X9XqFQCLlcjg8//BA///nPEQgEsLCwgB/84Af4/ve/D4FAAKFQWNfvXI3Val33sXfyvb+Tz315eXmDR7L5mEymdR97J997du7bi/WeO7MQMBgMBoPBYIKAwWAwGAwGEwQMBoPBYDDABAGDwWAwGAwwQcBgMBgMBgNMEDAYDAaDwcBNpB02Ao7j8Pjjj6OzsxOpVArJZBJvv/024vF4o4fGuA1MJhPuu+8+FItFRCIRLC4uYnZ2ttHDaghisRhSqRQ9PT3o6emBVquFXC7HuXPnEAwG4ff7kc/nGz1MBmNDkEgkkEql0Ov1UCgUMBqNkEql8Pl8SKfTWFpaQrlcbvQw7xiaWhAIhUJ8+9vfxpNPPgmfz4e5uTkMDQ0xQbDFsdvtePbZZ5FKpTA2NoZPP/30jhUEZDK8//778dWvfhWdnZ0wGo34zW9+g1OnTiGRSDBBwNi2yOVyaLVadHd3w2az4a677oJOp8MHH3yAhYUFJBIJJgg2kaYWBASBQACdTodsNguz2YxoNIp4PI5qtdrooTFuArFYDIvFgra2NhiNRuj1ekilUoTDYahUKhSLRRSLxUYPc1Pp6enBQw89hP3796OtrQ1KpRIA0NfXh2q1ipGREYTD4QaPkrGZmM1maDQaPPTQQ1CpVHjppZe2VZEojuNgs9nQ1dWF9vZ2OJ1OuN1umEwmmEwmiMVilMtlzM3N4cqVK8hkMo0e8m0hEomgVCqhUqlgMplgNBphtVphNps/s1AWKZDE8zwymQympqYQiUQwNTWFZDK5IfNC0wsCUmFJo9GgUCigpaUFOp0OyWSSCYIthlgshtVqhdVqhV6vh0QigclkgsfjgVKpBM/zd5wgcLvdeOKJJ+BwOGCz2SAQCMDzPLq6uiCTyaBSqRo9RMYmIhAI0NLSAqfTiW9961swGAx4/fXXt40g4DgOQqEQNpsNBw8exMDAAHbs2IG2tjYYDAYAQKVSQalUQktLC/761782eMS3j0gkglarhdlsRn9/Pzo7O7Fr1y709fWht7f3un+zulri8vIy3nvvPUxOTqJYLGJhYeHOEgStra0wGAxQq9V0kgS2Z1nJO4VisYiZmRlotVoEg0GYTCa0trZCrVZDLBaD4+68GFeNRoOuri4oFAoIBAL6Y7FYIBaLIZfLGza2WxHcQqFww95RYjpe71xQW66V47gtM3e0tLSgtbUVYrF43SVnmx25XA6bzQaj0Yiuri7s3r0bDzzwAPR6PXQ6HbWMAVfvWzabRSqV2rKbPoFAALlcju7ubjidTjzyyCNoaWmB1WqFRqOh570Wq++7SqXCvn374HQ6YTAY8Mknn2Bubg6VSqWu16hpBYFOp4PdbqcTYq35ZKs+JHc65XIZkUgEy8vLSKfTdCKQy+UQiUR3pCCQyWRoaWm5ZrHSarXgOA5isbhBI8MtxS7IZDKIRPWdVgQCAarVKkqlEqrVKn3/17PAk4lVJpNtiedLIBBAqVRCp9NteD+TzUQikcBiscDlcuHgwYPYtWsX7r333jWPz+fzyOVyW04QEeEpFAqhVCrhdruxY8cOPPbYY9BoNFQEkPOqVqvgeZ7+XO+zBAIBpFIpXC4XdbMuLy9DJpOhUCjU1aratIJgz549OHLkCGw2G4CrF7BSqWBxcRFLS0uoVCoNHiHjZpFKpWhvb0dvby9sNhtdCJVKJex2OyqVChKJRKOHual4vV689tprK8yHxCLWaOH7/e9/f93HCoVCSKVSfP3rX0dPT09dd+QSiQSxWAx/+9vfEA6HEQwGUa1W12UhqFarUCgUeO655+BwOGA2m+sypo2C4zi0trbC5XJBJpM1ejh1Q6PRYN++fejr68PDDz8MrVa75rGVSgWTk5O4fPkyXViz2WzTiwOxWIxDhw7R+2cymXDw4EEYDAYYjUZIJBIAQDQaRSAQoCIgEAggGo0iFoshl8sBuPrM79mzBwaDAd3d3XRjoFQq0dHRgQceeADlchknT57E+++/X7dzaFpBYDab0dXVRX3LpVIJhUIB6XQa2Wy20cNj3AJCoRBqtRoqlQpyuZw+5DKZDAaDYdv4SW+GZDKJmZmZaxYqMvk1chI8efLkuo8lguDgwYN13d2S3dHy8jKGhobg9/sxNzeHarW6ru8ol8vQarX42te+1vRigOwGtVotjEYjBAIBisVi0y+EN4LcQ5vNhra2NrhcrhuKuUKhgEKhAJVKhXK5jEKhgEql0tTXQigUor29nVoFzGYz9u7dS4VApVJBJpNBOBzG3NwcFQQzMzNYXFzE4uIiDaCUy+XQaDTI5XKw2+1QKBQQi8UQiUTQaDRwOp3Yt28ffD4fJBIJyuVyXTYQTSsIyAWRSCSoVCo4c+YMpqamtnzE6Z1MPp+Hx+OBXC7HwsICqtUq9Ho9urq68NRTT+Hvf/87rly50uhhbipkESC+dzJR1sYTNIqurq51HScQCBAMBpFMJvHCCy/U3TzPcRzK5TJCoRDNRFGr1Whra4NAIFjzu4h7Qa1WQ6FQNL0JnuM4iEQi7Nu3D8eOHcPY2Bjm5ua29AZIKBRCo9HAYrGgt7cXdrv9hn8jkUjwzDPPIJlMYnR0FLOzs/j973+PaDSKVCrVlKJALBZDqVRicHAQO3bsQGdnJ13EgavP4tjYGP76179ienoaly5dogt4Pp9HqVRCqVSilm+O43D27FmYTCY888wzcLvdeOCBB6i4MJlM2LdvH5aXlxEKhTA+Pl6X1O2mEwQkClUqlUIul0MoFILneWoqLJVKjR4i4xapVqvIZDJIJpNIJBLUn6ZWq+FyuWj2QTabRaFQaOxgNwmJREKFby3NEADncrlueAwZJ9nRLS8vb0jeeK1oksvl0Ol0cDqdEIlEn3mtKpUKFAoFjVNpdoiFwGQyIR6PIxAIbNk5j+M4SKVStLa2wmq1wmg0Qq1Wg+d5FAqFFTECpEARuZ8Oh4PGjCiVSupCCYVC1/xtMyCTyaBWq9Ha2krdoUQMlEolxGIxzM/PY3h4GF6vFx6P54afGYlEEI1GMT4+jmq1in379kGpVEIqlUIikUCv18NsNsPlcsHn89XlPJruDdFqtTTtpq2tDTKZDNVqFaFQaEu/HIz/kclkcPnyZXAch507d8JsNkOn0yGfz8NqteK1117DuXPnGj3MTaGjowNf+cpXIJFIVkTP1wYbNYqf/vSnNzyGLNKvvvoqhoaGkE6nUSqVNsy6QSwC/f39eOaZZyAWi2/4PQKBYEtYCGqpVCoYGRnB0NDQlrQQcBxHRdvzzz8Pl8uFu+++G0KhEPl8HhcuXMA777yDYrGISqWCwcFB9PX1wW63Q61WA7i66+7u7obD4YDJZEIwGMTJkycxNTWFN998c8WOupFwHIe77roLbrcbXV1dsNvtK4KBvV4vfve732FychKnTp1a92anWq0iGo3i5ZdfRm9vL1387733XnAcB47jaDzB7OwsLl26dNvn0nSCwGAwoKurCwaDAVKplOamR6NRhMPhpngAGLdHoVBAIBCA1WpFpVKBSCSCQqGAxWJBX18fPvroo0YPcdOQSCTXRB43CxaL5YbHEEHgdruRzWaRzWZRLpc31MIhFArR2dkJq9V6QwsBYSv44pVKJbUWVatVJJNJxGKxLVmpTygUoqWlBRaLBR0dHdQPns1mEQwG4fV6MT4+Thd1rVaLSqWCcrkMo9EIjUYDsVgMiUQCoVAIl8sFpVKJaDSKYrFIBXSj1wMyRofDgc7OTppCDYDGviUSCUxNTdFyzDfzHFYqFcRiMSwtLSEQCFALC3nm1Wo1rFbrirTN26HpBMHnPvc5fOc734HL5YJUKkUsFkM0GsWZM2dw4cIFpNPpRg+RcZtEIhGcOHEClUoFX/ziFyGTySCXy2nlsjfeeKPRQ9w0Vkfjk3+vTTlqFOtNOxQIBDh06BAGBwc3eEQrqVQqN7UgNLsg2LlzJwYGBmA0GlEoFLC0tITFxcUtKQhkMhnuv/9+9Pf3Y+fOndBoNOA4DtPT0/jDH/6A0dFRnDp1CsDV+/LRRx9BJpPh4YcfRm9vL44dOwabzUbrMZhMJhgMBnR0dMBsNuPf//434vF4w12LpLril770Jezfv39F4GqpVILP58Pk5CTOnz+PZDJ5y89gMpnE6dOnUalUcOzYMfp7i8VC67nUg6YTBKsVTzqdRjQaRSKRQDqdbngqVrNTG5QG3FpxmY2mXC4jkUhgeXkZPp8PJpOJ5q+TnzudZli8VgfrrZUvzfM83cmRv9no565arTZ8Mag3RqMRbrcb5XIZS0tLSCQSyGQyTfkOfxYCgQAikQg6nQ46nQ5isRilUgl+vx8zMzOYnJxEMBikKXYAaAbZ9PQ0SqUSDAYDlpaWsHv3bmg0GigUCnAcRxsg9fb2Ym5uDtFotIFnelUQuN1uWK1WtLS0rJi7isUi5ufnEQgEkMvlbsvdTWq4rA6qFAqFK96726XpZl6NRgOHwwHg6kQzPz+PqakphMNhpFKpBo+uueF5ngZlkt0lMck1Q5AaoVQqIRKJYGJiAv/617+wd+/edZmntyMkVgBYKQLWWnw3k9oqiSTlq1KpoFgsXvM8kShpxq3T39+PBx98EIuLi/B4PPB6vVhaWmoKcXgzkAqbRqMRBoMBIpEIS0tLOH78OIaHh/Huu+9e86zwPI9yuYwzZ87g3LlzOH36NCwWC55//nm43W709PRALpdDJpPB5XLhueeew1tvvYXLly836CyvCp977rkHn/vc59Dd3Q29Xr/ivUgmkzhx4gQ8Hg9yudxtuTfy+TxmZ2fR1dW1oc9D0wgCknJDKtaRyTCRSGBpaYl1fKvheqZk8rtMJoNUKoVyuUz9cUqlcsXC0wyQ2JBYLIZsNksLzZAI4x07dmB+fh7JZLLRQ20IxDLWSHPxyZMnaXEfiUQCtVoNtVoNs9l8TeGkZnq2thoymYzW4jAYDJiYmIDP50M+n99y15XUHFAoFHA6nbDb7RAKhahUKshmszTFbq3FkZTiJQXKPvnkEwSDwRXN0EjmQmtrK8xmMzKZzKano2s0Gmg0GrjdbrjdbiiVyhXVdIvFIuLxOKanp7GwsHDb95GI8Y22FjWNIJBIJFCpVLQ6F6nW5vf7ceXKFSYI/j/Xy1kHrpqORCIRfD4fxsbGEIvFkMlk8NBDD6GlpYUW9mgmCoUCIpEIkskkisUixGIxhEIh9u/fD7FYjJdffhljY2ONHuaGcr0YAp7nsbi4CJ/Pt8Ksutn84he/oCWDW1pacPfdd2Pnzp147LHHqKWAcfvodDpYLBY4nU44HA4EAgGcP39+S1pEBQIBTb8bHBykRYiq1Spyudy6CgzxPI9YLIZYLIYXX3wRLpcLO3bsgNvtpuXO3W43uru7MTAwgLm5OczMzGziWQIOhwO9vb3Yv38/9u/fv8JVUK1WEY/Hsbi4iDNnzmB5ebnp5t61aBpBYDabcdddd1HTMbEQLC4uwuv1bhtBUFtwhvh9ruf/Wf0AkTSTbDaLoaEhms9PIk6JIPD7/Zifn4dOp4NarV5XiddGkUql4PF40NXVRe+vUCiEyWRCZ2cnzGYzFhYWkE6nt8wLdavUug1I9bLx8fGGFuLy+/0Arpori8Ui7HY7Hc+NSiuvDo5czw6p2axYm4VMJqO7XwBbes4jhZX6+/uhVCpRLBYxOzsLj8eD4eHhm86XJ9kW7733Hnp6eqDRaKgFQqVS0ViDzUKr1UKv11Mh0NbWdt2GXuT9uNnA17WordZLNgkb0eekaQRBW1sbHn74YXR2dq6YHL1eL0ZHR7d8dkFtsB8RAKt3+eS/a33H5P+T9Lx0Oo3jx49jYWEB09PT4Hmepl5xHIdcLodsNovDhw9j165dqFarTdvUJRqN4tNPP0Vvby+SySQ4joNMJoPD4YBSqUR7ezt8Ph+KxWJDd8obyermJhzHoVqt4tKlSzh16hTi8XjDxjY/P49qtYp8Po98Pg+n00lFaG2ToVpqn1siYsl/r0Xt+95s8S6bgUqlgsVigVwuR7VarVtOeSOQSCR49NFHcc8990Cj0SCTyeDTTz/FxYsX8cEHH6BQKNy06EskEnj55Zdx9913Y9euXTCbzXA4HLQr5MLCwgadzbUYjUbs3LkTn//85/GFL3wBUqn0uvMrea/rJXJrizkRy9G2FAREHff09ODgwYO0HOmFCxcwOTmJmZkZ6mNuJmp3+uuZwAqFAjWbcRwHuVxOff1kF59KpeD3+3H27FlMTU1d86CpVCrkcjn4fD4kk8kVYkGhUECn06Gzs5OWCDWbzdBqtbf0Em42tdeS+OM0Gg1UKtWWKihzs9S6DFb/s9GQ6GW5XI5yuYyZmRlada1arX5mfIPBYIBWq8WRI0egUCjW3CWRev3FYhH//e9/MTk5uVGnAwD42c9+tqGffyuYTCb09/ejUCjA4/FsWVeB1WqF2WyGzWaDwWCAQCBAOp3G2bNnceXKlVuut0+anoVCIXg8HmqtagROpxNHjx5FZ2cnJBLJijma53nE43FEo1EcP34ck5OTdSsqVSuaN9KS1hSCwGq1ore3FwcOHKCT4cWLF/HOO+9gbm6uaXeHq3dBax1Ddn8kiI6UX02lUrQ8J8dxSKVSCAQCeP311/HWW29RUxRxC+j1+hVpeWQ3WS6XIZFIYDQaceTIETz99NPUVMVxXNP7eleLK9L8SKvVbntBQKi9z7W/a6Q4IM+ZWCymgmB6ehoffPDBZ/6dUChER0cH2traMDAwAI7j1jR/cxxHg8JOnDiBN998s96nsYJmFwQTExNbVhDYbDaaglcrCM6fPw+fz3fLAbLEbbC0tASPxwOxWIz9+/fX+QzWBxEEJpPpmvRoEvswOzuLP/3pT5ifn6/72rXRmUcNFwSrISebSCSwuLjYlH40ks5HenZnMpnr3iQiGCqVCoLBIOLxOEZHR1GpVCCTyeDz+RAOh+nfT05O4j//+Q+y2SxcLhcVGsViEVKpFEeOHIHZbIbb7YZQKES5XKZmVrlcTs3sRIXXswXtRkIe8tXlenft2gWBQACfz7ft2iKbzWYMDAzQlsdr5fc30rJDqvsRt5RcLqdC7Xo7vWq1iqWlJeRyOSwsLCAej+OXv/wlLT+++lxqU2NLpRKmpqZohTuFQtF0VsF6o1AooFar0d3djd27d+PEiRO4cOECQqFQo4d2S2i1WhgMBqhUKgDAqVOnMDExAb/fj2g02vRWyvVAKqqubk1dLpeRy+XwzjvvYHx8nFZUrNc5q9Vq7N27Fzt27IBOp9uw1tgNFwRk0Vy9Q8pkMojFYk2X20zGWy6XkUwmEY/HEQ6HAeC6OzyycM/MzGB5eRmnTp1CuVyGVCpFKpWiVgIAtM+7SCSiJjGe55HP56FQKHDgwAG4XC7aUrN251+7qJKJdCuIAcLqoDoAcLvd1GKw3dDpdNi9ezfsdnvTCQECMfOTVsMk9dBqtV437ZBEV5N3NxqNwuv1rutceJ6nliHibmiW67BRSKVS6PV6WCwWuN1uxONxDA0NNTRu5FYh/SI0Gg3kcjkEAgHGx8dx+fJluunZDshksuuWGi+Xy8hmsxgeHsbFixepO7heyOVy9PT0oL29HUqlklpN6+1CaLggsNvtePLJJ+lukJi6iV+x2aLLSeDb3Nwc3nnnHfh8Pni9XhgMBuh0umtuTqVSgVAohNlsht1ux49//GPIZDIIhUIMDw/jpZdeQiQSwdLSEu6//3489dRTtLhQ7WeIRCJYrVbqz71e6s52mjyJiySfz29I8EyjUalU6Orqgtlsvq6boBnE3ODgIC0Yo1ar0dHRAbfbjQMHDtC68wTiunrllVcwMzND39v1ZhdUq1UYjUZotVocPXoULperbj3em5XW1lYcOHAALS0tiEQiCIfDCIfDTe/iawQki0omk13TGbTRVKtVnDt3DjMzMzh//jxmZmbqfg8lEglsNhuMRuOKuWFubg5TU1N1C6xsmCAgO22dToeBgQFYrVYAVxe/QqFABUGzLXK10fzz8/OYnZ3FlStX4HA46EJdK2JKpRLEYjEtbXnkyBGq8Hiep5G4PM+jra0Nhw8fpsWZVkMEEhEDzbBo1IO1VK5cLodKpdqWpYxJ+1KFQrHi9yRVaa0o/s2EVEUrlUrQ6XTo7e1Fb28vDh06tEIQkPtXKpUwOTlJY1vW++6SczWbzdDr9di3bx96enqa8v2vJyqVinZ0TafTNJZiK/Yu2EgEAgEkEgntedJsgoDneQSDQczMzCAUCtW9nDI5f71ef421NBaLYWpqqm5WpYbNtBqNBjt27MDhw4cxODhIexfMz89jenoaHo8Hfr+/6dQy2QlpNBqaBiMQCOB0OmGz2a6pJkUWbqJs8/k8KpUKpFLpCh8tcFUM5XK5NQVB7eduFzEAXK3Kt7CwAKFQCK1W2+jhbAqryxWT5yAYDCIcDmN8fBxXrlxpaOvb733ve3RsJNtAKpXS1MPrWageffRRPPjgg7e0kItEIvoMJBKJDREDteWYGwnHcbBarThw4AB4nsfIyAj8fj9SqVTTWUUbjVqtxoMPPoj+/n488cQT0Ov1DUulJoJ9tRVPJBLRwmqkMmM9EIlEMJlM6OjowKFDh2Aymei58/zV0v4ff/wxAoFAfb6vLp9yC0gkEpjNZpjNZhiNRnqS0WiU9i5o1uyCSqUCiUSC1tZWKJVKqNVqOBwOtLa2rllekpg/SUMWkUhEgwLJ8WRXVS6X74jIekImk0EgEIBWq4XFYqELA7HGqFQqaDQaZLPZG1Y520qsFgUAkM1mEYvFEI/HkUgkGro4dHV1rSjHSqwC1+tlAPwv9exmn93a7yD51tt5USQBmnq9HjabDX6/H4uLi3X3O28HSClkt9uNzs5OOBwOiMViFAoFFAoFZLPZTY0zKxaLSKVSkMlktJAUiZ9Qq9XQarWIx+NIJpN1sfCJRCIYDAa6Vmo0GgD/25jG43FavK0eNEwQyGQyOJ1OmEymFQV5Tp48id/+9rdNG1hTrVZRLBah0+mwb98+OpmJRKIb7mq2y0JWb4aHh/HrX/8aTzzxBCqVCpxOJ4xGIxQKBUwmE77xjW/g4MGDOH78OJaWlm66p3gzslYdi0wmg3A4jOXlZYTD4YYujMQ6UVtp8LNcVWQxv1nI54nFYohEIkil0oa7SzYSm82Gw4cP48iRI+jv78fk5CROnjyJxcXFRg+tqSABxa2trXjwwQfhcrkgEokQj8dx+fJlnDp1Ch999BFisdimjcnj8eDll1/GoUOHsHv3bgBXrT2HDh3Czp07IRQKMTExgT//+c+3PS4iBp5++mn09vZCrVbTeKp0Ok3rMgwNDdWt82fDBIFYLEZLSwvUavWK3yeTSVoytVnheR5CoRAymWxFZH+5XN5WpvzNIpVK0a6WHo8HGo2G5vmS3QHHcVAqlbQq41YXBGtBduHkp5Gs5aJaT9XBm/0enueRSqVQKBQ29B3q7u7esM9eL8Q6qtfrIZPJkMvlEAgEGuoeajaUSiWkUimtSmixWKDT6ZDL5RCJRDA5OQmfz4doNLqpluREIoHp6Wn09vZSSy7p3yCVStHe3o5CoQCtVksrfN7KO0Hi60h7ZYfDQQuF8TyPXC6HUChEe9bUi4bGEAwODsLhcFyzA2l2SLOO1Wl/TAzcGiSI9JVXXsGbb76JX/3qV+jp6YFUKoVEIsHBgwfhdrvxpz/9CcvLy9uiA2JtueLVcQTN9g4QK0a93Vjkc3O5HHK5HI4fP47Tp09vaP2M48ePb8jn3gwymQwWiwUKhQKFQgFzc3M4e/ZsU9ZcaQQikYhWrb3vvvtgt9vR19eHUqmE0dFRnDt3Di+88AJisdiGxZqshdfrRSaToWOqDQwWiUTo7++HRqPB3r17MTs7i4sXL96SS0Mmk+GRRx5Bb28vjh49Cp1OR60DAoEACwsLOHHiRN0re266ICA+Yb1eD6PRSC0EpVIJuVyu6YIIP4tmm7i3KmQRTKfTSKfTVPGTRUEmk0GlUsFmsyEWiyESiWwLH3NtqWLyLJGGLQaDAS0tLUgmkw2zFJCiWvl8HtlsFqFQiAYY1gNy/qTA1+TkJLxe73WbxWwnSFpxsVhEOBxGIpFALpfbVm4SjuNgMplgsVjQ0tJCqxbeCFL8yul0oqOjA52dnWhpaUE+n0csFsPo6CgmJiYQCoWQz+c3/Zrl83lEo1HMzMzg4sWLNENOLBZTF4fBYEB/fz+kUinm5uZWzGnrgRT/6ujoQEdHx4ouwIVCAYlEAn6/HzMzM7QGTr3YdEGgUChw9913Y/fu3ejq6oJCoQDHcYhGo5ient7UzlWM5qS2yBJBKpXi4YcfRltbG7xe75YSjtdjrf4F3d3dcLlcOH/+PDiOw+nTp+uexrReRCIRisUifD4fRkZG8OKLL6JardY97YsELIbDYdrkSiAQUPfQdiUQCCCdTmN2dnZbCNxaFAoFjh49io6ODly8eBFerxfDw8Of2dNCIBCgra0NFosFx44dQ19fH3p6elCtVjE8PIyxsTH8+te/RiwWo5kum00ul0M+n8err76K06dP4/nnn8exY8dgMBioi6OlpQXPPvssZmZmEAgE4Pf7MTU1tS7xwnEc2tra4HQ68cgjj6Cjo2NFZszy8jJOnjyJDz/8EK+//nrd58FNFwRisRh2ux2tra0r0utisRg8Hk/dFQ9j65FMJrG4uAi1Wk2VMcmHJ5kZ2xVilm9ra0N3dzcuXrzYMEFAIIXCUqkUeJ7fkDxwIgBFIhH0ej3kcjncbjctYbyd7rlEIoFOp4NIJKK9TLY6PM8jmUwiEonQWBCZTAaDwYA9e/bAYDBQa1OxWEQikUAkEoFUKqXHaTQa7Nq1Cw6HAy6Xiy6y6XQaExMTuHLlCvWZN/J5IPEuADAyMgKVSoV7770Xra2tkEgkEIlENGPq8OHD8Pv91MqRz+dRKpVW1NgoFosol8vQ6XRQqVS455570NbWBpPJRBu9kWsXDocxOjqK+fn5W45P+Cw2XRDI5XLs2bMHfX19NEACAKanp/GPf/xjw7udMZofr9eLs2fP4p577oHFYgHwvzr5oVBoW+ym1oohAK7ulu69916YzWa89957m9retVGQ1C2FQoE9e/bA6XTim9/8JpxOZ0NMwxuJSqVCZ2cnstksIpFIo4dTF3iex9zcHEqlEgKBAHQ6HSwWCxwOB7773e8iFAph165diMfjCAaDGBkZwYcffkhLN993330YGBjA0aNHaRAxsRokk0n885//hNfrbXgqLiGVSiGdTuP//u//8P777+NHP/oRDhw4ALPZDIlEAqVSiY6ODvzwhz9ENBqlwt7n8yEej2NpaYm+++FwGOl0Gnv37kV7ezsef/xx2Gw2yOVyumHO5/OIRCIYHx/H8ePHEY/HN+SdaIiFwGKx0G5YxWIR6XQai4uLmJ+f33ZNbBg3TyQSwezsLPr6+ujvyuUyAoEAFhYWmmJCuF1q0w1rYwhIhc6JiQl4PJ6GBlCSLpw6nQ5dXV348pe/TBsdbQTkvjqdTtrPgGSabDcLgdFoxOLiIpLJZN1SxhoJ2TWHQiF89NFHCAaDOHr0KDQaDRQKBYxGIwYGBpDNZuF2u2G329He3g6dTgedToe+vj44HA7o9XoIhUKEw2GkUikMDw9jdnaWLqTNJAxJ3JNAIMDIyAgEAgEOHDhALVwcx0EqlUKj0cDlcsFkMsFkMiGbzdL3uja7xuVywWg0Qq/XQyqVrnCXka6Rly5dojEnG8GmCwKpVEpbowqFQqTTaZpyNjY2ttnDYTQhxGc9ODhIf1cul+HxeDAxMdF0Da9uh9UxBGSy+PDDD3Hq1KmG7iDL5TKtqGez2XDo0KFN+d7aGBKSyrudYgkUCgUcDgdtjLZd0g0jkQhisRheeukltLW1ob29nabMqVSqFU2xSqUSCoUCDbAkpnbgqjD0er3wer34yU9+gvn5eWSz2aZ0HSWTSaTTabz99tuYmpqirnCpVEqrFmo0GgwMDNzS55PzjUQiePXVVzE9PY1IJLJhwmjTBIFIJILD4UBnZydMJhOtuJRKpXDlypUt2/KTUX9I9a1QKIRIJIJkMolQKIRkMrkhfrNmgCyCQ0NDGB0dxcjICILBYNMET64O8tzo79rOEH9wNBrF3Nxc0xZhuxXIjjcYDOKNN96A3W7HXXfdBZPJhJ07d4LjOHAcRwNGyc/8/DzC4TDi8TjS6TSGhobg9/sRDoepy6hZnwue5xEIBGiwYW1KYldXF2Qy2TU9Sz6LbDaLXC6H6elpJBIJ+Hw+LCwsYGxsDOFweEOvw6YJArFYjJ6eHvT29qK1tZW2N00kEhgbG0MwGNysoTCanEgkAp7nEQqFsLS0hIWFBQQCASQSiW0jCFbHEJCfTz/9FK+++ioNoGoGanttMG6fcrlMA8Smp6eb5j7XAxJcmMvl8I9//ANmsxmDg4Po7+9Hf38/JBIJzSCpdT15vV6MjIxgenoa4XAYw8PD1OLQ7M8dz/Pw+Xzw+/2IRCK0iR1xiRAXwnqsXERQxWIxfPzxx7RXQSwWg9/v3/BrsWmCoFQqwePxYGlpCblcjkYqh8NhTE5OsnRDBiWdTqNcLuMvf/kL3n//faRSKWQyGYRCoW3TAW9ubg5//OMfodfrYTKZAFydDC5cuIDZ2dltY0ZmXEs0GsWZM2ewsLAAuVy+Ldt7kzTSTCaDfD6P4eFhDA8Pr1nDwufz0aJj2WwW4XAYhUKhqWIGbgTP84jH48jlcvj4448xMjKCqakpqNVqtLa2QiaTQaPRQKvVwmAwIJfL0etTLBapkCLW0PHxcaRSKfj9/k27FpsmCMrlMubn5zE/P4/h4eHN+lrGFoRUrnv33XcbPZQNY3FxkdWuv0NJJBK4dOkSotEoJBJJwzr3bSTVahXxeJxmFQDY1u8z8L8gw3Q6jUgkAqFQiNHRUSgUClgsFmg0GlgsFtjtdnR2diIcDiMajdINDwkyDQaDyGazDel8uf0azTMYDEYTEwgE8Nprr9FufSSnnbG9qFar1AKQy+UgFovh9Xpp5VWSUUTqq5AaBblcjnbH3WyYIGAwGIxNJJlMsoyqOwCe52lQ8GY2YLodtp+tisFgMBgMxk3DBAGDwWAwGAwmCBgMBoPBYDBBwGAwGAwGA4CA3w5J3QwGg8FgMG4LZiFgMBgMBoPBBAGDwWAwGAwmCBgMBoPBYIAJAgaDwWAwGGCCgMFgMBgMBpggYDAYDAaDASYIGAwGg8FggAkCBoPBYDAYYIKAwWAwGAwGgP8H2HdP4cOznIIAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "First false equation with length 5 in the training dataset:\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAABpCAYAAABF9zs7AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAN8ElEQVR4nO3daWxU1R/G8YOIBQENUjGAAipbEMEAVkpilACGoCZVEK0iL5RNjVI0bkQUYyJUY2TTxDSoURJ2ColEUeMWWVTciBgoLhAQCNQlLLJU5P+Kn8+9/7mdO517Z/1+Xj0zvbdzGHpnTs7vnnOanT59+rQDAABF7axsNwAAAGQfHQIAAECHAAAA0CEAAACODgEAAHB0CAAAgKNDAAAAHB0CAADgnDs77IHNmjWLsx1FK4p1oerr6yNoSWaUlJRY/vfffy03NDRkozmNKi0tTet8rpl4RHHNHDhwIIKWwK9Dhw5pnc81E4+w1wwjBAAAgA4BAABIoWQANJWWCTZv3my5VatWlnv16mX51KlTmWkYAMAwQgAAAOgQAAAASgaIyVln/dfXXLt2reUJEyZY7tu3r+VPP/3U8uHDh2NuHQDAjxECAABAhwAAAFAyQEzOPvu/P60ZM2ZYvvDCCy1PmjTJ8okTJzLTMABAQowQAAAAOgQAAKDISgaXXHKJ5SVLllieO3eu5alTp1ouLy+37F8LWtfcHjt2rOUVK1ZE09gC0rp1a8v6vo0fP97yn3/+mfAY5JehQ4danjhxoudndXV1lvX627ZtW/wNA5AUIwQAAIAOAQAAKLKSgQ5TlpWVWV68eLFl3ZJXywT6vHPObdq0KWFG4/Q9/fvvvy1TJginXbt2nsddunSxPGbMGMvV1dWWjxw5Enk7dJvb6667zvIrr7xiuX379oHn79692zIlAxSqzp07W77nnnssDxo0KOHxNTU1nsfffPON5b1790bcuv/HCAEAAKBDAAAACrRkoHf966yBIUOGWNYSgK67r3QY23/Mvn37LO/Zs6fpjQWS0L/ne++91/Oz4cOHW66trbV88uTJyNuh21WvXLnSsl5Xjdm1a5fljRs3RtcwIMt0Ro2WCfR67dSpU9Lfc9NNN3keb9261fKNN95oWUtuUWKEAAAA0CEAAAB5XjIYPHiw5aqqKsu33XabZS0NBGUVVErwHx90PhAFXURL94Lo06eP57hDhw5ZXrhwoeV0Sgb9+vWzXFlZaVmvsXPOOSfl33vrrbda/vHHH5vWOCDDdF8WnR2gZbOOHTta9i9ilw693rX88Nxzz1k+depUZK/HCAEAAKBDAAAA6BAAAACX5/cQaJ1V7xsImkaoz2/YsMHytddea3nZsmWWdeU3/+/UFeIuvvhiy0xBzIyWLVtaDvr/zmc6rUj/Jp955hnPcb///rvlL774IqXX6N69u2WdnqvTm7p27ZrS71S6YZVz3pUpkVv0GtJrS504ccLyP//8Y7kQVxnt1q2bZV19c+TIkUnP1ft6li9fblmn2k6bNi3hueeff77nsX636L1Eb775puWdO3cmbVNYhfdJCgAAUkaHAAAA5F/JQIdadEg/zDRCLRPodKqg4xvb3Oiaa65JmCkZpE/fd10dT4cyV69ebfno0aOe86MqIdx9992R/J506RSj6dOne3526aWXWl6xYoXlJ5980rJeJ2rKlCmWzz333KTt0OFPLdEpHUr2H1NXV5f0NTIlzL+3kOlUOuecq6+vt7x27dqE51x11VWWdSg7ymlvuWLmzJmWw5QJdBMinY44a9ashMfrtaT8G4J9+OGHli+77DLLq1atsjxgwICk7QuLEQIAAECHAAAA5GHJoLy83HKYzYpuv/12yzqkGkTPbWxzI/1ZId5l2xQ61K9DskHvz3nnnWdZ72AuKSmxrENjX375peVXX33V8uHDhz2/N6r/j1wpGaiKigrP44ceesiyDm3qUGOY4fHPP//csm7c9cQTT1ju3bu35aCSwQMPPGD5448/Tvq62aJ3jhcj/zWi5ZxFixZZ1uuypqbG8uWXX245n2eP6GyCNWvWWL7iiisSHq/lkcmTJ1teunSp5TDvx5EjRxI+7y/lBG2IpJ+FUWKEAAAA0CEAAAA5XDLQ2QRaJigrK7MctFmRzibYtGlTSq8bdpaBlhCi3MwiXc2bN7esd+hngs4CmD17tuU//vjDcosWLSxPmjTJsg6h6cY5H3zwgWUt/2jJIJfe/7itW7fO81iH+vU6mTdvnmWdhaGzD9T3339v+eDBg5b1b2j+/PkJzz1+/LhlXSgpl+nfZ7HQ68S/AI7eVX/99ddb1iFynWWgpYR8o2UC/XzRu/jVb7/9ZlkXBnvjjTcib9t9993neRy0SNQ777wT+Ws7xwgBAABwdAgAAIDLcslg8ODBnse6N4EuphI0m0CHcsaOHWs51TKBCpo9kMuzDFq3bm1569atlh977DHLQUNPTaFDj7oQjZYrtmzZYrmhocGyvlfffvut5erqast6l+4dd9xheeDAgZbbtm1rOWhRqmKg5QCdWaD7qKdD9/kYNWpUwmP0jufa2tpIXjduOnulWOh1q2U557zlAD1Or1ctE+TzYkQTJ060HFQmUM8++6zlOMoEavTo0aGOO3bsWCyvzwgBAACgQwAAALJcMqiqqvI81sVOgmYQaI6qTKClC92XIF9mGeiwrg4h6/rk/iHCVOkQlS6Ao8Np+tq6wIa+P3ruRRddZFnvbPbvTXCGlh6CFvaIUrGvd++cc/3790/4/M8//2xZ90TIFzpErmWvYuH/zPrrr78sF9p24uPGjfM81pJBEJ3FpNuPx+Hpp5+2HHS9Oefcr7/+almvvygV1v88AABoEjoEAAAgMyUDHZLXsoB/PfSgoao4ZhMobYfOdGhsloG2SXM27N692/IFF1xgWdfiDxraD0uHGLUcoLm0tDThuUFDslqGOXnypOV8XvSkELRp08bygw8+mPAYnU2wffv22NsUNb1mb775Zsv+teSRmh07dmS7Cc4553r27Gl5zpw5np/pZ6T65JNPLD/++OOWo9qrQWdh6QJHTz31VOA569evt6yl4bgwQgAAAOgQAACADJUMdDZB0EwCvzhmEyjdK0HbF7SNsr+tcbSpqW644QbLr732mmUdetcc5bBomAVK8nkRk2KhZQJdtKdz587ZaE7sdE+NK6+80jIlg/yl+27MmDHDcrt27TzHafnz0KFDlu+//37LUZUJ9PrR2TjTp09P2J6dO3d6zn/++ecjaUdYjBAAAAA6BAAAIMaSgQ6pa5mgsrLS8uLFiz3n6BC9bnUbx5C8znwImk2gz/vbkO0ygdIyQVRDXSguWjIYNmxYwmP0b37WrFmxtylOOoy8dOlSy5neMjxf6UJluVIS7Nixo+U777wz1DnLly+3vG3btkjaoSXcF154wbKWptQvv/xiWfefcc65d999N5I2hcUIAQAAoEMAAABiLBkE7QMwderUhM83dn5UUt2zQMsH/sUtcgllAqRr8uTJCZ/XRaV0+FPXvs9HukiMrgu/evVqyzoTAd69RCoqKizrUL2WnjLt7bffDnXcmjVrLOvMglTpXifr1q2zrN8t+nem3y0//PCDZX0vd+3a1eT2RIERAgAAQIcAAADEWDLQ4Xkdeh8yZIjlxrYU1jv806F3EOtsBy0TBM0yyKX9CoA4DRgwIOHzH330kWUdas13ugDR3r17LeueH9kc/s5FuuV4WVmZZd3/JZvKy8stN1Zy/uqrrywH7bOiQ/1XX3215VtuucWyzsYZOHCgZf1eO3jwoOWamhrLunBSLmGEAAAA0CEAAAAxlgx02CRoNkFcswzSmU3w0ksvWdY13XNpISIgCrq2+tChQ7PYkszT7bZ79epl+b333rMctB17sdLPzO7du1vW9zKb6urqLPfo0SPwuPHjx1v27x1whn5vBG0BrvS75bPPPrP86KOPWt68eXPS35Nt/MUDAAA6BAAAIMaSwbx58yzrzAK9E9Q/JKd3+y9btizh8zqk/8gjj1gOmjWQ6mwCygTIBSUlJZbDzrjRWTQjRoxIevzo0aMtt2zZMuExffv2tfzWW28lPGbHjh2WX3zxxcDX06HlxsqFmaCfC7rAjN4tjmC6SFEci8g1hZZ7GisZ9OzZ0/KiRYua/Ho6Q+H999+3rGWCqPZHyBRGCAAAAB0CAACQhb0Mgp53zjuMHzQLoKqqKunvCvN79HndqpkyAbJF72Z++OGHLXfp0iUbzXHOOdepUyfLd911V9LjZ86cGfiz/v37W9a13LNNP0dy5Y55pG7JkiWWu3XrZlm3I3bOW44LQ0vKX3/9teXq6mrLhfK9wQgBAACgQwAAAOgQAAAAF+M9BHv27LGsU/n27dtnWVeDcs67SUbQRkepPq+1HW3Hyy+/HOJfAWSOTtXN9rS8M7777jvLYerrr7/+uuWNGzd6fvbTTz9F1i7ATz/rKyoqLPu/Z3RK7qBBgyzrRnj79++3vH37dsu6EVYhYoQAAADQIQAAAM41Ox1ymamwq6WlQjchcs67cppOL0x1GmFlZaVlHUbSMkauiGKVr/r6+ghaEh9d0UtXgtNpaFrO0X3Xs6m0tDSt81O9ZiZMmGA5rtXfdCh11KhRlhcsWGB5y5YtlnUYNVf+X6J4bw4cOBBBS+DXoUOHtM6P43sG4a8ZRggAAAAdAgAAkOWSASgZnEHJAGFRMshdlAxyEyUDAAAQGh0CAAAQ38JEQCJHjx61fPz48Sy2BACgGCEAAAB0CAAAACUDZEDz5s0t19bWWm7VqpXlY8eOZbRNAAAvRggAAAAdAgAAQMkAGaCLjeh2o7r/RENDQ0bbBADwYoQAAADQIQAAACnsZQAAAAoXIwQAAIAOAQAAoEMAAAAcHQIAAODoEAAAAEeHAAAAODoEAADA0SEAAACODgEAAHDO/Q+TMfZv3h+WFgAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "First false equation with length 8 in the training dataset:\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAABICAYAAACJB+2oAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAiBklEQVR4nO2d2W8b19nGn+Ey3EmRFElRlChZoqzFtuLYcqTEdpo4TRqnaVK39UV6USAoUKAXvepfkOsGBXpVoCjaBA3cJb1omyBxviDxksSxI0eLJUuyJGqjRIqkuO/bzHdhnAnlJZFsyTOkzg8QEoxo+x3OmXOe8553YXie50GhUCgUCmVPIxPbAAqFQqFQKOJDBQGFQqFQKBQqCCgUCoVCoVBBQKFQKBQKBVQQUCgUCoVCARUEFAqFQqFQQAUBhUKhUCgUUEFAoVAoFAoFgGKrH2QYZjftEIWt1mTay/cO7O37DwaD2/o7eZ5HKpVCsVgU/g2VSgWFQgGdTgeGYbb13e8GDodjS5/by88dANbX13fREnFoamra0uf28rPfy/e+ZUFAoVC+G57nsba2hng8Do7jIJPJYDabodPpJCMIKBQK5V5QQUCh7BAcx6FYLOLtt9/GtWvXUC6XoVarMTg4iM7OTrz++utgWRb5fF5sUykUCuUuqCCgUHYI4mpUKpVgWRbpdBrlchn5fB7FYnHTZygUCkVq0KBCCmWHYBgGcrkcHo8Hjz32GPR6PWQymfBDoVAoUobOUhTKDkEEwb59+9DX1weWZVEul8U2i0KhULYEFQQUyg4hk8mgVCoxODiIF154AVqtFvl8ngYRUiiUmkAyMQRkd9XR0YGOjg44nU6YTCaMjIzA7/djbW0NuVxObDMplPtCFn65XA61Wo2enh5oNBoolUrhOjk64HmexhNQKHsIhmGgUChgMpnQ2NiI1tZWuFwuaDQasCyLS5cuYWlpSYg9EgPJCAKyu3r88cfx8ssvY3BwEB0dHfjDH/6Azz//HPF4nAoCiqSpFgQajQZHjhxBU1MTEokEeJ7fJAgoFMreQi6XQ6VSobm5Gf39/fje976HkydPwuFwQKvV4re//S2y2SwKhQIVBDqdDhaLBV1dXTh8+DCsVqtQ6KXeaWxsREtLCw4dOoQDBw4I12OxGNLpNCYmJrCxsYGFhYW6TFlTqVTQaDR44okncPToUSwuLiIQCODmzZvY2NgQ27xto1AooFar0dbWBoVCgfPnzyMWi2F0dBR2ux1OpxMAUKlURLaUQpEWCoUCKpUKL730Epqbm6HVapFKpfDee+8hHo8jmUzWzJrAMAyUSiX0ej3cbjecTif6+vrQ0tICj8eD1tZWNDY2QqVSgWEYeDwebGxsiLr5lYwg0Gg0cDgcaG9vR19fH4DbE2atPPwHhWEYWK1WHDp0CGfOnMGrr74q7CIXFxcRDofx73//G3Nzc1hbW6tLQcCyLIxGI06cOIHXX38dFy9exOTkJNbX12tSEBBPgMvlAsdxCAaDqFQqmJqaQi6Xg8vlAsMwVBDg3mmY9f7OU+6PQqGAVqvF6dOn8fjjj6OxsRF+vx8jIyPgOA6pVKpmxgfDMGBZFg0NDTh48CB6e3vx4osvwmazweVyAfhmrHMch7a2NqRSKVy9elU0myUjCJqamvDUU0+hpaVFbFN2DY1Gg+7ubigUt792h8OBgwcPoqWlBd3d3ejo6ADwzSCxWq3Q6XQ4c+YMAoEAstkslpeX4fV66yp6PZfLgeM48DyPhoYGtLW1oVAoQK/Xi23aA0E8W5VKBaVSCZlMBuVyGfF4HOl0GgzD7On4AYPBAJfLhaNHj+K5554TrpPv609/+hPGxsbEM5AiGqVSCfl8Huvr6wgEAmhubobJZILNZkMymYTf7xfbxO9ELpfDbrfDarXixIkTcLlceOqpp2CxWISYAakiGUHQ0NAAj8cDi8Uitim7hlqtRmdnJ1iWBQB4PB48//zzsNvtcLlcUCqVm9SvXq+HTqeDyWRCNBrFp59+ikqlAp/PV1fek3K5LAgcnU4Hq9UKp9MJtVotsmUPB8dx4DgO+Xwe5XIZ2Wy2Lj0820Wr1aK1tRVDQ0N4/fXXAdwWUaSI08cff4yJiQnRPSh7WbSJBRGFyWQSiURCOH7TarXQarU18UzkcjmsVitaW1tx/PhxtLW1YXBwEHK5HMA3Y10mk226HynM55IRBDKZDBqNRtg91xOknr3H48Gvf/1rNDQ0ALg9MdpsNqhUKuEc6V7I5XIYDAb8+Mc/xsrKCliWRSAQkMSkuRu4XC7o9XrY7XYhdY/jOLHNemBIYSKGYWhQIQCn04lXXnkF/f39myZBElg8MDCAfD6Pa9euIRqNimZnPc5FUodkm6nVarAsi3g8jkgkAp/Ph2AwWBPzgE6nw09/+lN4PB4cP34cBoNh03t/8+ZNfPbZZzh27BgGBgaE66FQCEtLS6JuGiQz4uVyORQKRV1OmDKZDCaTCQ6HA/39/bBarQDurwjvvE7Oorq6uqDX69HZ2Qng9sCqR0Gg0+mESUGpVKJQKIht0kNBjgju3BHsVfR6PTweD+x2+11jXSaTwW63o62tDePj4yJZeBsp7Nj2GiTAWKPRQK1Wo1AoIJvNIpVKIZPJ1MQzUSqV6O7uRnd3N5xOp+D5LZfLKBQK8Pl8GBkZQXNzMw4fPgy5XA6e55HP55HJZEQVPZIRBPWKTCaDwWDAD3/4Q/T09GzJDZ5Op5HP56HT6aBSqSCTyYRzKa1Wi1dffRVjY2O4dOkSSqXSI7iLRwvZKRIXG11E6wuDwYCOjg6YzeZN13meB8dxyGQySCQSosfJzM3Nifrv7waNjY1im3BPiGfg5MmT2L9/P06dOgW3241IJIJkMilE39eKIGhra4Pb7YZCoUClUkE6ncbU1BT+85//4NatW/j6669hNBpht9vR2dkJm80Gp9OJzs5OXLlyRTTbJScI6m3yJzvdffv2ob29XVjk7kW5XEapVEIwGEQsFoPT6YTBYBB2zCzLQq/Xo6WlBaFQCDqdDuVyWWicU+uQYDyO41CpVATvRy1MApStI5fLodVqhViaanieRzabRTKZFF0QLC8vbxp79TAOn3zySbFNuCfEg9bS0oKenh40NTXBZDJhdXUVqVRK1Nz87VK922cYBsViEaFQCF6vF+Pj41hdXUU0GkUymUShUBDmOZVKBa1WK6qXXDKCgOyC60kQMAwDo9EIp9OJZ599Fh0dHVCpVPf9/MrKChYWFnDu3Dl88cUXOHv2LPr7+/Hss8/CZrMB+EZ9ZrNZPPnkk1heXsaNGzfqYrIqFApIp9NIp9PIZDLIZDIoFot1cW+U74bjOJRKJYyPj+PSpUuiFyJ74403AEDIGKkH4f3zn/9cbBPuCUk3PHXqFE6fPg2DwYBCoYDh4WFMT0/X1LFhPB7Hm2++CYfDge7ubkSjUVy6dAnRaBSBQAAsy8JisWD//v144oknwLLspvRDMRFdEBD3sE6nQ0NDwyaXOtktkpS0WkMmk8HhcMDlcsFsNkOv1wuCp7p0LdkRr6+vY2pqCrOzs1hcXMTi4iJMJhMymQwsFouQ365SqWAwGOB2u5HP5yGXy+si66BSqSCfzyOXyyGdTqNYLNbss6dsn1wuh1QqhWQyiXQ6LbY5CAQCws5VJpPd06NB2RmUSiVUKhVMJhPMZrMwF/j9fvj9/pqKlSqVSlheXhYqlMZiMXi9XuRyOWSzWTAMA71eL3gEAAhHZclkUtR7FV0QkIJEfX19eOaZZ4S672SRzOVyogdaPCgqlQpnzpxBf38/TCbTpvzz6kUun88jHo/jgw8+wB//+EfkcjnwPI/R0VGEQiEMDg7CaDSioaFBOHKwWq145ZVXcPXqVVy9ehX5fL7mdzCZTAahUEjI10+n0zW1M6A8HF6vF/Pz86JmFlRTLpehVCphNpvhdDpx4sQJWlBqlzCZTLDb7dBoNOB5HvF4HH6/HxcvXsTs7GxNzQOlUgmzs7OQyWQYGxsDx3Hf6enkeR6Tk5P47LPPkEgkHqG1mxFdEKjVajidTlgslk3eAeI+TCQS2NjYqMngOYZhhPzZe50LkcjTcDiM+fl5rK6ubhoMyWQSLMsiFAphY2MDer1eEAQsy6KpqQlWqxUsy9bk93MnCoUCGo0GyWRS8A5Q9g6xWAyrq6uiHxUQPB4PlEolGhsb4XK50NvbC4ZhauIsm8w3tXIEa7PZ0NnZCZ1OB57nEQwG4fP5hM1Brc0FZIzca5NGPOJk80uORzc2NkRf60QXBDabDSdOnIDH49l0PZ/PI51OY25urqbz7clAJgFz1SqxWCwik8ngq6++wrlz5zAzM7PpzwaDQSQSCYyMjKBUKsFutwsxCBqNBr29vVhbWxOCC6UykT4oZrMZ7e3tVBDsUbxeL65cuYJIJCK2KQCAX/3qV1AqlWhoaIDNZsORI0cA1EYPCqVSCZlMBoVCAYZhJH/sduzYMbzwwgtoaWlBuVzG1atXcfPmTQQCAUkcH+0kJpMJHo9HSD9fXl7G4uIipqensbS0tLePDADct/7AndHm9QLP8ygWi0ilUvD7/fD5fMKZ052fq1QqWFtbg8Fg2OR2qsfcdnI/iUQCfr+/5gUOZXuYTCY0NzdLpkJlZ2enkBFhNBo3ta6u/u9OQd7jh3mfiY0bGxtIpVJIpVIolUrC9ZMnTz68oTuI2WyGxWJBR0cH2tvbUS6XEQwGMT8/j/n5+Zo6KtgqWq0WDocDOp0ODMMIxyPZbFb0WDBJCIK9RrlcRiaTgd/vx/Xr1zEyMnLfTIFyuYzx8XEkk0m89tprIlj76FleXsbIyAji8bjYplAeIZ2dneA4DteuXYPX6xXbHAwNDQH4ZuEngpwEuu6WIHiYbCuWZSGTyfD1119jamoKN27cQDweFzwGUhMEnZ2dGBgYwPHjx3H06FFMTk5iZWUFly5dwtTUVN15B4DbIqi3t1eoCbG6uio8J7E9OZIVBMViUWh6U28UCgWsra1hZmYGn376KWZnZ791IKRSKUSjUQSDQRgMBpjN5m+tZ1APSKX19beNv+og0eoJvPqa1Lw3UrAnHA7j8uXL6O7uxuOPPy5cb2xsRKlUEiKvxYbE+CSTSYTDYYyOjsJoNKKlpQUWiwUOh0PIgnoYyDghBclmZmYQiUQeyPtHMpFGR0exuroKn8+HTCYjCAWpYTKZ4Ha7odVqUSqVMDw8jNHRUayvr6NQKEhiDtgNqueGVCqFUCgkCW+IZAVBPp9HKpWqiQCe7ZLL5TA3N4dr167hn//857dOKBzHIRaLQalUYnl5GWq1Gkajse4FgVT4tuOq6toZMpls03EO+b0UFmCCVBaE1dVVvPvuu3jppZc2CYLm5maYzWYYjUYRrfsG0pjK7/djdHQUv//974WGZP39/XC5XCgWiw99pEnGUCKRQDgcxr/+9S+MjY0J1Tq3syiShSaVSiGfzwt9QFiWldRYJDQ2NqK7uxt6vR65XA4fffQR3n//fRQKhbrcDFZDNj3RaFQywbSSFQR+vx+zs7OipmBIhep63rUYcftdsCwLjUYjBExGIhGsrKwgm82KbBnw17/+9a5rHMehXC6jpaVFKDtqsViE3aJCoRAq7uXz+bviPsgCsB1RVx2cul2IEJBKkyiSY36vaGopLlqkA182m4XP58OVK1fg9XoxPDyMSqWyYx6CZDKJbDYLr9eLTCYDtVoNhUIhLOZbefbkM0qlEgzDQKlUgmVZPP3003eVihYT0sV137592L9/P0wmk+CRKZVKdesZAG6njB86dAgOh2PTdSncs2QFQSAQwNTUFFKplNimiA4RBKSCnxQm9Z1EpVJtKkoVjUaxsrIiCcX81ltv3XWNlIs+duwYnnvuOXR3d0Oj0aBcLoPjOGFXRwQBx3GbCtyQn60KguogtgcVBKScqhQ8bpVKpSZK0RJxUi0IstksYrHYrnh/yLMtlUrCmHnQ3T15lziOg16vx8svv4y2trYdtfdh0Ol0aG5uRnt7O7q6ugSRWN0KvV6xWCw4cOAA9Hq92KbcheiCQK/Xo729XUjBIJCXUAqq6WG4Mx+YTCRbfcEZhkFTUxNcLhcee+wxeDwesCyLYrGIcDiMYDAomYn+QTEajdi3bx/MZjMYhkE2m0U8HpdEbYXm5ua7rpHsD4ZhcOvWLczNzQk1y0ulkrDQk/ziUqkEjUYjxIKsrq6CYZgtnRlyHAeZTAa9Xg+FQiG0yd7Ke0E+l0gkkEql8MEHHyASieDNN9/c/hexByHfMcuyMBqNaG9vB4BvbVW+ExSLRTAMg76+Ptjtdhw8eBBqtXrbEehEUCiVSuzfvx8Gg2HXbN4ubrcbL774Irq6uqBUKuH3+7G+vl7XG0Cr1Yru7m709vbCaDSC4zgkEgnhRwpzuOiCQK1Ww+Fw3DVYiSCop93wg0wiDMPAYrGgqakJ7e3taG1tBQChkAVZOKUwmB4UjUaDpqYmQTGT+gxSuCfSQ+J+rK+vIxAIIBwOI5fLoVQq3ZUSSv6fCB1SEncrlSWJIGhqahKOVbaTV07Ok8PhML744gv4fL4t/bndprouhxSPCYDN7nedTgeXywWe54WCMrsFEQT9/f1wu904deoUjEbjQwXZSS2ehdR1aG5uBsMwiEajWF5elsQx4W5hMBjQ29uLlpYWaLVaoaUz+ZHCfCe6INDpdOjs7LzLQzAzM4MLFy4gHA6LZNnO87Cu32qy2SxmZmawuLgoNAGqdaSSWVDNb37zm/v+jrj9R0dHMTk5iVgshmw2K8QIFAoFof85adQyPj6Ozz77TNi9fReFQgFqtRpnz56F2+3GY489BrlcvqU/q1AowHEcLl++jOnpaaytrUkijSubzWJlZQXz8/OYnJyEw+GA3W4X26y7qFQqUCgUQs54c3MzeJ7f1eBMIpBkMhlsNht0Oh00Gs2m2JTtQEotz87OIpvN4plnntkdw7cIKZZEgglJM6t//OMfuHz5cl22nCaQSqwk4yOXywnVCTc2NiQxh4suCFiWhdlsvivVKBKJ1IVivDMYbLsvNGn+RHYl1TnRgUAAkUik7jwpUmJgYOC+v5PL5VAoFEIgFCkGQ54VCQINBoPIZrMIhULb7pqXzWah1+tx8uRJmEwmAFvPFiCxAysrK5idnRUaRokNKUkeiUQQDAbvOkslMRZij2lyjq/T6TYdGewW1cGn1fdPxteDwDAMSqUSAoGAJAK0FQoF9Ho9Ghoa0NjYCL/fj9XVVUxOTuKrr74S27xdp7oIH/GEZrNZScRLARIQBJT7o9FooNfr8fzzz6Ovr09YEIDbgXfnz5+Hz+cTfeKsZ/L5/H1/R44DOjo64HQ6hZgXspNbXl7G2toaJicnwfM8urq6YDKZ4HK5AHx3q1Oe51EqlcCyLHp7e4Wo5O2muZFiOlKhVCohmUwiFoshFAqhqakJwDcLodvtRl9fn+D92iuQMsOhUAjJZBKTk5NIJpMPVahIJpOhUCjg/Pnz2NjYwC9+8YsdtnrrdqhUKhw4cAA/+clPcOTIEZhMJkxOTmJsbAwbGxui2PWoqT4qy2aziEaj3zrHPGpEEwQk7Yq8BJS70Wg0MBqNcLvd2Ldv36b2q8QVHQ6HJedmrye+bSEl3ztp3U0EglKpFFITq3d2VqsVNpsNPT09kMlk33lmSIIXFQoFLBYLdDrdpn93q6jVauh0uodaWHYSInTS6TQikYjgBSSCwGw2w2azYXV1VWRLv3kGJDNiNyFFhTY2NhCNRjE7O4toNCqkED4IRBDMzc2J2kWSlIC22Ww4fPgwWltbIZfLkU6nsba2VvOe4O1Csn4SiYQkvHYE0QSBWq2Gy+VCU1MT1Gr1rgfq1BoMw6C3txcejwdPPvkkurq6oNFohN+TLANa3lc8qtPSSNYBAMFTkM/nUSgUhMwDj8eDrq4uvPTSS1vOMqg+Uyauxq16CMrlMniex/Hjx9Ha2op33nkH6+vrD3i3O4/X68W7774LnU6HI0eOCMdjhw8fhkKhwOLioqjjW6FQoFgsIhKJYG1tDZcvXwbP87smrEjgH2lqNjo6ilgs9tA9DkjvFDE3DgaDAUePHsXQ0BCGhoaEQkzj4+P43//+tyfnsaWlJVy4cAF+v19sUwREEwRyuRw6nQ5qtVpQxpTbKJVKqFQquFwudHR03LU7JPnQpVKpLho/kWAbEgQnJff2VrgzPoTUuif3QhZ1slM3m82QyWSbihbdD5JRQOJEyPPeyuJAXJONjY2oVCpwuVySes8ymQzW1taQSqWE70EmkwlFaxQKcU80yXecz+cRi8UwMzOzq1kGRBBEIhEh8pzjOGg0mgd+bmTsmUwmUb1DpGukyWSCwWBAMplEKBRCOBxGJBKRRIT9bsOyLEwmk7CxKxQKSKVSkihZTBDtjVMoFDAYDJJyZUoFl8sFl8uFs2fPYmhoaFMGRi6Xw/Xr1zExMVEXLxHDMGhsbMSBAwdgMpmE9Jt6OgYhY5scIeTzeTAMs62zwzvLIm8FIh7a2trgcrmg0Wgk5ZpNpVLIZrNIJBKSfN4ksC+VSsHn8+HChQvgOE6oqLlbkNTVoaEhtLa24vjx4zAYDA+Vdih2Xw2lUgmbzSbEQc3Pz+P8+fOYmJiQ1IK4mzidTpw+fRpOp1NsU+6LqDEECoVCEAP1KAjIbr663DA5Y7ZarbDb7UI99GKxCIPBAIPBgP3796OlpQWtra1oaGiAUqkEz/OIx+OIRqOYmJjA3NxczXsHiGfAarXC7XaDZVlEIhFh5yzFReJhIPdEotcf1f0RD5zD4ZDU5EtK1ZLxT+YEvV4v9DTQarWilVyuLkzU0NAAj8cjVA7cTcrlMhQKBXp6eoRqfnq9XnS3/4Mgk8mg0WhgsVjQ1tYGq9UqHMN4vV5R4xp2E7lcLoggUqjM5XLBYrFAq9WCYRhYrVZ0dXWB53khsBa4vekjsUMymQwsy6JUKgnxNqFQaNfsFlUQKJVKIaiwHgVBpVLBwsICVCoVTp06Jdyn2WwWVH84HEY0GsX6+joGBwdx7NgxdHd3w+l0QqVSCYMpn89jeHgYt27dwu9+9ztEIhFJTe4PAqlSOTAwgB/96EeYmZnBtWvX4Pf7JVGlsNap9kwAgMPhkOR7lsvlEIlEYLVaoVQq0d3dDYfDgY8++gjFYhHz8/OiRGKThdntdsNms+HgwYMAdr/fAln0jUYjVCqVcHxSa0dpwO2qjl1dXTh69Chee+01KJVKRKNRTE9P46OPPpJEXYydhmEYodja2bNnhbRa0vdErVaDYRg8/fTTGBoa2nRMWqlUMD09jXQ6DbPZDI1Gg+bmZoTDYbz//vu4ceMG/v73v+/aZlB0D0F1XiZw242YTCaRTCaF2vC1CsdxWF9fh8FgQCKRgFarFWIBSOR4f38/kskkotEo+vr6BBWt1+uF3WQymUQikYDX68Xi4qLkzp0eBJlMBqPRiK6uLjQ3N4NlWYRCIYyNjSESiYhtXl0iRTEAAMFgEDdv3sTBgweh1+vBsix0Oh1aWlqEeiRiCAKyMFd7soBHJwjujK+S6vP7NlQqFfbt24e2tjbo9Xqk02ksLCwgEAjUTUG1OyGbXZ1Oh56eHqF7p8PhEDpYAhA2xNVwHIfm5mZks1kYjUYolUpotVrhmGq3PUSiBhWSG60uq7m8vIzJyUmh212tuciqKZfLGB8fFyY1UoueDIiWlhb87Gc/E1KbWJbd1NmMXPd6vVhdXcUnn3yCpaWlmhcD5IVpbW3Fyy+/jJ6eHgDA6Ogo/vKXv9R1PXMxkeoR040bN8BxHH75y1/C7XZDq9VCqVRiYGAAOp0Oo6OjoowJEpAnl8shl8uFhkGPGpLFUosYjUahZ4FWq8XKygrOnz+P8fHxuvQOALfnN5VKBZvNhu9///tC+XPiIa6e3+/1Z6ubUFUqFaRSKaE19m7H24gmCCqVCrLZrND3mnxRJJKaRGfXMjzPo1AoIBaL4eLFiwgEAjhz5oxQlZGcD5HBUR1PwfM8/H4/otEoLl++jOXlZSwtLWFjY6NmJwcCKV9qNBrh8XjQ0NCAZDIp1Pauh2BJytYplUrIZDLI5/MoFouC15DslHb7zF7KPGiFU6kgk8mg1Wohk8ng9/vh9Xpx48YNrK2tiW3arkHm/Xg8junpaSSTSbS0tGzyCJD4mXK5jEQicd9g31wuh1u3bmF1dRUTExPw+/31KwiqF4BqL0GtDv57kcvlsL6+jrfeegsHDx7ED37wA0EQkN3HveB5HtPT07h16xbefvtteL1e5HK5mj5CITAMA5Zl4XA4MDAwgEwmg1AohHg8LqkoeMqjoVgsIp1OC+29SWdHj8cDtVot2s4c2DwX1dO89KiQyWQwGAzgeR5TU1MYHh7GJ598UtcxQhzHIZPJIBgM4uOPP8b+/ftx+vRpaLXaTWW6i8UistksvF7vfWsRxGIx/Pe//0UgEMDk5OSub5ZEEwT5fB4+nw+Tk5M4f/48Ojs7cejQIaTTaYRCIcnUdt4JSBxAMBjE2NgY3G43Ojo67hIDlUoF5XIZN2/exPLyMq5cuYKlpSWEw+GajDD+NqoDSaPRKKampuqqkRVl6yQSCaysrAg/7e3twrkrpfYhnt9isYhyuSx4gOuZSqWCZDKJL7/8EgsLC1hbW0NbWxsGBwdhNBphsVgwMTGBsbExzM3NIRAI3PPvyeVy8Hq9SKVSj8QzLKogWF5eBnC7itXzzz+PQ4cOCcqqnnaKPM8jlUohFArh+vXrSKVScLvdQgU6QrFYRKFQwJdffomLFy9idHQU6+vrKBaLdfUC3XnfkUgEN27cQDAYFNEqiljEYjHEYjEsLi5iaWkJNpuNCoI6ghyJFotFoZhaPW1u7kWlUkEikcDly5ehVCpx8eJFDAwMwGQyobW1VYiNOXfuHObn5yUz94ne3CgajWJ4eBjBYBDDw8NYWVmBz+eTVDnHnSKRSOC9997D7OwsDAYDmpub0dPTg3A4jKWlJSGL4MsvvxTqmNdjJ0O9Xo9Dhw6hra0N5XIZCwsL+PDDD+Hz+cQ2jSIiw8PDSCQScDqdki7eQtk6lUoF0WgUwWAQn3/+ORYXF+teDNxJpVJBOp3G9PQ0/vznPwvdHmdmZrCwsCCpIGrRBUEqlcLMzAxmZmbwf//3f2Kbs6tkMhlcvXoVGxsbOHDgAIrFIrq6urCxsYGbN2/iq6++wvDwMHw+X12n3mk0GnR2dsJut6NYLGJtbQ1Xr14V2yyKyMzMzGB1dRWvvfbanls06hWyU45Go7hy5Qri8fiee7YcxyGXy2FpaQlLS0tim/OtiC4I9iLr6+s4d+4cTCYT3nnnHaEOQSQS2dT9rV6JxWK4cOECxsbG8Omnn2J+fl5skygSgPSHHx8fh8FgQGtrq9gmUR6SSCSCv/3tbygWiwgGg3UdTFgPUEEgAul0GmNjY2KbIRokUAYArl+/LrI1FKlQqVRQLBaxsrKCmZkZ6PX6mi9OttfJ5XIYGRkR2wzKFqGCgELZRUhxKbKw0cXt/pDc7A8//BCff/45TCaTUI+DQqHsPlQQUCi7AM/zQgEmstNVqVS71jq3XuB5vq6L1lAoUoYKAgplhyH5wna7HQaDAW+88QZ4nofT6YRWq6WVGCkUiiShgoBC2WGqm+LodDp0dHSA53lotVooFIo9F2VNoVBqA4ansxOFQqFQKHse2Xd/hEKhUCgUSr1DBQGFQqFQKBQqCCgUCoVCoVBBQKFQKBQKBVQQUCgUCoVCARUEFAqFQqFQQAUBhUKhUCgUUEFAoVAoFAoFVBBQKBQKhUIB8P/jvwwAboqkNAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "true_train_equation_with_length_5 = true_train_equation[5]\n", + "true_train_equation_with_length_8 = true_train_equation[8]\n", + "print(f\"First true equation with length 5 in the training dataset:\")\n", + "for i, x in enumerate(true_train_equation_with_length_5[0]):\n", + " plt.subplot(1, 5, i + 1)\n", + " plt.axis(\"off\")\n", + " plt.imshow(x.squeeze(), cmap=\"gray\")\n", + "plt.show()\n", + "print(f\"First true equation with length 8 in the training dataset:\")\n", + "for i, x in enumerate(true_train_equation_with_length_8[0]):\n", + " plt.subplot(1, 8, i + 1)\n", + " plt.axis(\"off\")\n", + " plt.imshow(x.squeeze(), cmap=\"gray\")\n", + "plt.show()\n", + "\n", + "false_train_equation_with_length_5 = false_train_equation[5]\n", + "false_train_equation_with_length_8 = false_train_equation[8]\n", + "print(f\"First false equation with length 5 in the training dataset:\")\n", + "for i, x in enumerate(false_train_equation_with_length_5[0]):\n", + " plt.subplot(1, 5, i + 1)\n", + " plt.axis(\"off\")\n", + " plt.imshow(x.squeeze(), cmap=\"gray\")\n", + "plt.show()\n", + "print(f\"First false equation with length 8 in the training dataset:\")\n", + "for i, x in enumerate(false_train_equation_with_length_8[0]):\n", + " plt.subplot(1, 8, i + 1)\n", + " plt.axis(\"off\")\n", + " plt.imshow(x.squeeze(), cmap=\"gray\")\n", + "plt.show()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Building the Learning Part" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To build the learning part, we need to first build a machine learning base model. We use SymbolNet, and encapsulate it within a `BasicNN` object to create the base model. `BasicNN` is a class that encapsulates a PyTorch model, transforming it into a base model with an sklearn-style interface. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# class of symbol may be one of ['0', '1', '+', '='], total of 4 classes\n", + "cls = SymbolNet(num_classes=4)\n", + "loss_fn = nn.CrossEntropyLoss()\n", + "optimizer = torch.optim.RMSprop(cls.parameters(), lr=0.001, weight_decay=1e-4)\n", + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + "\n", + "base_model = BasicNN(\n", + " cls,\n", + " loss_fn,\n", + " optimizer,\n", + " device=device,\n", + " batch_size=32,\n", + " num_epochs=1,\n", + " stop_loss=None,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "However, the base model built above deals with instance-level data (i.e., individual images), and can not directly deal with example-level data (i.e., a list of images comprising the equation). Therefore, we wrap the base model into `ABLModel`, which enables the learning part to train, test, and predict on example-level data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model = ABLModel(base_model)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Building the Reasoning Part" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the reasoning part, we first build a knowledge base. As mentioned before, the knowledge base in this task involves the structure of the equations and a recursive definition of bit-wise operations, which are defined in Prolog file ``reasoning/BK.pl`` and ``reasoning/learn_add.pl``, respectively. Specifically, the knowledge about the structure of equations is a set of DCG rules recursively define that a digit is a sequence of ‘0’ and ‘1’, and equations share the structure of X+Y=Z, though the length of X, Y and Z can be varied. The knowledge about bit-wise operations is a recursive logic program, which reversely calculates X+Y, i.e., it operates on X and Y digit-by-digit and from the last digit to the first.\n", + "\n", + "The knowledge base is already built in ``HedKB``. ``HedKB`` is derived from class ``PrologKB``, and is built upon the aformentioned Prolog files. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "kb = HedKB()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note: Please notice that, the specific rules for calculating the operations are undefined in the knowledge base, i.e., results of '0+0', '0+1' and '1+1' could be '0', '1', '00', '01' or even '10'. The missing calculation rules are required to be learned from the data. Therefore, `HedKB` incorporates methods for abducing rules from data. Users interested can refer to the specific implementation of `HedKB`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then, we create a reasoner. Due to the indeterminism of abductive reasoning, there could be multiple candidates compatible to the knowledge base. When this happens, reasoner can minimize inconsistencies between the knowledge base and pseudo-labels predicted by the learning part, and then return only one candidate that has the highest consistency. \n", + "\n", + "In this task, we create the reasoner by instantiating the class `HedReasoner`, which is a reasoner derived from `Reasoner` and tailored specifically for this task. `HedReasoner` leverages [ZOOpt library](https://github.com/polixir/ZOOpt) for acceleration, and has designed a specific strategy to better harness ZOOpt’s capabilities. Additionally, methods for abducing rules from data have been incorporated. Users interested can refer to the specific implementation of `HedReasoner` in `reasoning/reasoning.py`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "reasoner = HedReasoner(kb, dist_func=\"hamming\", use_zoopt=True, max_revision=10)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Building Evaluation Metrics" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, we set up evaluation metrics. These metrics will be used to evaluate the model performance during training and testing. Specifically, we use `SymbolAccuracy` and `ReasoningMetric`, which are used to evaluate the accuracy of the machine learning model’s predictions and the accuracy of the final reasoning results, respectively." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metric_list = [ConsistencyMetric(kb=kb)]" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Bridge Learning and Reasoning\n", + "\n", + "Now, the last step is to bridge the learning and reasoning part. We proceed this step by creating an instance of `HedBridge`, which is derived from `SimpleBridge` and tailored specific for this task." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "bridge = HedBridge(model, reasoner, metric_list)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Perform pretraining, training and testing by invoking the `pretrain`, `train` and `test` methods of `HedBridge`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Build logger\n", + "print_log(\"Abductive Learning on the HED example.\", logger=\"current\")\n", + "\n", + "# Retrieve the directory of the Log file and define the directory for saving the model weights.\n", + "log_dir = ABLLogger.get_current_instance().log_dir\n", + "weights_dir = osp.join(log_dir, \"weights\")\n", + "\n", + "bridge.pretrain(weights_dir)\n", + "bridge.train(train_data, val_data, save_dir=weights_dir)\n", + "bridge.test(test_data)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "ABL", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.18" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "fb6f4ceeabb9a733f366948eb80109f83aedf798cc984df1e68fb411adb27d58" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/hed/main.py b/examples/hed/main.py new file mode 100644 index 0000000..d66d197 --- /dev/null +++ b/examples/hed/main.py @@ -0,0 +1,111 @@ +import argparse +import os.path as osp + +import torch +import torch.nn as nn + +from abl.learning import ABLModel, BasicNN +from abl.utils import ABLLogger, print_log + +from bridge import HedBridge +from consistency_metric import ConsistencyMetric +from datasets import get_dataset, split_equation +from models.nn import SymbolNet +from reasoning import HedKB, HedReasoner + + +def main(): + parser = argparse.ArgumentParser(description="Handwritten Equation Decipherment example") + parser.add_argument( + "--no-cuda", action="store_true", default=False, help="disables CUDA training" + ) + parser.add_argument( + "--epochs", + type=int, + default=1, + help="number of epochs in each learning loop iteration (default : 1)", + ) + parser.add_argument( + "--lr", type=float, default=1e-3, help="base model learning rate (default : 0.001)" + ) + parser.add_argument( + "--weight-decay", type=float, default=1e-4, help="weight decay (default : 0.0001)" + ) + parser.add_argument( + "--batch-size", type=int, default=32, help="base model batch size (default : 32)" + ) + parser.add_argument( + "--segment_size", type=int, default=1000, help="segment size (default : 1000)" + ) + parser.add_argument("--save_interval", type=int, default=1, help="save interval (default : 1)") + parser.add_argument( + "--max-revision", + type=int, + default=10, + help="maximum revision in reasoner (default : 10)", + ) + + args = parser.parse_args() + + # Build logger + print_log("Abductive Learning on the HED example.", logger="current") + + ### Working with Data + print_log("Working with Data.", logger="current") + + total_train_data = get_dataset(train=True) + train_data, val_data = split_equation(total_train_data, 3, 1) + test_data = get_dataset(train=False) + + ### Building the Learning Part + print_log("Building the Learning Part.", logger="current") + + # Build necessary components for BasicNN + cls = SymbolNet(num_classes=4) + loss_fn = nn.CrossEntropyLoss() + optimizer = torch.optim.RMSprop(cls.parameters(), lr=args.lr, weight_decay=args.weight_decay) + use_cuda = not args.no_cuda and torch.cuda.is_available() + device = torch.device("cuda" if use_cuda else "cpu") + + # Build BasicNN + base_model = BasicNN( + cls, + loss_fn, + optimizer, + device=device, + batch_size=args.batch_size, + num_epochs=args.epochs, + stop_loss=None, + ) + + # Build ABLModel + model = ABLModel(base_model) + + ### Building the Reasoning Part + print_log("Building the Reasoning Part.", logger="current") + + # Build knowledge base + kb = HedKB() + + # Create reasoner + reasoner = HedReasoner(kb, dist_func="hamming", use_zoopt=True, max_revision=args.max_revision) + + ### Building Evaluation Metrics + print_log("Building Evaluation Metrics.", logger="current") + metric_list = [ConsistencyMetric(kb=kb)] + + ### Bridge Learning and Reasoning + print_log("Bridge Learning and Reasoning.", logger="current") + bridge = HedBridge(model, reasoner, metric_list) + + # Retrieve the directory of the Log file and define the directory for saving the model weights. + log_dir = ABLLogger.get_current_instance().log_dir + weights_dir = osp.join(log_dir, "weights") + + bridge.pretrain(weights_dir) + bridge.train(train_data, val_data, save_dir=weights_dir) + bridge.test(test_data) + + +if __name__ == "__main__": + main() diff --git a/examples/hed/models/nn.py b/examples/hed/models/nn.py new file mode 100644 index 0000000..f65b9e0 --- /dev/null +++ b/examples/hed/models/nn.py @@ -0,0 +1,49 @@ +import torch +from torch import nn + + +class SymbolNet(nn.Module): + def __init__(self, num_classes=4, image_size=(28, 28, 1)): + super(SymbolNet, self).__init__() + self.conv1 = nn.Sequential( + nn.Conv2d(1, 32, 5, stride=1), + nn.ReLU(), + nn.MaxPool2d(kernel_size=2, stride=2), + nn.BatchNorm2d(32, momentum=0.99, eps=0.001), + ) + self.conv2 = nn.Sequential( + nn.Conv2d(32, 64, 5, padding=2, stride=1), + nn.ReLU(), + nn.MaxPool2d(kernel_size=2, stride=2), + nn.BatchNorm2d(64, momentum=0.99, eps=0.001), + ) + + num_features = 64 * (image_size[0] // 4 - 1) * (image_size[1] // 4 - 1) + self.fc1 = nn.Sequential(nn.Linear(num_features, 120), nn.ReLU()) + self.fc2 = nn.Sequential(nn.Linear(120, 84), nn.ReLU()) + self.fc3 = nn.Sequential(nn.Linear(84, num_classes)) + + def forward(self, x): + x = self.conv1(x) + x = self.conv2(x) + x = torch.flatten(x, 1) + x = self.fc1(x) + x = self.fc2(x) + x = self.fc3(x) + return x + + +class SymbolNetAutoencoder(nn.Module): + def __init__(self, num_classes=4, image_size=(28, 28, 1)): + super(SymbolNetAutoencoder, self).__init__() + self.base_model = SymbolNet(num_classes, image_size) + self.softmax = nn.Softmax(dim=1) + self.fc1 = nn.Sequential(nn.Linear(num_classes, 100), nn.ReLU()) + self.fc2 = nn.Sequential(nn.Linear(100, image_size[0] * image_size[1]), nn.ReLU()) + + def forward(self, x): + x = self.base_model(x) + # x = self.softmax(x) + x = self.fc1(x) + x = self.fc2(x) + return x diff --git a/examples/hed/reasoning/BK.pl b/examples/hed/reasoning/BK.pl new file mode 100644 index 0000000..441df4e --- /dev/null +++ b/examples/hed/reasoning/BK.pl @@ -0,0 +1,83 @@ +:- use_module(library(apply)). +:- use_module(library(lists)). +% :- use_module(library(tabling)). +% :- table valid_rules/2, op_rule/2. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% DCG parser for equations +%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% symbols to be mapped +digit(1). +digit(0). + +% digits +digits([D]) --> [D], { digit(D) }. % empty list [] is not a digit +digits([D | T]) --> [D], !, digits(T), { digit(D) }. +digits(X):- + phrase(digits(X), X). +% More integrity constraints 1: +% This two clauses forbid the first digit to be 0. +% You may uncomment them to prune the search space +% length(X, L), +% (L > 1 -> X \= [0 | _]; true). + +% Equation definition +eq_arg([D]) --> [D], { \+ D == '+', \+ D == '=' }. +eq_arg([D | T]) --> [D], !, eq_arg(T), { \+ D == '+', \+ D == '=' }. +equation(eq(X, Y, Z)) --> + eq_arg(X), [+], eq_arg(Y), [=], eq_arg(Z). +% More integrity constraints 2: +% This clause restricts the length of arguments to be sane, +% You may uncomment them to prune the search space +% { length(X, LX), length(Y, LY), length(Z, LZ), +% LZ =< max(LX, LY) + 1, LZ >= max(LX, LY) }. +parse_eq(List_of_Terms, Eq) :- + phrase(equation(Eq), List_of_Terms). + +%%%%%%%%%%%%%%%%%%%%%% +%% Bit-wise operation +%%%%%%%%%%%%%%%%%%%%%% +% Abductive calculation with given pseudo-labels, abduces pseudo-labels as well as operation rules +calc(Rules, Pseudo) :- + calc([], Rules, Pseudo). +calc(Rules0, Rules1, Pseudo) :- + parse_eq(Pseudo, eq(X,Y,Z)), + bitwise_calc(Rules0, Rules1, X, Y, Z). + +% Bit-wise calculation that handles carrying +bitwise_calc(Rules, Rules1, X, Y, Z) :- + reverse(X, X1), reverse(Y, Y1), reverse(Z, Z1), + bitwise_calc_r(Rules, Rules1, X1, Y1, Z1), + maplist(digits, [X,Y,Z]). +bitwise_calc_r(Rs, Rs, [], Y, Y). +bitwise_calc_r(Rs, Rs, X, [], X). +bitwise_calc_r(Rules, Rules1, [D1 | X], [D2 | Y], [D3 | Z]) :- + abduce_op_rule(my_op([D1],[D2],Sum), Rules, Rules2), + ((Sum = [D3], Carry = []); (Sum = [C, D3], Carry = [C])), + bitwise_calc_r(Rules2, Rules3, X, Carry, X_carried), + bitwise_calc_r(Rules3, Rules1, X_carried, Y, Z). + +%%%%%%%%%%%%%%%%%%%%%%%%% +% Abduce operation rules +%%%%%%%%%%%%%%%%%%%%%%%%% +% Get an existed rule +abduce_op_rule(R, Rules, Rules) :- + member(R, Rules). +% Add a new rule +abduce_op_rule(R, Rules, [R|Rules]) :- + op_rule(R), + valid_rules(Rules, R). + +% Integrity Constraints +valid_rules([], _). +valid_rules([my_op([X1],[Y1],_)|Rs], my_op([X],[Y],Z)) :- + op_rule(my_op([X],[Y],Z)), + [X,Y] \= [X1,Y1], + [X,Y] \= [Y1,X1], + valid_rules(Rs, my_op([X],[Y],Z)). +valid_rules([my_op([Y],[X],Z)|Rs], my_op([X],[Y],Z)) :- + X \= Y, + valid_rules(Rs, my_op([X],[Y],Z)). + +op_rule(my_op([X],[Y],[Z])) :- digit(X), digit(Y), digit(Z). +op_rule(my_op([X],[Y],[Z1,Z2])) :- digit(X), digit(Y), digits([Z1,Z2]). diff --git a/examples/hed/reasoning/__init__.py b/examples/hed/reasoning/__init__.py new file mode 100644 index 0000000..6fdae78 --- /dev/null +++ b/examples/hed/reasoning/__init__.py @@ -0,0 +1,3 @@ +from .reasoning import HedKB, HedReasoner + +__all__ = ["HedKB", "HedReasoner"] diff --git a/examples/hed/reasoning/learn_add.pl b/examples/hed/reasoning/learn_add.pl new file mode 100644 index 0000000..3bb23b5 --- /dev/null +++ b/examples/hed/reasoning/learn_add.pl @@ -0,0 +1,84 @@ +:- ensure_loaded(['BK.pl']). +:- thread_setconcurrency(_, 8). +:- use_module(library(thread)). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% For propositionalisation +%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +eval_inst_feature(Ex, Feature):- + eval_eq(Ex, Feature). + +%% Evaluate instance given feature +eval_eq(Ex, Feature):- + parse_eq(Ex, eq(X,Y,Z)), + bitwise_calc(Feature,_,X,Y,Z), !. + +%%%%%%%%%%%%%% +%% Abduction +%%%%%%%%%%%%%% +% Make abduction when given samples that have been interpreted as pseudo-labels +abduce(Exs, Delta_C) :- + abduce(Exs, [], Delta_C). +abduce([], Delta_C, Delta_C). +abduce([E|Exs], Delta_C0, Delta_C1) :- + calc(Delta_C0, Delta_C2, E), + abduce(Exs, Delta_C2, Delta_C1). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Abduce pseudo-labels only +%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +abduce_consistent_insts(Exs):- + abduce(Exs, _), !. +% (Experimental) Uncomment to use parallel abduction +% abduce_consistent_exs_concurrent(Exs), !. + +logic_forward(Exs, X) :- abduce_consistent_insts(Exs) -> X = true ; X = false. +logic_forward(Exs) :- abduce_consistent_insts(Exs). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Abduce Delta_C given pseudo-labels +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +consistent_inst_feature(Exs, Delta_C):- + abduce(Exs, Delta_C), !. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% (Experimental) Parallel abduction +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +abduce_consistent_exs_concurrent(Exs) :- + % Split the current data batch into grounding samples and variable samples (which need to be revised) + split_exs(Exs, Ground_Exs, Var_Exs), + % Find the simplest Delta_C for grounding samples. + abduce(Ground_Exs, Ground_Delta_C), !, + % Extend Ground Delta_C into all possible variations + extend_op_rule(Ground_Delta_C, Possible_Deltas), + % Concurrently abduce the variable samples + maplist(append([abduce2, Var_Exs, Ground_Exs]), [[Possible_Deltas]], Call_List), + maplist(=.., Goals, Call_List), + % writeln(Goals), + first_solution(Var_Exs, Goals, [local(inf)]). + +split_exs([],[],[]). +split_exs([E | Exs], [E | G_Exs], V_Exs):- + ground(E), !, + split_exs(Exs, G_Exs, V_Exs). +split_exs([E | Exs], G_Exs, [E | V_Exs]):- + split_exs(Exs, G_Exs, V_Exs). + +:- table extend_op_rule/2. + +extend_op_rule(Rules, Rules) :- + length(Rules, 4). +extend_op_rule(Rules, Ext) :- + op_rule(R), + valid_rules(Rules, R), + extend_op_rule([R|Rules], Ext). + +% abduction without learning new Delta_C (Because they have been extended!) +abduce2([], _, _). +abduce2([E|Exs], Ground_Exs, Delta_C) :- + % abduce by finding ground samples + member(E, Ground_Exs), + abduce2(Exs, Ground_Exs, Delta_C). +abduce2([E|Exs], Ground_Exs, Delta_C) :- + eval_inst_feature(E, Delta_C), + abduce2(Exs, Ground_Exs, Delta_C). diff --git a/examples/hed/reasoning/reasoning.py b/examples/hed/reasoning/reasoning.py new file mode 100644 index 0000000..02910ef --- /dev/null +++ b/examples/hed/reasoning/reasoning.py @@ -0,0 +1,102 @@ +import math +import os + +import numpy as np + +from abl.reasoning import PrologKB, Reasoner +from abl.utils import reform_list + +CURRENT_DIR = os.path.abspath(os.path.dirname(__file__)) + + +class HedKB(PrologKB): + def __init__( + self, pseudo_label_list=[1, 0, "+", "="], pl_file=os.path.join(CURRENT_DIR, "learn_add.pl") + ): + pl_file = pl_file.replace("\\", "/") + super().__init__(pseudo_label_list, pl_file) + self.learned_rules = {} + + def consist_rule(self, exs, rules): + rules = str(rules).replace("'", "") + return len(list(self.prolog.query("eval_inst_feature(%s, %s)." % (exs, rules)))) != 0 + + def abduce_rules(self, pred_res): + prolog_result = list(self.prolog.query("consistent_inst_feature(%s, X)." % pred_res)) + if len(prolog_result) == 0: + return None + prolog_rules = prolog_result[0]["X"] + rules = [rule.value for rule in prolog_rules] + return rules + + +class HedReasoner(Reasoner): + def revise_at_idx(self, data_example): + revision_idx = np.where(np.array(data_example.flatten("revision_flag")) != 0)[0] + candidate = self.kb.revise_at_idx( + data_example.pred_pseudo_label, data_example.Y, data_example.X, revision_idx + ) + return candidate + + def zoopt_budget(self, symbol_num): + return 200 + + def zoopt_score(self, symbol_num, data_example, sol, get_score=True): + revision_flag = reform_list( + list(sol.get_x().astype(np.int32)), data_example.pred_pseudo_label + ) + data_example.revision_flag = revision_flag + + lefted_idxs = [i for i in range(len(data_example.pred_idx))] + candidate_size = [] + max_consistent_idxs = [] + while lefted_idxs: + idxs = [] + idxs.append(lefted_idxs.pop(0)) + max_candidate_idxs = [] + found = False + for idx in range(-1, len(data_example.pred_idx)): + if (idx not in idxs) and (idx >= 0): + idxs.append(idx) + candidates, _ = self.revise_at_idx(data_example[idxs]) + if len(candidates) == 0: + if len(idxs) > 1: + idxs.pop() + else: + if len(idxs) > len(max_candidate_idxs): + found = True + max_candidate_idxs = idxs.copy() + removed = [i for i in lefted_idxs if i in max_candidate_idxs] + if found: + removed.insert(0, idxs[0]) + candidate_size.append(len(removed)) + max_consistent_idxs = max_candidate_idxs.copy() + lefted_idxs = [i for i in lefted_idxs if i not in max_candidate_idxs] + candidate_size.sort() + score = 0 + + for i in range(0, len(candidate_size)): + score -= math.exp(-i) * candidate_size[i] + if get_score: + return score + else: + return max_consistent_idxs + + def abduce(self, data_example): + symbol_num = data_example.elements_num("pred_pseudo_label") + max_revision_num = self._get_max_revision_num(self.max_revision, symbol_num) + + solution = self._zoopt_get_solution(symbol_num, data_example, max_revision_num) + max_candidate_idxs = self.zoopt_score(symbol_num, data_example, solution, get_score=False) + + abduced_pseudo_label = [[] for _ in range(len(data_example))] + + if len(max_candidate_idxs) > 0: + candidates, _ = self.revise_at_idx(data_example[max_candidate_idxs]) + for i, idx in enumerate(max_candidate_idxs): + abduced_pseudo_label[idx] = candidates[0][i] + data_example.abduced_pseudo_label = abduced_pseudo_label + return abduced_pseudo_label + + def abduce_rules(self, pred_res): + return self.kb.abduce_rules(pred_res) diff --git a/examples/hed/requirements.txt b/examples/hed/requirements.txt new file mode 100644 index 0000000..11aaa3a --- /dev/null +++ b/examples/hed/requirements.txt @@ -0,0 +1,2 @@ +abl +gdown \ No newline at end of file diff --git a/examples/hed/utils.py b/examples/hed/utils.py new file mode 100644 index 0000000..84c00be --- /dev/null +++ b/examples/hed/utils.py @@ -0,0 +1,65 @@ +import numpy as np +import torch +import torch.nn as nn +import torch.utils.data.sampler as sampler + + +class InfiniteSampler(sampler.Sampler): + def __init__(self, num_examples, batch_size=1): + self.num_examples = num_examples + self.batch_size = batch_size + + def __iter__(self): + while True: + order = np.random.permutation(self.num_examples) + for i in range(self.num_examples): + yield order[i : i + self.batch_size] + i += self.batch_size + + def __len__(self): + return None + + +def gen_mappings(chars, symbs): + n_char = len(chars) + n_symbs = len(symbs) + if n_char != n_symbs: + print("Characters and symbols size dosen't match.") + return + from itertools import permutations + + mappings = [] + # returned mappings + perms = permutations(symbs) + for p in perms: + if p.index(1) < p.index(0): + continue + mappings.append(dict(zip(chars, list(p)))) + return mappings + + +def mapping_res(original_pred_res, m): + return [[m[symbol] for symbol in formula] for formula in original_pred_res] + + +def remapping_res(pred_res, m): + remapping = {} + for key, value in m.items(): + remapping[value] = key + return [[remapping[symbol] for symbol in formula] for formula in pred_res] + + +def extract_feature(img): + extractor = nn.AvgPool2d(2, stride=2) + feature_map = np.array(extractor(torch.Tensor(img))) + return feature_map.reshape((-1,)) + + +def reduce_dimension(data): + for truth_value in [0, 1]: + for equation_len in range(5, 27): + equations = data[truth_value][equation_len] + reduced_equations = [ + [extract_feature(symbol_img) for symbol_img in equation] for equation in equations + ] + data[truth_value][equation_len] = reduced_equations diff --git a/examples/hed/weights/all_weights_here.txt b/examples/hed/weights/all_weights_here.txt new file mode 100644 index 0000000..e69de29 diff --git a/examples/hwf/README.md b/examples/hwf/README.md new file mode 100644 index 0000000..4174c9d --- /dev/null +++ b/examples/hwf/README.md @@ -0,0 +1,44 @@ +# Handwritten Formula + +This example shows a simple implementation of [Handwritten Formula](https://arxiv.org/abs/2006.06649) task, where handwritten images of decimal formulas and their computed results are given, alongwith a domain knowledge base containing information on how to compute the decimal formula. The task is to recognize the symbols (which can be digits or operators '+', '-', '×', '÷') of handwritten images and accurately determine their results. + +## Run + +```bash +pip install -r requirements.txt +python main.py +``` + +## Usage + +```bash +usage: main.py [-h] [--no-cuda] [--epochs EPOCHS] [--lr LR] + [--batch-size BATCH_SIZE] + [--loops LOOPS] [--segment_size SEGMENT_SIZE] + [--save_interval SAVE_INTERVAL] [--max-revision MAX_REVISION] + [--require-more-revision REQUIRE_MORE_REVISION] + [--ground] [--max-err MAX_ERR] + +Handwritten Formula example + +optional arguments: + -h, --help show this help message and exit + --no-cuda disables CUDA training + --epochs EPOCHS number of epochs in each learning loop iteration + (default : 1) + --lr LR base model learning rate (default : 0.001) + --batch-size BATCH_SIZE + base model batch size (default : 32) + --loops LOOPS number of loop iterations (default : 5) + --segment_size SEGMENT_SIZE + segment size (default : 1/3) + --save_interval SAVE_INTERVAL + save interval (default : 1) + --max-revision MAX_REVISION + maximum revision in reasoner (default : -1) + --require-more-revision REQUIRE_MORE_REVISION + require more revision in reasoner (default : 0) + --ground use GroundKB (default: False) + --max-err MAX_ERR max tolerance during abductive reasoning (default : 1e-10) + +``` diff --git a/examples/hwf/datasets/__init__.py b/examples/hwf/datasets/__init__.py new file mode 100644 index 0000000..5d3ddab --- /dev/null +++ b/examples/hwf/datasets/__init__.py @@ -0,0 +1,3 @@ +from .get_dataset import get_dataset + +__all__ = ["get_dataset"] diff --git a/examples/hwf/datasets/get_dataset.py b/examples/hwf/datasets/get_dataset.py new file mode 100644 index 0000000..f8ce374 --- /dev/null +++ b/examples/hwf/datasets/get_dataset.py @@ -0,0 +1,67 @@ +import json +import os +import zipfile + +import gdown +from PIL import Image +from torchvision.transforms import transforms + +CURRENT_DIR = os.path.abspath(os.path.dirname(__file__)) + +img_transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1,))]) + + +def download_and_unzip(url, zip_file_name): + try: + gdown.download(url, zip_file_name) + with zipfile.ZipFile(zip_file_name, "r") as zip_ref: + zip_ref.extractall(CURRENT_DIR) + os.remove(zip_file_name) + except Exception as e: + if os.path.exists(zip_file_name): + os.remove(zip_file_name) + raise Exception( + f"An error occurred during download or unzip: {e}. Instead, you can download " + + f"the dataset from {url} and unzip it in 'examples/hwf/datasets' folder" + ) + + +def get_dataset(train=True, get_pseudo_label=False): + data_dir = CURRENT_DIR + "/data" + + if not os.path.exists(data_dir): + print("Dataset not exist, downloading it...") + url = "https://drive.google.com/u/0/uc?id=1G07kw-wK-rqbg_85tuB7FNfA49q8lvoy&export=download" + download_and_unzip(url, os.path.join(CURRENT_DIR, "HWF.zip")) + print("Download and extraction complete.") + + if train: + file = os.path.join(data_dir, "expr_train.json") + else: + file = os.path.join(data_dir, "expr_test.json") + + X = [] + pseudo_label = [] if get_pseudo_label else None + Y = [] + img_dir = os.path.join(CURRENT_DIR, "data/Handwritten_Math_Symbols/") + with open(file) as f: + data = json.load(f) + for idx in range(len(data)): + imgs = [] + if get_pseudo_label: + imgs_pseudo_label = [] + for img_path in data[idx]["img_paths"]: + img = Image.open(img_dir + img_path).convert("L") + img = img_transform(img) + imgs.append(img) + if get_pseudo_label: + label_mappings = {"times": "*", "div": "/"} + label = img_path.split("/")[0] + label = label_mappings.get(label, label) + imgs_pseudo_label.append(label) + X.append(imgs) + if get_pseudo_label: + pseudo_label.append(imgs_pseudo_label) + Y.append(data[idx]["res"]) + + return X, pseudo_label, Y diff --git a/examples/hwf/hwf.ipynb b/examples/hwf/hwf.ipynb new file mode 100644 index 0000000..db140f2 --- /dev/null +++ b/examples/hwf/hwf.ipynb @@ -0,0 +1,454 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Handwritten Formula (HWF)\n", + "\n", + "This notebook shows an implementation of [Handwritten Formula](https://arxiv.org/abs/2006.06649). In this task, handwritten images of decimal formulas and their computed results are given, alongwith a domain knowledge base containing information on how to compute the decimal formula. The task is to recognize the symbols (which can be digits or operators '+', '-', '×', '÷') of handwritten images and accurately determine their results.\n", + "\n", + "Intuitively, we first use a machine learning model (learning part) to convert the input images to symbols (we call them pseudo-labels), and then use the knowledge base (reasoning part) to calculate the results of these symbols. Since we do not have ground-truth of the symbols, in Abductive Learning, the reasoning part will leverage domain knowledge and revise the initial symbols yielded by the learning part through abductive reasoning. This process enables us to further update the machine learning model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Import necessary libraries and modules\n", + "import os.path as osp\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import torch\n", + "import torch.nn as nn\n", + "\n", + "from abl.bridge import SimpleBridge\n", + "from abl.data.evaluation import ReasoningMetric, SymbolAccuracy\n", + "from abl.learning import ABLModel, BasicNN\n", + "from abl.reasoning import KBBase, Reasoner\n", + "from abl.utils import ABLLogger, print_log\n", + "\n", + "from datasets import get_dataset\n", + "from models.nn import SymbolNet" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Working with Data\n", + "\n", + "First, we get the training and testing datasets:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "train_data = get_dataset(train=True, get_pseudo_label=True)\n", + "test_data = get_dataset(train=False, get_pseudo_label=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Both `train_data` and `test_data` have the same structures: tuples with three components: X (list where each element is a list of images), gt_pseudo_label (list where each element is a list of symbols, i.e., pseudo-labels) and Y (list where each element is the computed result). The length and structures of datasets are illustrated as follows.\n", + "\n", + "Note: ``gt_pseudo_label`` is only used to evaluate the performance of the learning part but not to train the model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(f\"Both train_data and test_data consist of 3 components: X, gt_pseudo_label, Y\")\n", + "print()\n", + "train_X, train_gt_pseudo_label, train_Y = train_data\n", + "print(\n", + " f\"Length of X, gt_pseudo_label, Y in train_data: \"\n", + " + f\"{len(train_X)}, {len(train_gt_pseudo_label)}, {len(train_Y)}\"\n", + ")\n", + "test_X, test_gt_pseudo_label, test_Y = test_data\n", + "print(\n", + " f\"Length of X, gt_pseudo_label, Y in test_data: \"\n", + " + f\"{len(test_X)}, {len(test_gt_pseudo_label)}, {len(test_Y)}\"\n", + ")\n", + "print()\n", + "\n", + "X_0, gt_pseudo_label_0, Y_0 = train_X[0], train_gt_pseudo_label[0], train_Y[0]\n", + "print(\n", + " f\"X is a {type(train_X).__name__}, \"\n", + " + f\"with each element being a {type(X_0).__name__} of {type(X_0[0]).__name__}.\"\n", + ")\n", + "print(\n", + " f\"gt_pseudo_label is a {type(train_gt_pseudo_label).__name__}, \"\n", + " + f\"with each element being a {type(gt_pseudo_label_0).__name__} \"\n", + " + f\"of {type(gt_pseudo_label_0[0]).__name__}.\"\n", + ")\n", + "print(f\"Y is a {type(train_Y).__name__}, \" + f\"with each element being a {type(Y_0).__name__}.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The ith element of X, gt_pseudo_label, and Y together constitute the ith data example. Here we use two of them (the 1001st and the 3001st) as illstrations:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "X_1000, gt_pseudo_label_1000, Y_1000 = train_X[1000], train_gt_pseudo_label[1000], train_Y[1000]\n", + "print(f\"X in the 1001st data example (a list of images):\")\n", + "for i, x in enumerate(X_1000):\n", + " plt.subplot(1, len(X_1000), i + 1)\n", + " plt.axis(\"off\")\n", + " plt.imshow(x.squeeze(), cmap=\"gray\")\n", + "plt.show()\n", + "print(\n", + " f\"gt_pseudo_label in the 1001st data example (a list of ground truth pseudo-labels): {gt_pseudo_label_1000}\"\n", + ")\n", + "print(f\"Y in the 1001st data example (the computed result): {Y_1000}\")\n", + "print()\n", + "X_3000, gt_pseudo_label_3000, Y_3000 = train_X[3000], train_gt_pseudo_label[3000], train_Y[3000]\n", + "print(f\"X in the 3001st data example (a list of images):\")\n", + "for i, x in enumerate(X_3000):\n", + " plt.subplot(1, len(X_3000), i + 1)\n", + " plt.axis(\"off\")\n", + " plt.imshow(x.squeeze(), cmap=\"gray\")\n", + "plt.show()\n", + "print(\n", + " f\"gt_pseudo_label in the 3001st data example (a list of ground truth pseudo-labels): {gt_pseudo_label_3000}\"\n", + ")\n", + "print(f\"Y in the 3001st data example (the computed result): {Y_3000}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note: The symbols in the HWF dataset can be one of digits or operators '+', '-', '×', '÷'. \n", + "\n", + "Note: We may see that, in the 1001st data example, the length of the formula is 3, while in the 3001st data example, the length of the formula is 5. In the HWF dataset, the length of the formula varies from 1 to 7." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Building the Learning Part" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To build the learning part, we need to first build a machine learning base model. We use SymbolNet, and encapsulate it within a `BasicNN` object to create the base model. `BasicNN` is a class that encapsulates a PyTorch model, transforming it into a base model with an sklearn-style interface. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# class of symbol may be one of ['1', ..., '9', '+', '-', '*', '/'], total of 13 classes\n", + "cls = SymbolNet(num_classes=13, image_size=(45, 45, 1))\n", + "loss_fn = nn.CrossEntropyLoss()\n", + "optimizer = torch.optim.Adam(cls.parameters(), lr=0.001)\n", + "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n", + "\n", + "base_model = BasicNN(\n", + " model=cls,\n", + " loss_fn=loss_fn,\n", + " optimizer=optimizer,\n", + " device=device,\n", + " batch_size=128,\n", + " num_epochs=3,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`BasicNN` offers methods like `predict` and `predict_prob`, which are used to predict the class index and the probabilities of each class for images. As shown below:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "data_instances = [torch.randn(1, 45, 45).to(device) for _ in range(32)]\n", + "pred_idx = base_model.predict(X=data_instances)\n", + "print(\n", + " f\"Predicted class index for a batch of 32 instances: \"\n", + " + f\"{type(pred_idx).__name__} with shape {pred_idx.shape}\"\n", + ")\n", + "pred_prob = base_model.predict_proba(X=data_instances)\n", + "print(\n", + " f\"Predicted class probabilities for a batch of 32 instances: \"\n", + " + f\"{type(pred_prob).__name__} with shape {pred_prob.shape}\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "However, the base model built above deals with instance-level data (i.e., individual images), and can not directly deal with example-level data (i.e., a list of images comprising the formula). Therefore, we wrap the base model into `ABLModel`, which enables the learning part to train, test, and predict on example-level data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model = ABLModel(base_model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As an illustration, consider this example of training on example-level data using the `predict` method in `ABLModel`. In this process, the method accepts data examples as input and outputs the class labels and the probabilities of each class for all instances within these data examples." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from abl.data.structures import ListData\n", + "\n", + "# ListData is a data structure provided by ABL-Package that can be used to organize data examples\n", + "data_examples = ListData()\n", + "# We use the first 1001st and 3001st data examples in the training set as an illustration\n", + "data_examples.X = [X_1000, X_3000]\n", + "data_examples.gt_pseudo_label = [gt_pseudo_label_1000, gt_pseudo_label_3000]\n", + "data_examples.Y = [Y_1000, Y_3000]\n", + "\n", + "# Perform prediction on the two data examples\n", + "# Remind that, in the 1001st data example, the length of the formula is 3,\n", + "# while in the 3001st data example, the length of the formula is 5.\n", + "pred_label, pred_prob = model.predict(data_examples)[\"label\"], model.predict(data_examples)[\"prob\"]\n", + "print(\n", + " f\"Predicted class labels for the 100 data examples: a list of length {len(pred_label)}, \\n\"\n", + " + f\"the first element is a {type(pred_label[0]).__name__} of shape {pred_label[0].shape}, \"\n", + " + f\"and the second element is a {type(pred_label[1]).__name__} of shape {pred_label[1].shape}.\\n\"\n", + ")\n", + "print(\n", + " f\"Predicted class probabilities for the 100 data examples: a list of length {len(pred_prob)}, \\n\"\n", + " f\"the first element is a {type(pred_prob[0]).__name__} of shape {pred_prob[0].shape}, \"\n", + " + f\"and the second element is a {type(pred_prob[1]).__name__} of shape {pred_prob[1].shape}.\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Building the Reasoning Part" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the reasoning part, we first build a knowledge base which contain information on how to perform addition operations. We build it by creating a subclass of `KBBase`. In the derived subclass, we initialize the `pseudo_label_list` parameter specifying list of possible pseudo-labels, and override the `logic_forward` function defining how to perform (deductive) reasoning." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class HwfKB(KBBase):\n", + " def __init__(\n", + " self, pseudo_label_list=[\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"+\", \"-\", \"*\", \"/\"]\n", + " ):\n", + " super().__init__(pseudo_label_list)\n", + "\n", + " def _valid_candidate(self, formula):\n", + " if len(formula) % 2 == 0:\n", + " return False\n", + " for i in range(len(formula)):\n", + " if i % 2 == 0 and formula[i] not in [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\"]:\n", + " return False\n", + " if i % 2 != 0 and formula[i] not in [\"+\", \"-\", \"*\", \"/\"]:\n", + " return False\n", + " return True\n", + "\n", + " # Implement the deduction function\n", + " def logic_forward(self, formula):\n", + " if not self._valid_candidate(formula):\n", + " return np.inf\n", + " return eval(\"\".join(formula))\n", + "\n", + "\n", + "kb = HwfKB()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The knowledge base can perform logical reasoning (both deductive reasoning and abductive reasoning). Below is an example of performing (deductive) reasoning, and users can refer to [Documentation]() for details of abductive reasoning." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pseudo_labels = [\"1\", \"-\", \"2\", \"*\", \"5\"]\n", + "reasoning_result = kb.logic_forward(pseudo_labels)\n", + "print(f\"Reasoning result of pseudo-labels {pseudo_labels} is {reasoning_result}.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note: In addition to building a knowledge base based on `KBBase`, we can also establish a knowledge base with a ground KB using `GroundKB`. The corresponding code can be found in the `main.py` file. Those interested are encouraged to examine it for further insights.\n", + "\n", + "Note: Also, when building the knowledge base, we can also set the `max_err` parameter during initialization, which is shown in the `main.py` file. This parameter specifies the upper tolerance limit when comparing the similarity between the reasoning result of pseudo-labels and the ground truth during abductive reasoning, with a default value of 1e-10." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then, we create a reasoner by instantiating the class ``Reasoner``. Due to the indeterminism of abductive reasoning, there could be multiple candidates compatible to the knowledge base. When this happens, reasoner can minimize inconsistencies between the knowledge base and pseudo-labels predicted by the learning part, and then return only one candidate that has the highest consistency." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "reasoner = Reasoner(kb)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note: During creating reasoner, the definition of \"consistency\" can be customized within the `dist_func` parameter. In the code above, we employ a consistency measurement based on confidence, which calculates the consistency between the data example and candidates based on the confidence derived from the predicted probability. In `main.py`, we provide options for utilizing other forms of consistency measurement.\n", + "\n", + "Note: Also, during process of inconsistency minimization, we can leverage [ZOOpt library](https://github.com/polixir/ZOOpt) for acceleration. Options for this are also available in `main.py`. Those interested are encouraged to explore these features." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Building Evaluation Metrics" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, we set up evaluation metrics. These metrics will be used to evaluate the model performance during training and testing. Specifically, we use `SymbolAccuracy` and `ReasoningMetric`, which are used to evaluate the accuracy of the machine learning model’s predictions and the accuracy of the final reasoning results, respectively." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metric_list = [SymbolAccuracy(prefix=\"hwf\"), ReasoningMetric(kb=kb, prefix=\"hwf\")]" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Bridge Learning and Reasoning\n", + "\n", + "Now, the last step is to bridge the learning and reasoning part. We proceed this step by creating an instance of `SimpleBridge`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "bridge = SimpleBridge(model, reasoner, metric_list)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Perform training and testing by invoking the `train` and `test` methods of `SimpleBridge`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Build logger\n", + "print_log(\"Abductive Learning on the HWF example.\", logger=\"current\")\n", + "log_dir = ABLLogger.get_current_instance().log_dir\n", + "weights_dir = osp.join(log_dir, \"weights\")\n", + "\n", + "bridge.train(train_data, train_data, loops=3, segment_size=1000, save_dir=weights_dir)\n", + "bridge.test(test_data)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "abl", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.18" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "9c8d454494e49869a4ee4046edcac9a39ff683f7d38abf0769f648402670238e" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/hwf/main.py b/examples/hwf/main.py new file mode 100644 index 0000000..f8e10d9 --- /dev/null +++ b/examples/hwf/main.py @@ -0,0 +1,187 @@ +import argparse +import os.path as osp + +import numpy as np +import torch +from torch import nn + +from abl.bridge import SimpleBridge +from abl.data.evaluation import ReasoningMetric, SymbolAccuracy +from abl.learning import ABLModel, BasicNN +from abl.reasoning import GroundKB, KBBase, Reasoner +from abl.utils import ABLLogger, print_log + +from datasets import get_dataset +from models.nn import SymbolNet + + +class HwfKB(KBBase): + def __init__( + self, + pseudo_label_list=["1", "2", "3", "4", "5", "6", "7", "8", "9", "+", "-", "*", "/"], + max_err=1e-10, + ): + super().__init__(pseudo_label_list, max_err) + + def _valid_candidate(self, formula): + if len(formula) % 2 == 0: + return False + for i in range(len(formula)): + if i % 2 == 0 and formula[i] not in ["1", "2", "3", "4", "5", "6", "7", "8", "9"]: + return False + if i % 2 != 0 and formula[i] not in ["+", "-", "*", "/"]: + return False + return True + + # Implement the deduction function + def logic_forward(self, formula): + if not self._valid_candidate(formula): + return np.inf + return eval("".join(formula)) + + +class HwfGroundKB(GroundKB): + def __init__( + self, + pseudo_label_list=["1", "2", "3", "4", "5", "6", "7", "8", "9", "+", "-", "*", "/"], + GKB_len_list=[1, 3, 5, 7], + max_err=1e-10, + ): + super().__init__(pseudo_label_list, GKB_len_list, max_err) + + def _valid_candidate(self, formula): + if len(formula) % 2 == 0: + return False + for i in range(len(formula)): + if i % 2 == 0 and formula[i] not in ["1", "2", "3", "4", "5", "6", "7", "8", "9"]: + return False + if i % 2 != 0 and formula[i] not in ["+", "-", "*", "/"]: + return False + return True + + # Implement the deduction function + def logic_forward(self, formula): + if not self._valid_candidate(formula): + return np.inf + return eval("".join(formula)) + + +def main(): + parser = argparse.ArgumentParser(description="Handwritten Formula example") + parser.add_argument( + "--no-cuda", action="store_true", default=False, help="disables CUDA training" + ) + parser.add_argument( + "--epochs", + type=int, + default=3, + help="number of epochs in each learning loop iteration (default : 3)", + ) + parser.add_argument( + "--lr", type=float, default=1e-3, help="base model learning rate (default : 0.001)" + ) + parser.add_argument( + "--batch-size", type=int, default=128, help="base model batch size (default : 128)" + ) + parser.add_argument( + "--loops", type=int, default=5, help="number of loop iterations (default : 5)" + ) + parser.add_argument( + "--segment_size", type=int, default=1000, help="segment size (default : 1000)" + ) + parser.add_argument("--save_interval", type=int, default=1, help="save interval (default : 1)") + parser.add_argument( + "--max-revision", + type=int, + default=-1, + help="maximum revision in reasoner (default : -1)", + ) + parser.add_argument( + "--require-more-revision", + type=int, + default=0, + help="require more revision in reasoner (default : 0)", + ) + parser.add_argument( + "--ground", action="store_true", default=False, help="use GroundKB (default: False)" + ) + parser.add_argument( + "--max-err", + type=float, + default=1e-10, + help="max tolerance during abductive reasoning (default : 1e-10)", + ) + + args = parser.parse_args() + + # Build logger + print_log("Abductive Learning on the HWF example.", logger="current") + + ### Working with Data + print_log("Working with Data.", logger="current") + + train_data = get_dataset(train=True, get_pseudo_label=True) + test_data = get_dataset(train=False, get_pseudo_label=True) + + ### Building the Learning Part + print_log("Building the Learning Part.", logger="current") + + # Build necessary components for BasicNN + cls = SymbolNet(num_classes=13, image_size=(45, 45, 1)) + loss_fn = nn.CrossEntropyLoss() + optimizer = torch.optim.Adam(cls.parameters(), lr=args.lr) + use_cuda = not args.no_cuda and torch.cuda.is_available() + device = torch.device("cuda" if use_cuda else "cpu") + + # Build BasicNN + base_model = BasicNN( + cls, + loss_fn, + optimizer, + device=device, + batch_size=args.batch_size, + num_epochs=args.epochs, + ) + + # Build ABLModel + model = ABLModel(base_model) + + ### Building the Reasoning Part + print_log("Building the Reasoning Part.", logger="current") + + # Build knowledge base + if args.ground: + kb = HwfGroundKB() + else: + kb = HwfKB() + + # Create reasoner + reasoner = Reasoner( + kb, max_revision=args.max_revision, require_more_revision=args.require_more_revision + ) + + ### Building Evaluation Metrics + print_log("Building Evaluation Metrics.", logger="current") + metric_list = [SymbolAccuracy(prefix="hwf"), ReasoningMetric(kb=kb, prefix="hwf")] + + ### Bridge Learning and Reasoning + print_log("Bridge Learning and Reasoning.", logger="current") + bridge = SimpleBridge(model, reasoner, metric_list) + + # Retrieve the directory of the Log file and define the directory for saving the model weights. + log_dir = ABLLogger.get_current_instance().log_dir + weights_dir = osp.join(log_dir, "weights") + + # Train and Test + bridge.train( + train_data, + loops=args.loops, + segment_size=args.segment_size, + save_interval=args.save_interval, + save_dir=weights_dir, + ) + bridge.test(test_data) + + +if __name__ == "__main__": + main() diff --git a/examples/hwf/models/nn.py b/examples/hwf/models/nn.py new file mode 100644 index 0000000..875283f --- /dev/null +++ b/examples/hwf/models/nn.py @@ -0,0 +1,33 @@ +import torch +from torch import nn + + +class SymbolNet(nn.Module): + def __init__(self, num_classes=4, image_size=(28, 28, 1)): + super(SymbolNet, self).__init__() + self.conv1 = nn.Sequential( + nn.Conv2d(1, 32, 5, stride=1), + nn.ReLU(), + nn.MaxPool2d(kernel_size=2, stride=2), + nn.BatchNorm2d(32, momentum=0.99, eps=0.001), + ) + self.conv2 = nn.Sequential( + nn.Conv2d(32, 64, 5, padding=2, stride=1), + nn.ReLU(), + nn.MaxPool2d(kernel_size=2, stride=2), + nn.BatchNorm2d(64, momentum=0.99, eps=0.001), + ) + + num_features = 64 * (image_size[0] // 4 - 1) * (image_size[1] // 4 - 1) + self.fc1 = nn.Sequential(nn.Linear(num_features, 120), nn.ReLU()) + self.fc2 = nn.Sequential(nn.Linear(120, 84), nn.ReLU()) + self.fc3 = nn.Sequential(nn.Linear(84, num_classes)) + + def forward(self, x): + x = self.conv1(x) + x = self.conv2(x) + x = torch.flatten(x, 1) + x = self.fc1(x) + x = self.fc2(x) + x = self.fc3(x) + return x diff --git a/examples/hwf/requirements.txt b/examples/hwf/requirements.txt new file mode 100644 index 0000000..ee7cc5e --- /dev/null +++ b/examples/hwf/requirements.txt @@ -0,0 +1,3 @@ +abl +gdown +matplotlib \ No newline at end of file diff --git a/examples/hwf/weights/all_weights_here.txt b/examples/hwf/weights/all_weights_here.txt new file mode 100644 index 0000000..e69de29 diff --git a/examples/mnist_add/README.md b/examples/mnist_add/README.md new file mode 100644 index 0000000..98254ab --- /dev/null +++ b/examples/mnist_add/README.md @@ -0,0 +1,45 @@ +# MNIST Addition + +This example shows a simple implementation of [MNIST Addition](https://arxiv.org/abs/1805.10872) task, where pairs of MNIST handwritten images and their sums are given, alongwith a domain knowledge base containing information on how to perform addition operations. The task is to recognize the digits of handwritten images and accurately determine their sum. + +## Run + +```bash +pip install -r requirements.txt +python main.py +``` + +## Usage + +```bash +usage: main.py [-h] [--no-cuda] [--epochs EPOCHS] [--lr LR] + [--alpha ALPHA] [--batch-size BATCH_SIZE] + [--loops LOOPS] [--segment_size SEGMENT_SIZE] + [--save_interval SAVE_INTERVAL] [--max-revision MAX_REVISION] + [--require-more-revision REQUIRE_MORE_REVISION] + [--prolog | --ground] + +MNIST Addition example + +optional arguments: + -h, --help show this help message and exit + --no-cuda disables CUDA training + --epochs EPOCHS number of epochs in each learning loop iteration + (default : 1) + --lr LR base model learning rate (default : 0.001) + --alpha ALPHA alpha in RMSprop (default : 0.9) + --batch-size BATCH_SIZE + base model batch size (default : 32) + --loops LOOPS number of loop iterations (default : 5) + --segment_size SEGMENT_SIZE + segment size (default : 1/3) + --save_interval SAVE_INTERVAL + save interval (default : 1) + --max-revision MAX_REVISION + maximum revision in reasoner (default : -1) + --require-more-revision REQUIRE_MORE_REVISION + require more revision in reasoner (default : 0) + --prolog use PrologKB (default: False) + --ground use GroundKB (default: False) + +``` diff --git a/examples/mnist_add/add.pl b/examples/mnist_add/add.pl new file mode 100644 index 0000000..96f0869 --- /dev/null +++ b/examples/mnist_add/add.pl @@ -0,0 +1,2 @@ +pseudo_label(N) :- between(0, 9, N). +logic_forward([Z1, Z2], Res) :- pseudo_label(Z1), pseudo_label(Z2), Res is Z1+Z2. diff --git a/examples/mnist_add/datasets/__init__.py b/examples/mnist_add/datasets/__init__.py new file mode 100644 index 0000000..5d3ddab --- /dev/null +++ b/examples/mnist_add/datasets/__init__.py @@ -0,0 +1,3 @@ +from .get_dataset import get_dataset + +__all__ = ["get_dataset"] diff --git a/examples/mnist_add/datasets/get_dataset.py b/examples/mnist_add/datasets/get_dataset.py new file mode 100644 index 0000000..bfa7b93 --- /dev/null +++ b/examples/mnist_add/datasets/get_dataset.py @@ -0,0 +1,31 @@ +import os + +import torchvision +from torchvision.transforms import transforms + +CURRENT_DIR = os.path.abspath(os.path.dirname(__file__)) + + +def get_dataset(train=True, get_pseudo_label=True): + transform = transforms.Compose( + [transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))] + ) + img_dataset = torchvision.datasets.MNIST( + root=CURRENT_DIR, train=train, download=True, transform=transform + ) + if train: + file = os.path.join(CURRENT_DIR, "train_data.txt") + else: + file = os.path.join(CURRENT_DIR, "test_data.txt") + + X = [] + pseudo_label = [] if get_pseudo_label else None + Y = [] + with open(file) as f: + for line in f: + x1, x2, y = map(int, line.strip().split(" ")) + X.append([img_dataset[x1][0], img_dataset[x2][0]]) + if get_pseudo_label: + pseudo_label.append([img_dataset[x1][1], img_dataset[x2][1]]) + Y.append(y) + return X, pseudo_label, Y diff --git a/examples/mnist_add/datasets/test_data.txt b/examples/mnist_add/datasets/test_data.txt new file mode 100644 index 0000000..3987383 --- /dev/null +++ b/examples/mnist_add/datasets/test_data.txt @@ -0,0 +1,5000 @@ +5236 2931 11 +7455 7918 13 +7743 9396 6 +8167 8516 9 +3767 674 12 +4307 4808 8 +1567 712 12 +2456 1692 9 +6328 7863 9 +3976 3765 8 +6323 7115 6 +1815 1123 10 +5468 4196 8 +5831 6752 1 +9228 6807 14 +5915 6729 2 +8708 5377 10 +8273 296 0 +7748 4928 4 +6034 6210 12 +8180 2248 7 +3120 3309 10 +9020 8743 12 +5098 377 6 +2250 6812 13 +3914 6356 7 +2702 5181 9 +5696 2223 8 +2195 180 8 +2781 5425 16 +7979 5795 15 +4131 6977 10 +8991 1086 9 +1079 951 11 +661 6684 7 +3787 7889 3 +9057 6200 18 +9940 2852 14 +2737 510 13 +272 6374 2 +5885 4255 10 +1589 4594 2 +5833 773 14 +2424 6619 10 +4735 9625 13 +6154 7079 15 +4230 1377 14 +5109 1627 9 +4305 9508 8 +6026 2372 14 +57 8286 2 +4416 9463 13 +5657 994 8 +2981 4606 3 +844 8821 11 +6753 123 6 +1208 6162 4 +4224 4101 18 +9620 6451 18 +1694 6245 16 +1839 4866 6 +4014 4947 10 +3118 6126 15 +5358 9134 3 +7338 7093 6 +4294 412 14 +8521 2730 9 +668 7959 10 +1449 6344 9 +6124 8961 13 +4792 8220 12 +8129 5908 11 +924 3021 4 +8036 2548 11 +4188 5587 9 +352 7596 11 +2650 3884 17 +7914 4714 9 +2550 7217 9 +6392 3430 6 +2646 6708 14 +7886 6113 8 +3214 2406 10 +4643 4611 1 +8380 869 11 +888 5329 5 +9881 1360 4 +8846 7688 9 +7544 3992 16 +4160 7801 13 +7171 5346 4 +2243 1519 10 +902 1143 11 +6803 5559 9 +9884 9667 15 +1654 7052 2 +7920 1158 5 +3363 3410 7 +9628 3453 14 +5068 6175 8 +3856 9818 9 +5802 4516 6 +8283 2309 8 +7869 104 10 +6402 2619 6 +2159 3962 8 +8761 3023 13 +8346 7497 5 +4870 3192 9 +6220 604 9 +5352 1175 16 +3829 6565 17 +1407 6472 3 +430 9987 5 +432 7073 5 +8268 9496 9 +5841 3554 9 +7242 9323 16 +8340 522 8 +4132 9330 14 +7574 6697 11 +4249 6726 2 +1757 2807 3 +5975 5451 9 +1052 7172 14 +9132 1564 12 +3112 4988 17 +2025 4231 11 +3182 7964 15 +7012 3542 6 +964 1069 4 +4094 3822 11 +9439 8595 0 +8422 9140 10 +6333 6012 6 +2626 4769 10 +2470 2927 11 +8978 0 7 +3858 3994 6 +6825 2344 14 +2322 7 14 +5219 2610 11 +7457 5752 7 +4010 5689 2 +2964 7951 17 +9891 4938 16 +7441 5229 11 +177 2536 8 +3751 2137 8 +740 1611 7 +4269 4415 6 +4683 9580 18 +1852 1321 13 +1957 7646 2 +791 2262 12 +1454 1087 5 +9459 6916 9 +262 598 16 +8440 7520 12 +4073 3672 11 +3327 2564 9 +1883 1769 10 +2375 8760 10 +3983 109 5 +9980 3147 11 +5835 8878 12 +2818 9638 13 +4368 4371 13 +7601 1008 1 +9669 6731 11 +6349 5205 10 +3274 3048 4 +2181 4721 13 +6389 2750 9 +8003 224 8 +468 2247 12 +277 1792 15 +5473 792 8 +1470 6188 15 +5837 4173 11 +8364 6004 15 +4216 2045 4 +7535 1652 13 +6046 4534 12 +3710 2537 4 +2366 4767 8 +7349 1671 9 +7068 6498 12 +477 6041 4 +8191 5433 14 +3109 5907 9 +5905 8939 11 +8212 5707 9 +3134 2509 6 +9573 759 16 +5300 3606 9 +4487 1712 7 +3104 9386 12 +7113 5523 15 +2601 5577 10 +9050 1091 13 +1278 7566 3 +3228 3916 7 +3243 9334 6 +8406 3745 5 +2806 6020 10 +6110 5791 1 +2008 6189 11 +6542 2302 4 +8593 1935 16 +8941 5669 4 +6420 2559 14 +2175 6294 7 +928 5399 10 +1900 4837 8 +8141 2549 7 +2782 8189 10 +7200 7547 14 +9655 6628 4 +163 9126 12 +7797 1194 12 +5080 102 14 +7434 4729 5 +4856 9835 13 +2812 2953 12 +9405 5866 7 +6827 6183 11 +2882 2307 8 +3763 98 11 +4158 172 11 +4024 8345 6 +6808 2749 7 +1183 1573 17 +3426 2907 13 +1695 7185 12 +6354 5404 15 +5241 6028 5 +7210 972 6 +932 6929 2 +7636 2158 3 +3885 2467 17 +5137 6742 13 +875 4831 8 +8228 4483 4 +5228 5319 10 +5991 5318 2 +8005 2228 2 +7899 7942 10 +9810 1791 2 +8145 8176 12 +2330 6422 13 +1220 7925 6 +5307 1333 0 +3954 9642 17 +1472 5680 11 +7937 2303 7 +4082 149 2 +3446 1460 5 +8584 3369 11 +8109 1670 14 +5153 3160 15 +2433 8200 8 +7261 1493 5 +3318 4899 4 +4922 2190 8 +8138 4107 8 +9523 8507 10 +5510 8234 12 +1838 4910 10 +1326 3552 12 +1717 2911 16 +7908 7202 7 +8170 3368 5 +2726 8874 9 +8977 5206 14 +1778 8626 8 +1381 1509 6 +3807 7140 11 +6633 956 1 +4738 2747 4 +9073 5396 9 +8260 24 4 +9183 4658 11 +1093 7010 11 +1480 9498 3 +3173 3815 7 +1276 5927 11 +364 6050 12 +7706 9817 8 +3982 255 7 +4474 94 8 +9762 1899 11 +2048 132 12 +2499 3414 9 +8557 6272 12 +8361 9739 2 +790 3702 6 +47 3179 2 +8272 2007 7 +2177 201 14 +4875 1309 15 +6912 4380 10 +5534 337 8 +5963 9434 7 +7644 6520 12 +9824 4565 14 +8686 8697 11 +1217 7949 15 +6851 6985 11 +423 236 4 +2082 9082 4 +5388 5351 14 +5728 4377 11 +2849 1249 16 +2153 2924 6 +5574 267 6 +6683 7827 10 +9346 6907 5 +3065 7807 17 +7871 823 10 +6939 9722 10 +7105 2866 12 +6695 3463 8 +2013 2694 10 +1746 1764 3 +9401 4745 5 +5216 3093 7 +2232 8502 14 +3838 6112 16 +1318 9979 8 +8728 697 12 +663 6778 10 +3051 3433 7 +6698 2206 6 +541 51 7 +6568 2639 12 +2599 1142 5 +6712 1176 1 +846 6450 15 +5140 5612 12 +5390 7205 4 +3188 5544 5 +1774 7817 8 +6084 9313 3 +9357 7661 10 +273 7404 15 +4098 5217 4 +8670 899 17 +7333 4475 7 +7390 8777 10 +8318 2135 8 +3299 7778 9 +349 121 7 +5932 7436 3 +2905 4907 11 +870 4426 15 +9444 3346 1 +8526 5762 2 +11 8063 12 +6485 2056 16 +1253 7001 6 +7875 9992 12 +5148 6051 3 +8727 9080 7 +7136 9194 14 +7005 3576 9 +4422 6784 14 +8844 3263 10 +8411 7861 10 +836 5466 8 +9234 3386 6 +1759 7383 16 +2616 9844 6 +5037 6992 9 +1661 2256 7 +5486 6790 4 +2825 8912 9 +5237 4016 12 +1300 4568 11 +8798 1522 7 +2984 5314 2 +5437 6332 11 +7588 3650 9 +7098 3846 13 +1534 2324 1 +4411 9211 12 +7848 999 11 +9196 3044 7 +8575 7099 9 +4115 7189 9 +9574 8356 15 +4272 3168 18 +2600 6310 9 +1884 6740 10 +1411 9630 8 +6248 3907 13 +4579 4580 7 +8368 2979 15 +1741 9984 8 +3756 4378 10 +1870 9634 0 +361 884 6 +7103 9558 6 +6037 6917 5 +2461 1056 2 +6448 2881 7 +4636 1213 3 +1292 2112 8 +2032 9857 11 +9575 5684 8 +9938 6623 7 +3547 4760 9 +9870 5330 8 +6101 1780 2 +9483 976 12 +8227 9878 10 +7158 6819 7 +4751 2784 6 +5551 7211 10 +8301 1782 16 +8573 8940 12 +3835 4612 13 +5321 4500 9 +9191 2597 7 +601 3385 17 +7639 4891 3 +5191 2087 4 +5243 2063 7 +7965 2464 9 +7990 4710 3 +2891 8174 4 +2748 7260 7 +8455 9111 9 +8849 5447 9 +160 1114 7 +1067 3819 7 +6799 5602 10 +9353 5764 7 +5786 3777 2 +4096 403 16 +5119 1943 8 +4136 8215 11 +2508 9997 10 +8112 4939 4 +4508 5252 9 +5771 1508 17 +1344 2529 7 +7606 2753 8 +3909 9482 7 +8712 9542 2 +1033 8211 14 +5536 9239 2 +7617 9636 3 +3932 293 4 +7793 8168 7 +5661 7966 8 +9418 6786 8 +2757 5740 4 +4238 2146 15 +4150 4924 1 +3628 4850 10 +7764 7773 6 +5088 3497 10 +3437 8385 10 +2797 5670 12 +6228 3573 16 +4349 8514 1 +878 7732 13 +5177 8554 13 +7972 7815 6 +8470 89 3 +8102 9046 4 +1681 3420 4 +9761 1801 9 +9410 5941 17 +407 6746 5 +8250 6269 16 +1908 2889 10 +9548 592 3 +5895 83 9 +1248 7421 12 +4736 2098 9 +6331 4858 4 +3920 5507 9 +3076 4277 7 +1679 2101 6 +9054 196 10 +8833 173 10 +6591 5283 9 +9570 1809 8 +7780 43 2 +9079 4785 12 +2449 8544 2 +7551 3898 8 +4578 3039 8 +7950 8048 8 +7825 3700 11 +6089 7165 8 +496 4918 18 +1571 554 11 +1184 9321 10 +1038 7838 1 +9771 7394 11 +8709 1643 5 +564 8115 3 +799 7816 8 +8447 8720 6 +806 3034 17 +9802 3840 10 +8477 2052 17 +8580 3126 7 +2286 3500 10 +2472 408 12 +2260 4457 2 +2212 1705 16 +3988 9308 11 +6292 6891 8 +343 4836 6 +1685 726 13 +6435 6299 11 +20 225 11 +4672 9589 6 +2058 5335 6 +2046 4185 5 +5415 1286 8 +5334 8108 9 +9171 5008 1 +495 4332 12 +2421 8016 1 +5308 8449 3 +9391 6078 11 +9608 1749 9 +4418 6222 3 +3564 8506 15 +3744 6401 15 +2975 9028 17 +787 6748 9 +6668 1252 11 +1133 2268 14 +8713 528 6 +3602 993 6 +4971 3193 8 +2196 7070 17 +2039 621 14 +7940 8233 8 +9839 1807 2 +9764 7903 11 +4726 6192 9 +8457 861 15 +7129 4264 1 +5047 2047 10 +2308 3984 12 +3174 6806 12 +7273 167 9 +448 2630 13 +431 1440 12 +488 6867 13 +2586 6709 13 +1313 3222 10 +4720 202 1 +1684 6239 6 +2005 7449 4 +6621 2414 9 +2621 1539 5 +4613 5517 7 +3790 2563 7 +7900 9456 3 +1446 1085 8 +9282 9514 7 +8067 2841 7 +5631 8375 11 +2105 271 3 +2355 6955 9 +2218 2325 11 +4295 3421 8 +8331 7796 12 +44 942 9 +4436 8061 13 +8315 6384 4 +8801 8785 6 +4810 9988 10 +2544 6136 14 +3478 3282 13 +9480 424 2 +9861 8783 9 +3968 5477 6 +6148 4961 7 +8271 5636 5 +952 9825 15 +2528 7063 10 +8432 8059 3 +7804 681 12 +2386 4770 7 +3686 2090 7 +2864 5789 9 +1227 3942 2 +4106 7356 10 +6870 7363 9 +945 797 7 +2892 2754 15 +9015 9713 16 +5224 3357 7 +6941 35 6 +2234 1026 15 +401 5659 8 +9942 3329 10 +6488 2578 11 +1790 3053 7 +669 7741 7 +2685 1427 13 +5485 4718 7 +5395 1950 11 +8045 7557 10 +338 7924 15 +4337 4539 8 +4497 9269 14 +705 7042 13 +8737 138 11 +8230 8739 15 +6545 414 10 +9086 8692 7 +7011 7718 16 +8623 4965 11 +3251 7322 9 +1331 6670 6 +1854 5956 10 +7368 2916 9 +3467 619 10 +278 563 10 +8865 3197 8 +7106 1012 16 +4758 5105 5 +6357 8762 11 +126 1750 7 +6630 5355 10 +5688 4299 16 +2201 9205 12 +87 6894 3 +8732 7037 6 +6453 4012 7 +3809 410 8 +9349 6925 5 +3291 2016 11 +1903 4442 9 +9709 9695 13 +8150 3991 15 +1776 4823 16 +9225 556 6 +4697 767 8 +6317 8609 16 +2009 2764 9 +5782 1622 13 +8711 5260 9 +873 1946 10 +6447 7695 10 +4030 2577 16 +4800 986 13 +7540 5078 9 +8254 6905 12 +5650 6186 13 +3458 366 6 +6362 5607 8 +1441 6325 14 +3868 6441 15 +5492 4435 6 +3638 8581 4 +8673 9852 9 +8942 2490 15 +3515 9887 11 +2957 2148 9 +5550 3106 3 +5072 211 6 +1599 7771 7 +3721 9594 8 +8022 5256 5 +1369 2129 16 +2334 8274 16 +7380 3215 9 +923 1927 5 +4013 3507 10 +9796 8413 9 +5042 9019 16 +9719 3206 13 +9505 8796 16 +3170 7104 3 +4846 6298 15 +6234 735 3 +82 6031 8 +1514 269 2 +2977 4921 9 +2974 6660 5 +6044 8299 6 +6643 1498 7 +3239 3563 10 +3915 7782 4 +9742 6911 7 +3760 7245 11 +5255 6559 4 +5844 9040 12 +7373 3340 9 +3532 5445 6 +9718 1634 8 +6514 3272 9 +353 1734 13 +5881 3006 17 +6679 5223 12 +6833 2875 9 +3585 5979 11 +1608 9189 3 +4292 6642 10 +6486 816 7 +3766 7092 7 +5889 4830 6 +13 7572 2 +7110 9559 9 +3486 9975 6 +6963 4923 7 +3078 2942 3 +5448 5576 11 +2871 7144 10 +2615 7003 9 +713 5469 6 +385 6137 9 +9122 1558 6 +382 3108 6 +1193 2712 4 +8240 3726 10 +2341 7561 10 +749 4699 7 +8107 876 12 +7528 7849 5 +1327 5871 18 +8490 231 3 +3633 4355 13 +8594 6351 2 +7318 9707 12 +5757 8827 18 +2820 8467 16 +7425 5637 5 +2513 716 8 +8417 3011 14 +8294 1537 14 +7772 1605 5 +6693 4876 8 +124 3209 13 +8443 3794 9 +6055 8025 11 +330 732 7 +5380 5700 11 +5058 8438 9 +5959 6919 16 +8999 3557 6 +2017 8389 1 +9998 5588 6 +8682 7080 9 +9460 3967 12 +7219 8870 9 +1239 9479 6 +265 7094 9 +2582 7812 10 +9681 1877 16 +150 6049 17 +3105 7016 10 +3058 5978 7 +15 2510 6 +6246 2959 4 +1120 7880 14 +3014 5518 6 +3629 585 15 +1875 9414 7 +709 3857 17 +5385 8175 9 +393 5450 1 +8122 2819 9 +471 8341 15 +8714 8750 11 +8852 4818 9 +5248 7669 7 +2438 7413 13 +9841 6946 14 +2990 1989 12 +7678 4547 8 +1623 2337 6 +1524 4719 9 +8858 4716 2 +367 8489 8 +8767 5653 7 +1418 3404 18 +6300 3185 4 +8665 2257 14 +8303 5706 6 +6324 9615 9 +416 3960 6 +8445 5918 5 +6671 2861 11 +8079 9989 8 +2026 1364 14 +7020 7161 6 +2081 8216 8 +3245 5113 9 +1874 4883 11 +2996 8531 5 +2208 2899 8 +9083 1423 12 +8688 7163 9 +1967 9255 14 +9250 2697 14 +5843 4354 7 +9587 2968 16 +9277 1649 9 +5619 5924 8 +1885 2210 7 +3399 9178 13 +8663 5676 13 +6285 8136 12 +7546 5204 14 +7201 7147 6 +9543 4974 11 +8660 969 7 +4239 4348 6 +7471 6744 11 +5774 3080 7 +8944 7007 8 +6471 9172 14 +4081 7641 15 +535 5144 13 +1476 8065 13 +6320 5075 9 +5691 8628 10 +2524 658 8 +8394 3655 4 +7085 5522 12 +828 3895 2 +9639 1083 5 +2956 405 9 +6106 6772 11 +8265 7097 10 +3091 365 8 +5127 3668 4 +7720 1548 9 +5325 7233 8 +1921 4139 12 +165 1383 8 +8119 6736 5 +9756 961 14 +8266 3353 9 +2378 4833 3 +2350 1515 3 +27 2306 8 +1084 137 1 +3825 6793 12 +5354 3785 8 +7395 2729 3 +2261 8365 3 +4510 3665 8 +1553 980 11 +4327 1737 5 +3128 6719 2 +9096 7821 4 +4981 2422 8 +5648 8930 9 +7857 9531 8 +9143 3169 5 +6626 1775 4 +9337 2020 8 +1025 1565 1 +1568 3428 14 +2856 5842 12 +568 9048 8 +6788 9526 9 +3758 6036 6 +8652 9618 12 +7941 7112 8 +8101 9258 10 +851 3579 3 +8835 1285 10 +9268 5160 10 +8591 3981 16 +2810 7170 8 +4126 4367 10 +9666 7736 9 +5138 4685 9 +4083 7473 11 +5441 6122 12 +6230 6053 14 +4044 8392 7 +5986 2166 5 +9826 624 2 +4032 2151 4 +5840 2283 3 +7456 7829 7 +2669 5677 6 +5879 7879 6 +5919 9708 13 +3974 1999 6 +4759 521 10 +1975 623 13 +2961 5007 13 +8822 8431 13 +3706 2708 4 +3219 3413 0 +3043 8588 6 +4798 9699 7 +6844 8671 2 +8217 731 10 +6675 3689 7 +4599 2258 7 +6489 5379 8 +4832 4929 10 +62 9812 12 +7184 8695 6 +3544 8207 13 +3711 3038 16 +7839 1370 6 +947 8460 10 +4064 4273 8 +6338 3030 7 +6511 2392 6 +6704 8113 4 +6433 7111 11 +3560 9038 6 +4589 1904 10 +8639 1664 10 +4121 1669 16 +1572 9489 9 +9847 4871 3 +4181 6193 3 +6197 8882 15 +8523 6280 9 +7858 5920 9 +6236 5301 5 +4496 5784 17 +2938 2385 4 +2179 5904 11 +332 6866 6 +2128 6634 4 +3003 2589 10 +1929 8811 7 +2971 6557 2 +8848 6887 15 +4154 5914 16 +659 9237 10 +200 591 11 +9249 8310 3 +2342 9259 11 +1045 2239 10 +6667 6984 15 +3832 9665 8 +9611 3681 2 +1616 1603 7 +6658 1906 18 +2278 3611 9 +2915 1006 14 +2094 5718 4 +8125 1596 17 +8900 7579 18 +8645 7231 5 +3934 461 14 +9712 6150 16 +7332 9041 10 +8751 1379 9 +7798 5503 12 +8465 8794 9 +3317 4315 5 +4755 9785 12 +6176 72 6 +9651 3339 7 +7591 579 8 +6969 2774 9 +274 1161 15 +6108 9725 2 +2126 9751 4 +2402 9307 12 +6211 1816 4 +2388 6910 15 +4034 315 9 +3107 1592 14 +7738 825 8 +1115 8030 12 +1473 7488 9 +2416 6567 1 +1017 434 15 +7495 7362 16 +7853 8415 13 +7320 9686 9 +1969 2460 11 +3830 9043 2 +2793 6430 9 +2588 4555 16 +3333 8370 10 +2923 9042 17 +8313 1893 15 +7933 9734 7 +3438 4056 6 +2012 8348 9 +8895 1563 7 +5792 7675 7 +5974 6115 1 +7257 7175 3 +3075 7691 17 +5649 7265 15 +6756 1075 8 +2215 7502 14 +8305 5891 6 +181 9297 8 +1768 8475 7 +1825 1216 16 +3270 4087 9 +2391 1704 3 +5621 6322 11 +1862 9438 5 +6168 7293 13 +6254 3680 1 +9302 7490 14 +2099 4522 15 +2515 3412 5 +6647 7835 4 +2145 1053 6 +7904 6335 13 +9275 7789 5 +8861 9159 9 +5386 3806 14 +4536 9094 15 +3352 7370 15 +1919 193 16 +6223 1601 10 +1500 1965 9 +562 7246 18 +6842 5998 7 +4521 7207 5 +2941 2594 12 +2847 2896 12 +9500 4906 6 +7724 3545 8 +9130 4915 11 +6085 1662 9 +4159 498 8 +4148 599 16 +8754 4331 5 +6090 8344 9 +2 5347 6 +3919 2795 7 +2117 2771 8 +8239 5230 7 +5565 587 15 +9929 7791 7 +9513 1677 10 +7994 9069 11 +504 1210 3 +3938 5702 10 +6371 8495 5 +4320 6283 6 +3394 616 18 +7703 3460 9 +213 8214 8 +4935 5537 9 +2848 4958 2 +9652 7668 13 +485 2428 6 +7777 2389 13 +2709 2038 12 +6082 5074 3 +9300 7323 8 +2960 2037 13 +1556 6336 16 +966 3004 11 +7766 2973 16 +5232 4358 10 +4552 6149 10 +6856 9014 13 +8949 2640 17 +1421 2842 5 +9426 5179 13 +4124 3496 8 +1127 3779 4 +5167 6421 5 +4009 567 9 +2590 4853 2 +5405 5444 4 +5389 842 12 +685 1725 14 +5701 6635 10 +3761 5455 3 +4357 1409 9 +8741 8745 6 +6843 7623 8 +2485 9511 3 +4940 5428 3 +7956 5218 9 +4694 8867 9 +7556 2854 10 +4470 8043 7 +4804 8165 2 +5935 6478 3 +7199 1808 9 +7806 7763 4 +3583 7023 9 +1985 7281 6 +9177 8918 10 +8091 8722 10 +3529 7751 13 +1271 2216 9 +1156 3667 14 +8669 3555 16 +2078 5622 14 +6079 6901 8 +3082 36 10 +7856 1691 2 +1675 6233 9 +3990 306 3 +8184 2617 17 +2542 559 12 +9351 9218 11 +8355 8123 10 +4090 6871 7 +6218 5131 12 +8678 8330 15 +1266 532 10 +6033 1021 13 +7936 6703 5 +4151 1102 12 +2779 6359 8 +3475 2417 3 +4638 3390 6 +7992 4478 10 +360 5947 9 +6103 2267 9 +8002 8317 13 +8643 1089 10 +5185 5035 3 +6426 7877 7 +930 8322 16 +2676 5970 2 +1745 8197 15 +5605 6913 3 +3558 5769 10 +1653 2329 5 +7134 1201 6 +371 2674 3 +5189 5422 9 +2182 7329 4 +6419 6118 3 +379 5402 11 +3337 3771 10 +3304 8583 10 +2487 3649 6 +9081 2651 7 +2928 3310 1 +7126 6329 9 +2585 3853 15 +4165 9726 10 +6551 3774 3 +3715 111 10 +2554 9077 12 +1112 4712 9 +7462 7402 13 +5571 4750 13 +2918 6829 3 +7532 178 7 +5157 6876 5 +6795 7477 7 +1868 4558 5 +1953 9047 7 +9856 5597 13 +2799 4086 11 +8795 9793 6 +2675 584 5 +9089 3546 10 +2115 7196 13 +9290 4911 9 +5834 4600 9 +5309 7891 9 +7634 1955 17 +5273 4190 4 +7538 2917 11 +8293 7218 7 +710 2613 9 +9488 9168 2 +571 5730 13 +2963 9266 6 +9293 2919 14 +3324 4948 16 +1618 9528 5 +3280 4287 11 +294 3743 7 +4408 6377 16 +3366 6158 11 +7935 9659 11 +3614 1845 9 +596 2921 11 +7911 8769 7 +2077 3230 6 +835 9203 5 +9670 8398 10 +8841 9804 12 +3699 4066 3 +4626 8611 8 +5944 8641 9 +8881 3522 2 +9935 90 10 +2069 9605 7 +3596 9355 10 +4932 5067 4 +1322 4771 14 +1479 8512 10 +618 2591 12 +1061 537 1 +4797 4117 16 +5643 7252 5 +6566 3285 5 +4661 1625 8 +8334 4141 17 +1373 1980 10 +6518 8126 14 +2431 3731 5 +6776 5071 15 +2429 718 10 +383 8540 8 +1720 2371 8 +1489 4114 5 +4290 4166 8 +2381 9736 14 +802 78 13 +9244 1399 0 +3811 6491 7 +4364 5504 8 +5275 3127 10 +974 2338 6 +775 8957 5 +4591 1810 11 +1268 1135 13 +6721 4855 9 +9070 1237 2 +786 5027 10 +6286 3972 6 +8256 328 13 +3335 7680 14 +1613 2140 14 +7995 1044 9 +9889 4291 7 +1400 8691 6 +420 6659 12 +8765 7325 1 +8816 766 13 +8403 6711 7 +5066 9710 12 +8603 7238 10 +1550 8975 8 +7701 736 11 +2547 9905 10 +6810 7799 11 +2304 3087 0 +1058 7025 18 +1391 7301 8 +1384 3927 12 +4801 5367 4 +6824 3231 7 +8001 6885 11 +9768 9816 9 +9697 386 9 +7282 53 8 +701 3833 8 +3199 7319 6 +7982 481 13 +6290 5827 3 +8672 2836 5 +1455 2249 13 +2277 8201 4 +1081 5744 17 +3255 1738 4 +4880 2131 6 +6586 7704 11 +4630 6858 10 +7331 8886 10 +7410 8879 4 +7041 5667 9 +3805 8190 7 +7580 9429 12 +8864 7756 12 +7027 9242 15 +589 4384 6 +6247 3589 16 +1023 3484 7 +1450 9055 14 +2336 309 7 +9609 9493 13 +5435 5773 7 +282 946 14 +751 5613 5 +6682 388 10 +9260 16 14 +5630 2766 7 +3142 3926 11 +3728 6015 4 +5666 8156 4 +8901 5115 12 +7365 5398 3 +9384 7719 10 +9380 9202 6 +2305 487 12 +463 3345 11 +3184 6414 12 +3240 5957 14 +9922 5465 11 +6800 7308 4 +9774 9360 6 +1096 6340 9 +6606 5884 13 +7906 4195 7 +3795 5207 8 +8089 9579 8 +3581 2163 8 +9616 7298 14 +845 2633 9 +288 7492 3 +9398 8800 7 +2430 8222 12 +4554 9807 8 +4339 245 5 +6549 5592 6 +5624 1461 9 +3676 7503 10 +454 8947 13 +3957 4839 13 +9581 3608 3 +5082 3847 4 +4222 6582 7 +4142 1570 7 +9299 457 15 +6404 9829 17 +7932 3937 4 +6492 858 10 +3770 6147 11 +9662 1043 7 +1727 8149 8 +1579 6595 14 +8292 4089 9 +8724 5257 1 +699 1595 10 +86 7268 14 +1464 8634 8 +378 1129 2 +8199 647 1 +9261 8482 5 +6899 707 9 +4816 3597 11 +4725 9516 10 +516 5535 3 +9296 3305 0 +292 1552 13 +3449 5903 10 +7216 5065 8 +300 958 7 +2821 8566 8 +3989 4023 0 +5010 4644 15 +7565 6490 14 +7222 380 3 +4122 1419 13 +508 7970 12 +6506 6243 8 +6465 3587 12 +8706 6054 10 +8563 6123 7 +8194 3064 8 +6385 3661 7 +7728 5194 6 +7693 6252 6 +8404 7340 10 +88 1533 6 +4741 6943 6 +8565 7428 15 +5476 2242 6 +6944 5061 15 +1721 7626 8 +6694 5540 8 +5412 5663 14 +6459 5135 15 +916 9937 6 +8295 6618 3 +4884 6454 9 +8517 1760 7 +9745 6474 8 +4744 3730 15 +5442 9720 7 +4003 1258 11 +2339 8367 6 +2442 1363 14 +690 1356 4 +6179 738 8 +7028 7463 8 +6583 8247 4 +8169 3349 13 +5860 2023 3 +5489 5628 11 +7684 1952 15 +3217 2958 12 +6714 7774 11 +5627 9186 16 +5497 9743 13 +1325 9978 9 +555 6364 11 +1970 5009 14 +3921 5787 11 +7855 4627 6 +8026 3129 6 +1351 9723 10 +1632 7530 10 +7467 6532 7 +3737 3749 14 +3249 2920 10 +75 3479 7 +8996 3659 9 +8118 5604 2 +8536 5023 5 +9561 9911 3 +8015 793 16 +9341 8154 5 +3079 9931 8 +2043 3436 6 +2293 2518 14 +4872 6990 5 +7760 4754 8 +4217 4268 8 +8053 8236 17 +8962 4519 13 +4950 691 10 +5197 3095 10 +6886 4481 9 +7866 9352 0 +9185 6473 6 +3187 5818 6 +5393 7915 11 +363 6023 5 +428 6342 4 +4668 8764 2 +2948 3155 9 +5212 3403 9 +5806 5100 9 +4617 8576 0 +1886 4490 10 +2883 2197 13 +291 5817 6 +258 2791 8 +7586 2312 11 +3945 9333 6 +3950 9830 14 +1615 1051 7 +5203 1702 9 +3122 5129 15 +317 3252 11 +8646 8219 3 +5732 5461 2 +3616 281 16 +2004 3613 8 +3125 6837 14 +1259 4633 17 +2667 4805 10 +32 5552 4 +7230 7819 12 +5526 3088 6 +4192 6523 12 +2096 9215 2 +8055 9917 8 +5186 6922 7 +7015 5031 2 +5222 8144 12 +9036 6587 15 +8733 8006 16 +7335 5868 15 +8376 141 8 +295 7489 13 +2985 9927 7 +4677 9449 15 +5162 5432 12 +565 4449 10 +8685 5753 13 +4369 7026 12 +1224 6382 5 +4789 4656 10 +6008 3284 9 +2755 6864 17 +3694 4878 4 +3316 1822 13 +883 9376 12 +5640 8541 14 +8505 9916 15 +8128 5383 7 +774 6834 12 +1225 2858 17 +2571 5470 0 +1350 8428 8 +5568 5937 11 +6505 2593 9 +1956 5141 4 +5500 4819 4 +8503 5614 10 +6481 795 6 +7149 8441 13 +1526 6264 0 +243 8182 16 +5742 789 9 +2679 8908 4 +2246 720 13 +610 6029 10 +3634 9163 3 +205 2292 12 +7548 517 12 +4979 2568 5 +631 8843 16 +8550 2670 7 +9078 2986 13 +7961 9368 9 +4756 342 9 +4588 7611 5 +4025 5000 4 +5349 7173 9 +3086 2496 6 +9289 8893 14 +9184 4072 10 +9509 762 5 +7133 5589 8 +1178 2967 11 +9468 2294 3 +3200 9966 15 +7800 5295 8 +5227 9530 10 +7336 6820 11 +3068 5180 15 +2732 2817 6 +4260 3903 2 +2656 4386 1 +9748 6902 9 +5832 4448 4 +5094 6495 15 +5235 7727 6 +1448 6318 10 +2962 9952 0 +935 975 7 +7922 3594 9 +9327 5045 9 +9154 602 8 +6889 5429 17 +7585 1367 9 +8157 1949 15 +1982 9819 7 +9124 3527 4 +6172 1835 10 +9428 6209 13 +7385 4625 2 +6730 8917 11 +4078 2822 10 +1546 2493 11 +5102 7752 10 +4028 219 12 +843 7614 11 +595 9994 4 +8778 9390 11 +5496 1581 14 +3562 3121 7 +8632 3328 12 +5481 3148 7 +8444 7555 5 +5943 4908 7 +2607 4250 9 +7558 904 0 +3066 2598 12 +4584 8624 12 +1297 2746 1 +3682 6883 3 +2401 5745 11 +9554 3408 14 +5853 3141 12 +6629 8997 14 +948 5858 8 +1659 2168 9 +5032 221 6 +4773 4896 11 +5527 3575 16 +6462 7302 10 +3113 4620 11 +198 3873 12 +2092 6250 2 +2557 750 0 +8325 9746 7 +9298 645 10 +1004 3171 6 +3002 2271 16 +4513 37 9 +8892 348 2 +2993 3204 15 +18 9219 6 +9065 9403 11 +9138 7660 14 +6976 7883 5 +1994 1826 1 +2763 8590 9 +4228 7629 11 +2053 93 7 +3161 384 13 +6862 882 12 +7116 6978 6 +4703 1711 5 +9007 9969 4 +8049 2010 6 +1390 6467 8 +8756 9780 16 +6553 10 4 +809 1198 9 +8278 5600 7 +9820 4208 2 +1311 2648 18 +2383 6242 13 +3720 6526 7 +4229 6080 12 +7372 6005 6 +4398 1265 11 +6547 7744 11 +6232 9153 10 +1789 4404 3 +9103 478 9 +1505 9850 7 +8543 1783 8 +4314 1136 10 +6791 2890 14 +1196 492 8 +6515 3658 3 +9515 69 7 +6935 8235 12 +5652 1894 9 +9209 9361 8 +1119 2833 14 +3872 3971 13 +1187 9475 8 +3100 2132 12 +7000 9875 9 +4982 2141 15 +8661 7072 8 +3111 8637 13 +3904 6848 3 +5525 5725 8 +965 1887 13 +4281 4356 5 +9569 5563 9 +5980 5323 1 +7664 4874 13 +2886 4059 9 +7366 6966 10 +7952 6528 11 +7479 1932 5 +5852 222 7 +984 1580 5 +266 100 14 +6555 5556 12 +1628 7968 6 +727 5553 4 +877 8515 14 +1434 4776 1 +6171 7750 10 +1641 8757 9 +5269 728 11 +8221 3359 0 +1413 7864 13 +5988 3018 13 +7519 2361 12 +1335 9214 11 +7454 6603 13 +2605 5634 9 +5971 5424 18 +9013 9728 9 +1402 4465 6 +7017 9683 10 +8171 1891 5 +7232 7062 1 +9549 7735 9 +4849 8289 9 +9248 5685 7 +4688 4041 11 +7159 6202 7 +2872 2843 11 +5420 1185 13 +1538 9828 5 +2567 9197 15 +6056 8225 10 +820 9845 7 +8195 1889 3 +8992 8834 5 +3010 5361 4 +3543 2202 8 +4634 6821 11 +4628 8658 1 +4815 1066 10 +7440 8542 6 +4074 1850 9 +4642 3336 5 +3878 8662 11 +5583 6934 10 +9024 247 11 +3220 6613 11 +2935 9052 8 +3150 9147 10 +2507 1417 16 +8693 4076 7 +1840 9744 11 +9267 8524 12 +4492 921 5 +3890 4992 5 +3154 7071 13 +1846 4285 11 +3325 4088 9 +7841 9859 10 +7501 6959 3 +5364 6143 9 +1346 259 13 +4288 3673 4 +3156 9088 14 +4541 474 11 +4423 9983 7 +2660 3481 0 +1320 7369 16 +1055 5426 11 +8857 472 14 +1064 9278 8 +6841 6718 9 +5196 40 6 +9967 5291 9 +8818 7178 5 +1766 4763 6 +1274 1162 10 +8297 8094 10 +5855 3061 15 +5888 1255 13 +38 6135 3 +4817 1682 10 +3466 2214 13 +4537 8862 4 +260 1686 16 +4293 9409 12 +818 9769 10 +4550 3851 10 +4702 6321 8 +5875 8529 12 +3424 5342 6 +374 9176 13 +9087 1116 11 +9631 4093 9 +4244 1420 9 +157 8856 3 +4241 7510 4 +3951 2714 8 +8518 7682 12 +4530 6991 12 +9959 4495 8 +8718 9684 14 +8909 2253 7 +8829 8231 5 +5939 7024 10 +8549 7195 6 +8896 8770 10 +8335 6119 7 +6868 6353 8 +1812 5231 17 +8070 9470 5 +8500 2545 9 +953 7674 7 +1937 7122 12 +5286 4343 6 +2701 3427 10 +3347 8776 1 +3315 2906 3 +6648 8208 9 +8555 3298 11 +1730 2121 11 +3047 4619 8 +5511 5049 14 +5946 7312 7 +9114 5114 12 +6330 4886 9 +4951 5735 6 +748 2478 13 +3755 8426 18 +1777 9827 8 +9823 7376 11 +9782 2925 11 +6952 110 13 +4484 1280 7 +2541 6207 7 +4605 3482 5 +7781 1157 6 +7989 8353 2 +4757 2377 7 +3975 2983 12 +1342 166 8 +4782 2840 6 +2505 4623 14 +4480 5288 17 +4211 6227 11 +1020 6846 9 +530 3865 13 +5108 7737 0 +8753 838 7 +3341 763 4 +3852 511 5 +1996 4806 7 +9131 2533 5 +834 8244 7 +5763 8866 10 +5983 2217 2 +4999 874 9 +3523 4340 9 +8527 3094 7 +3383 1794 6 +6739 853 10 +3344 3032 6 +8719 6651 2 +2604 1444 11 +8173 9477 9 +1993 9125 4 +8774 3498 12 +7387 7612 11 +8850 9640 8 +5310 9369 9 +1562 4428 9 +5755 8937 12 +7289 394 9 +5106 6569 12 +1003 5165 5 +3098 8020 8 +8080 9338 8 +1076 395 7 +3837 8875 6 +1876 5683 10 +3741 7139 1 +3250 8321 9 +531 8976 4 +63 9783 7 +9157 4119 10 +1898 8387 7 +334 2327 5 +6259 4972 1 +1633 1218 1 +5136 9923 7 +4828 3636 10 +9544 4338 14 +130 3654 11 +833 8690 18 +920 7884 10 +8981 4118 8 +9791 3292 7 +8352 1406 6 +7986 4471 9 +7749 3897 5 +6261 1753 7 +2199 8452 10 +7768 3692 13 +1689 7429 6 +7224 3891 18 +954 3137 13 +3541 1706 8 +7514 190 9 +1977 4045 12 +1531 1551 5 +9326 2723 11 +5584 9675 5 +7509 3603 9 +3097 737 7 +6219 303 10 +8959 4282 7 +2238 2041 8 +3012 9109 13 +7153 9245 13 +2203 1172 11 +2677 1301 10 +7867 9033 13 +4669 8114 7 +8967 3162 13 +2403 7655 0 +4707 215 8 +7651 4209 15 +9536 8973 5 +1130 3621 13 +3025 2636 12 +2080 9897 3 +8478 3928 11 +608 9075 5 +5954 6892 2 +7321 5034 8 +8350 9221 7 +4382 7417 12 +7907 7379 8 +2816 5913 6 +939 2665 9 +5533 9377 7 +114 6249 16 +2185 6424 0 +3561 5697 9 +7192 6798 2 +4138 7633 9 +1256 3513 4 +3077 4732 1 +1922 435 9 +2160 290 14 +5410 5132 6 +3232 4092 12 +2878 4067 6 +5118 4887 8 +9784 3684 13 +6962 739 5 +2504 569 4 +4043 7182 9 +1724 7206 8 +639 117 13 +9031 2634 10 +4691 6347 9 +3165 6981 14 +8597 538 11 +4952 1212 12 +2743 6035 7 +6782 3074 9 +7344 5012 7 +6212 8143 14 +4748 9538 12 +1851 4621 12 +4342 5969 6 +4967 8735 17 +2161 5338 11 +7649 2237 10 +7255 7967 0 +3115 3503 14 +6312 5063 7 +6818 714 8 +5655 7707 16 +7896 1913 4 +3707 6811 11 +256 4637 7 +8474 788 15 +9851 2281 10 +6860 6758 6 +5128 476 2 +1858 2384 6 +9115 3630 6 +6381 6878 5 +322 4975 18 +2724 4256 11 +1126 4361 7 +8082 667 12 +1958 4157 4 +1474 3049 11 +5457 8561 4 +280 7539 4 +3607 1991 6 +9406 1496 11 +5311 7044 18 +778 6809 6 +6797 8629 2 +7076 3172 8 +7330 8508 3 +5513 1818 6 +7587 5779 7 +2333 3492 8 +2139 5886 9 +2625 4060 6 +9314 1583 15 +9135 8140 5 +1329 8429 4 +3191 8050 3 +642 2929 9 +2631 4212 1 +4653 7174 10 +3688 8158 11 +9374 2897 15 +7582 8155 5 +4841 7249 5 +7635 4532 5 +1591 9359 5 +28 7316 4 +5407 376 6 +8873 4761 10 +5168 21 9 +9436 926 2 +2792 1345 11 +4491 5976 9 +8243 329 1 +2768 327 5 +4458 4053 11 +3780 5038 7 +4986 1597 17 +4827 3001 13 +2614 8905 9 +3775 3441 9 +6759 3254 10 +5087 5391 6 +2189 2036 16 +8922 1150 4 +2680 4099 11 +5306 230 11 +3973 2733 7 +4204 2395 10 +7593 31 1 +7357 9867 11 +3864 1511 4 +2555 5020 8 +440 1544 7 +2562 437 9 +1016 4129 11 +2556 543 13 +3024 8252 5 +3186 7107 17 +2759 8421 13 +6915 9306 10 +8894 5538 11 +940 9476 8 +9092 7424 9 +2176 1934 3 +8010 4515 3 +1349 2624 5 +909 7237 9 +7130 3360 10 +3493 7755 11 +7592 6745 12 +6144 656 4 +67 915 12 +6908 2777 5 +3262 9765 14 +4615 3069 6 +640 5487 3 +540 3377 9 +3031 3588 14 +8532 2354 4 +7887 4366 3 +7823 8446 14 +9714 9879 0 +5890 7562 9 +3762 6986 13 +2773 8596 14 +4201 3948 7 +8205 2785 7 +1323 7605 11 +4254 6932 10 +8488 5899 1 +1028 957 6 +5671 8535 9 +324 5573 3 +546 9910 8 +84 3264 9 +2710 4391 14 +4560 3703 11 +6724 2119 15 +2657 3823 2 +6580 6524 9 +8017 9527 9 +8257 76 10 +5509 8832 8 +5532 4717 8 +2503 8648 4 +4989 4061 16 +4968 2365 16 +6650 4528 12 +2575 9590 8 +3442 3491 15 +6121 2066 13 +97 8242 15 +3642 3566 5 +7830 480 1 +931 3645 9 +8369 3308 11 +997 8110 7 +7901 5292 10 +9658 1836 7 +3017 627 13 +3740 4563 4 +7405 8647 7 +5041 4169 9 +2469 439 12 +9264 9224 4 +3326 7464 12 +8589 863 6 +9222 890 4 +3320 333 6 +8196 3176 8 +5408 5081 4 +4054 4183 14 +6830 959 11 +9907 5809 6 +9831 6751 6 +2142 6817 17 +4893 3595 9 +1166 6360 10 +8610 3313 12 +4360 2991 8 +7494 4402 6 +4868 8427 8 +8763 9627 6 +9336 7569 9 +2316 9644 8 +8038 1097 6 +2744 3746 11 +5836 2113 13 +1803 7589 7 +9478 4311 8 +6690 6205 7 +5578 7123 9 +7177 6504 7 +1393 1811 7 +8683 5829 4 +6792 6617 14 +2672 6700 10 +4388 71 6 +8898 1642 6 +8934 5948 14 +3970 5854 16 +929 5739 3 +8408 9619 16 +9200 8604 2 +7445 6306 14 +1057 4399 12 +8793 9238 5 +3637 7571 16 +3776 1503 8 +6367 1221 9 +2332 8275 12 +6680 8954 5 +8955 4240 18 +4453 5316 8 +3887 6576 15 +4596 381 8 +6099 3119 5 +5378 4031 11 +1435 9674 10 +1992 3319 17 +7983 7359 5 +1246 6558 6 +4147 9129 3 +8687 5375 15 +7805 5717 10 +1945 539 3 +613 5164 11 +5958 6142 11 +4529 8062 10 +4959 9545 5 +6487 4498 16 +1236 8569 3 +3520 3287 15 +4822 2863 17 +8014 4234 6 +9928 1035 10 +1095 733 12 +4246 636 15 +6030 6116 6 +6428 4036 1 +4362 2970 13 +8407 5741 7 +9432 169 13 +9892 4583 13 +6445 6979 1 +6801 8574 11 +64 1525 12 +9563 2173 12 +9407 973 7 +8868 8607 3 +4252 7573 13 +5922 4303 6 +5110 912 10 +7082 2379 8 +693 4451 10 +7679 3041 14 +6898 5625 11 +3267 455 3 +6276 5333 10 +3842 1587 15 +9442 1576 9 +6710 1873 18 +1452 8752 9 +747 6125 4 +1907 1892 0 +6610 8586 10 +4120 30 6 +4232 9471 5 +8627 548 11 +4055 1261 2 +216 7013 5 +8746 4997 4 +4577 8826 13 +4319 2445 14 +2511 3791 12 +4812 9995 4 +4654 469 12 +4681 5999 16 +8033 1426 7 +892 9974 6 +3571 2623 7 +1040 1160 5 +4346 5575 15 +2719 9676 10 +5459 4485 7 +7060 5873 9 +2937 7228 1 +4325 1287 17 +7168 6217 2 +4671 4309 17 +438 4137 15 +7511 6949 7 +5962 978 13 +7699 8721 2 +6238 2855 13 +6410 1963 12 +4004 6904 11 +6614 3935 8 +6226 4289 5 +3036 9606 11 +5013 4859 2 +3470 2803 6 +3013 6743 1 +5175 7396 7 +6253 2314 14 +5332 5183 14 +5654 49 11 +4561 6018 15 +847 9319 12 +2475 1575 6 +6117 6494 3 +5847 8889 2 +5599 6087 11 +7560 1542 8 +9904 4571 8 +6244 5490 6 +3375 4796 5 +3055 6343 15 +5865 4278 0 +3164 5673 6 +8681 3662 8 +3490 5711 9 +7317 4164 3 +7381 9388 1 +7397 3600 7 +7507 3266 1 +7236 4852 17 +9506 8246 11 +6923 5867 5 +9037 4134 9 +4033 8958 8 +4507 635 3 +4607 527 14 +3456 2299 5 +9284 2183 8 +927 4191 4 +4572 4944 10 +1316 4352 4 +6231 678 10 +9946 7893 4 +7278 2075 12 +9012 9706 6 +3506 4421 9 +2662 4444 11 +7485 6564 3 +500 6781 3 +6151 2653 8 +9693 7226 7 +1529 6997 8 +3705 7740 11 +42 5272 6 +3518 2725 7 +3601 3218 7 +8935 6771 15 +9213 6896 14 +9253 894 7 +5411 5381 8 +6155 3591 10 +7466 5590 7 +7276 5672 16 +2565 66 13 +4235 9324 10 +2552 7834 12 +1154 3955 5 +1631 9772 10 +6702 8324 2 +6190 5353 9 +9312 6109 8 +4917 2951 6 +6339 6869 8 +3784 1799 8 +905 6201 2 +2255 399 7 +1680 113 13 +8824 2844 13 +1655 5906 16 +9416 2280 12 +5134 7873 5 +2171 2435 1 +8311 5279 15 +1968 5872 9 +2360 3152 7 +3237 9291 2 +9303 3889 11 +4326 2024 9 +9187 8614 8 +9270 3469 10 +7061 4701 18 +1665 3879 13 +1392 9420 9 +5593 5373 9 +3717 5646 4 +6375 4213 10 +9400 8058 6 +3362 7670 12 +6760 5149 2 +4646 2865 4 +1609 5463 2 +3867 2596 2 +2103 287 9 +4110 6773 3 +2870 3622 15 +6442 105 17 +3978 4006 7 +7169 3685 8 +4943 6164 9 +2824 5746 1 +6098 1214 8 +2057 3537 11 +3146 3370 7 +3411 3144 15 +6893 8481 6 +9100 646 5 +8192 1805 12 +1182 9740 15 +503 4108 7 +1398 8899 17 +7486 7537 2 +5997 6663 12 +5810 4431 10 +8869 5090 7 +6747 2432 11 +9990 3265 7 +7962 2711 15 +2162 9865 5 +77 5644 10 +6550 7811 7 +552 7250 3 +2466 4557 12 +2241 6789 6 +2089 249 11 +6664 4005 4 +6460 5569 12 +913 6479 15 +8799 8950 3 +2546 2535 11 +8162 6529 7 +871 7360 9 +4595 2751 7 +2348 4734 5 +4544 6948 5 +3512 4199 7 +3941 7733 10 +1007 6548 13 +665 4991 13 +717 3987 8 +1820 7977 3 +2244 2319 13 +4175 1530 15 +922 3769 5 +2313 2205 13 +6010 1046 7 +8723 4174 18 +5062 9900 11 +2830 1222 10 +768 6461 1 +1199 4753 14 +5942 232 8 +6921 5317 7 +1273 4953 1 +1347 2091 14 +2727 6358 13 +1310 357 4 +8863 1340 10 +7600 3302 7 +7663 944 11 +963 1372 3 +17 6678 8 +6611 8567 5 +6011 4403 3 +6735 638 13 +4015 3028 14 +8466 9991 16 +5092 9880 3 +1683 9700 5 +8717 1124 17 +5214 9702 8 +1880 1676 12 +2369 8513 5 +7554 5263 9 +1823 2543 16 +3550 7559 11 +6831 7621 6 +7767 9274 9 +9112 1520 8 +893 3056 18 +6045 9424 9 +3947 8092 7 +8451 8414 3 +4793 1747 12 +525 6777 6 +6940 4476 5 +8013 1879 11 +6466 1824 4 +5293 5502 5 +8378 2164 5 +246 2904 2 +5830 848 1 +4329 1733 6 +9716 6372 10 +5822 39 5 +9152 199 6 +6509 7624 15 +241 4186 9 +7266 9039 3 +3439 9231 3 +6221 182 14 +5188 4639 16 +7314 7653 10 +4413 134 14 +6722 3468 8 +2300 754 4 +3183 9272 14 +8585 6070 16 +7969 3674 9 +4336 8806 14 +2689 2367 11 +6749 7762 8 +108 6713 11 +59 9800 5 +4459 9576 14 +8166 3405 5 +7031 8004 1 +5883 3473 10 +6975 5781 12 +2952 1619 3 +4970 9393 14 +7444 61 16 +3664 8396 10 +3670 7775 7 +6599 2645 11 +955 5294 9 +9448 3471 8 +9944 4506 5 +283 101 5 +9809 9179 0 +6956 9694 17 +5945 6308 4 +8633 9358 1 +2673 1029 17 +6732 6924 6 +5400 572 13 +6002 9595 8 +1630 7927 8 +8985 6038 14 +9072 4987 6 +107 4419 9 +6620 320 14 +4128 2447 7 +6681 8046 11 +7860 2888 13 +544 8339 12 +4027 6439 9 +2652 5940 10 +5796 7128 12 +5931 3593 3 +3138 1336 5 +6400 1763 3 +1909 3389 9 +8906 6673 10 +1264 9770 9 +7461 6289 9 +8522 7348 10 +5104 2323 13 +1483 2263 10 +187 7253 6 +5897 302 10 +1740 7843 14 +6762 7352 12 +3461 9977 11 +4202 6394 14 +7290 5785 3 +8791 4263 12 +5446 3644 11 +3727 8306 9 +2809 7337 8 +7240 509 10 +4091 9195 6 +1502 4040 17 +6194 7958 8 +1697 2720 18 +3489 7527 3 +9985 5656 10 +8880 9107 10 +4429 644 8 +4651 9408 10 +7895 3448 12 +3646 7157 3 +3290 2070 10 +8807 1015 13 +6536 1905 10 +7045 6942 9 +4826 7576 10 +7285 7036 11 +5084 8164 5 +3882 6369 12 +5251 7874 9 +4180 58 11 +700 3331 7 +3570 4787 11 +2520 4973 11 +5816 5046 7 +3226 3786 1 +5595 9571 11 +214 8699 17 +9853 9030 6 +5440 9986 7 +4318 6685 7 +9201 5639 11 +5029 2409 10 +5803 7414 13 +8072 933 8 +1559 5201 13 +5548 1348 10 +5519 648 3 +6900 4678 13 +7787 5515 8 +8363 671 13 +1148 7480 1 +1219 813 17 +551 8729 8 +6594 760 6 +1293 3268 7 +4985 6282 7 +7868 2147 9 +369 4777 11 +8704 9924 11 +5992 1415 15 +7478 2501 8 +5289 4882 15 +7687 9656 5 +3739 7930 5 +4017 5560 11 +5690 307 13 +192 9945 9 +9276 2451 12 +7439 8972 10 +1032 3669 6 +4920 5438 10 +1063 3511 11 +8060 730 10 +6570 7888 8 +2838 5091 3 +580 7734 15 +4700 1528 10 +8766 3195 5 +7523 9503 12 +8031 9454 8 +3980 8069 2 +5024 2335 3 +3371 1137 1 +7564 9091 8 +4494 4867 8 +7705 7221 6 +4179 2900 5 +8084 1307 15 +9582 5419 7 +2213 609 10 +9499 9010 3 +1855 1302 9 +2514 3814 9 +5101 4236 9 +3007 8483 8 +7786 6850 17 +9779 8223 10 +880 7450 15 +9957 3966 16 +898 3181 13 +8945 9678 10 +6003 1394 8 +254 6262 15 +1586 1330 7 +2538 3083 12 +5491 4104 10 +1108 7702 6 +3713 9097 9 +6701 8803 5 +2479 9704 0 +4155 4722 5 +8296 1314 7 +8319 6738 13 +3042 4182 13 +7019 140 7 +8700 9557 17 +1507 8820 9 +6025 7297 12 +4889 7208 10 +1105 9815 15 +7795 3480 10 +4520 4153 6 +8599 5964 14 +6588 4698 6 +6334 5892 9 +1744 8979 7 +7446 8077 0 +3289 3297 8 +9139 9535 10 +8181 3089 11 +3964 3301 10 +8185 6534 7 +4489 3159 15 +7179 9624 9 +3464 514 7 +122 2595 14 +197 5083 11 +4387 8775 6 +6455 2155 3 +2581 7364 7 +1445 1303 2 +9286 8684 11 +7288 8657 9 +5686 2940 2 +1022 7978 12 +1438 3952 5 +2399 7065 10 +9315 2000 11 +9767 226 17 +2932 3742 3 +4865 9932 16 +5036 8983 1 +7957 1656 6 +5772 8450 4 +4665 5579 14 +1901 8105 17 +5953 8439 2 +5749 5698 12 +2287 9285 8 +9696 4452 10 +9750 3869 12 +995 1821 11 +2227 9021 15 +6764 9501 9 +8202 614 10 +268 1795 10 +577 1832 6 +1804 2071 4 +3690 7686 7 +8160 3735 5 +456 4251 10 +7119 4609 7 +2473 2980 7 +9280 8771 14 +3883 1488 4 +3569 3046 13 +460 3666 14 +6368 6277 10 +429 5615 10 +8932 2370 8 +3584 8153 2 +4002 3906 4 +4385 3354 3 +840 2740 10 +7043 4069 10 +9441 7945 8 +5823 4344 11 +8814 8133 12 +4614 9045 7 +8525 5498 14 +6350 6930 9 +4533 1940 6 +5002 1098 12 +8887 6379 4 +9090 2289 9 +3425 7529 10 +8998 9415 10 +1964 4447 9 +7422 5880 2 +9788 6063 8 +3845 5208 9 +5934 6689 6 +600 4321 15 +5312 1260 11 +8613 5123 8 +5893 4602 2 +1707 5760 4 +9757 1197 0 +856 6563 8 +7481 769 14 +4464 5608 11 +830 7176 3 +6995 8342 4 +6266 4434 10 +827 4466 10 +6699 8469 5 +116 1456 10 +7305 4178 7 +1864 5811 5 +2912 497 5 +8679 6859 12 +2976 8842 9 +9876 8096 7 +633 442 5 +7391 785 9 +4543 3836 15 +9564 6853 15 +2462 1073 7 +7921 8845 8 +9422 3998 9 +6804 8497 13 +6009 2776 6 +8261 855 4 +6260 6666 10 +2728 1974 12 +879 9540 9 +9836 3135 7 +8499 1243 11 +5494 3652 3 +2706 3781 9 +771 7534 15 +7938 2390 7 +4392 9292 9 +9343 859 13 +3198 1787 17 +7049 2180 5 +7809 1245 7 +841 1410 9 +4051 4664 9 +1094 6597 0 +1204 5995 11 +6016 5960 8 +368 5178 10 +6346 9832 9 +1151 7358 12 +3508 9512 6 +7761 1639 11 +2879 6271 8 +772 6593 4 +8262 7164 10 +7570 2107 10 +8081 7882 4 +8206 346 3 +7820 4373 4 +1319 1857 14 +6196 1202 13 +6059 3288 7 +6802 8163 4 +7432 7642 15 +9389 9466 11 +237 5539 8 +8839 9701 18 +7997 992 14 +1716 9776 13 +1739 8276 4 +7943 3939 6 +6386 4994 12 +4504 7568 5 +1173 4019 13 +4317 5692 5 +7620 3525 10 +9217 8876 6 +2608 4660 15 +7124 7632 14 +2620 3582 7 +7505 1715 1 +2453 7117 6 +5778 5281 1 +9018 8553 7 +4259 8124 16 +1431 2034 3 +3917 2846 11 +605 5297 14 +4432 5911 8 +7046 9939 8 +3054 2930 12 +4912 6138 8 +7452 3116 2 +5808 7059 14 +6411 8847 14 +800 3617 14 +2204 1180 1 +9519 6561 7 +2717 4802 3 +2837 1786 11 +9643 7946 4 +3381 5747 6 +703 2387 16 +2628 151 16 +6133 8773 4 +2804 228 5 +2611 1059 9 +5274 3443 7 +7828 5849 11 +1167 4926 10 +1696 7506 2 +7167 1484 2 +9256 4133 8 +5797 5586 16 +8203 6047 16 +2813 1139 4 +7279 5174 5 +7067 4375 13 +2111 8725 16 +2873 4749 5 +8146 2516 18 +4071 3900 5 +9921 3894 7 +6575 1941 10 +5142 8619 8 +1833 9556 6 +5916 4597 7 +4713 235 11 +889 7985 8 +7846 3130 6 +8885 9650 14 +6146 4860 9 +4503 4708 3 +837 7844 9 +2240 8263 10 +3275 3618 14 +1637 5193 6 +4493 9956 8 +1806 7802 8 +6083 4562 3 +5112 7681 7 +950 3859 16 +4052 7154 7 +8966 4624 9 +3114 2207 9 +7980 6440 4 +8068 427 2 +8051 5863 6 +9555 350 7 +3979 3296 13 +3139 1140 12 +1986 5737 1 +675 1645 10 +6397 8971 2 +8473 8007 5 +6696 1920 8 +9497 4397 12 +1019 1853 10 +536 3259 3 +9232 9053 10 +3881 459 9 +447 5738 5 +8178 4687 5 +9893 8600 3 +545 1936 2 +6496 4527 9 +6669 967 7 +6676 6872 7 +8943 8347 6 +73 433 14 +9795 3828 5 +5040 2902 8 +4000 1779 12 +6600 9430 8 +2405 8397 6 +6132 2500 4 +1317 7190 3 +7166 5694 12 +7912 2944 7 +8245 1371 10 +233 5629 10 +8159 2275 8 +4675 2349 8 +9622 1995 1 +5909 7609 8 +3514 6645 4 +2808 6624 10 +4659 1928 12 +8425 3057 2 +5813 9366 15 +2700 8298 16 +4472 982 8 +2558 7598 12 +9688 9247 6 +4570 6836 11 +5850 1071 10 +9539 9397 18 +6128 3850 15 +8148 239 7 +7533 566 13 +2587 6936 12 +2570 6341 14 +9254 9612 10 +5555 1678 5 +3037 4152 7 +1382 5759 5 +9715 50 7 +7917 4969 4 +5800 7156 11 +4693 6769 11 +3778 9871 9 +4026 9399 10 +5733 3943 5 +9199 1113 5 +5660 8476 14 +3153 3995 11 +1062 9395 4 +8391 8902 9 +4566 5731 7 +9534 2192 12 +7496 8715 10 +8463 8703 8 +2498 8838 6 +4172 2999 4 +465 2767 11 +3005 4795 18 +6531 400 8 +3896 8927 11 +8520 1429 13 +9553 3718 13 +4667 6577 15 +5912 2637 12 +1561 7832 5 +3256 458 10 +5130 6240 11 +741 9982 7 +3754 4135 7 +6519 9648 2 +3090 2947 6 +1 242 10 +7433 864 16 +3855 6305 7 +4903 5095 3 +5001 9792 13 +7747 9029 6 +4774 1281 6 +3040 3958 9 +8986 1848 8 +1132 7353 3 +9806 8667 15 +4936 8424 9 +9223 8213 9 +5464 7770 12 +1100 8484 11 +5966 5170 11 +3213 9150 15 +5977 8496 9 +9117 2138 7 +7162 206 9 +8232 9443 6 +70 9347 11 +7723 3440 8 +694 6160 8 +9490 8399 8 +6387 6370 9 +9381 5609 10 +7987 9672 4 +6826 8349 14 +8090 2412 3 +1916 2972 6 +2443 6319 9 +248 8085 4 +5581 2450 9 +9233 7415 7 +1491 6180 8 +3574 2880 1 +8883 8568 8 +575 1468 9 +3059 3590 12 +7919 1650 8 +1960 3820 17 +9158 7526 0 +2188 8872 8 +1771 6838 10 +1388 8323 8 +903 5006 6 +6863 3376 10 +5004 9123 6 +3499 1477 15 +9698 8255 10 +2526 8564 11 +2654 1942 14 +8587 2019 9 +4905 1828 5 +9208 5344 13 +4840 3358 9 +4350 6437 2 +238 8328 7 +6500 900 6 +3283 2358 4 +240 3953 8 +6607 2072 5 +5750 3782 8 +2123 1396 16 +5471 3734 7 +4297 2486 13 +7721 227 10 +3789 3808 8 +1074 6468 9 +6105 4454 16 +814 9960 10 +9908 6127 6 +5770 6787 15 +3834 9885 3 +1128 6598 8 +6304 6903 17 +3019 8759 9 +8651 3391 13 +7204 5221 13 +2320 1397 10 +5751 9084 11 +6725 2786 9 +8854 1674 4 +2106 3578 2 +2739 2612 9 +5416 3599 3 +6065 7055 6 +3524 8437 14 +6996 3733 10 +8705 396 11 +7326 6785 5 +2696 3993 16 +5099 5528 7 +1036 8075 10 +7710 8074 9 +6483 1189 6 +3382 7500 14 +4569 12 14 +4298 6443 18 +3447 4499 12 +3536 6540 1 +4258 1788 13 +5780 1549 7 +3099 1416 1 +7002 2436 6 +4980 9691 14 +3918 1361 12 +936 2731 11 +7998 4220 10 +7032 990 9 +857 3145 10 +1629 5059 13 +6297 2683 12 +5748 7066 6 +4443 9252 12 +3227 6716 6 +5305 5304 6 +5838 6765 8 +1385 4807 17 +6625 7039 14 +5709 8650 8 +7069 6632 16 +9447 5414 16 +1442 9281 3 +4304 4545 9 +4460 9588 12 +9537 7212 6 +689 6348 15 +7215 5520 4 +5821 5456 14 +85 1068 12 +7142 755 6 +8696 7100 7 +1118 1125 16 +4692 1843 9 +971 387 6 +8990 9165 15 +7442 5488 6 +2474 1699 14 +2030 5261 12 +7437 8087 8 +9491 1437 7 +7842 5641 7 +7698 1039 12 +9664 3175 3 +2116 4379 10 +8032 8326 9 +3505 4223 11 +9169 6897 15 +3949 8224 13 +776 4313 8 +8933 782 12 +5736 9550 10 +1230 9584 9 +1761 7291 7 +6839 3540 8 +3190 1338 10 +7947 3008 13 +4679 3293 11 +3509 2713 8 +8817 3444 13 +8279 573 11 +5086 8131 6 +3208 6081 16 +5340 9799 3 +8308 9102 7 +9970 6861 9 +911 4383 16 +8237 4511 15 +470 2534 11 +1593 9044 13 +3494 553 15 +2279 6522 10 +8419 1997 6 +7197 1506 9 +5070 1687 17 +917 3276 8 +253 8023 5 +1532 9596 17 +9455 7783 1 +4363 5139 9 +1487 5384 1 +6107 8911 10 +1584 9860 11 +2310 9049 10 +2736 4469 10 +5864 3180 10 +299 8135 16 +372 9127 6 +6677 1374 4 +9877 195 8 +402 9903 4 +6572 1366 8 +5756 5645 9 +9789 1644 6 +937 2136 7 +4610 491 14 +4218 4162 11 +8530 3472 17 +6589 5824 10 +7638 3675 3 +4791 2527 4 +2738 6378 10 +2802 7543 5 +6345 5894 11 +3863 1324 13 +6074 4177 7 +2494 5454 12 +8926 3612 11 +4888 6060 6 +6656 9144 18 +5951 588 14 +6167 664 14 +698 5767 6 +7145 1881 16 +7064 4406 5 +8994 4409 10 +7654 7306 12 +5572 3535 12 +5876 3548 11 +6499 2622 9 +5161 970 11 +9345 5198 9 +3238 4038 8 +2489 4372 12 +155 3110 8 +4075 5073 17 +7077 2644 10 +8464 7220 14 +515 5397 8 +9902 9364 6 +8782 250 13 +5163 9451 15 +5331 9484 5 +6391 7137 6 +9370 867 5 +725 8748 1 +9210 356 11 +6062 2887 7 +9920 2666 15 +9000 3395 13 +1990 5582 15 +3286 9626 14 +8430 3643 10 +7239 9525 10 +7712 398 10 +6130 9023 15 +6927 4945 9 +8855 2209 5 +4632 9790 5 +9331 2079 11 +3671 5182 15 +2949 4262 9 +6687 2051 5 +5394 3205 16 +9011 354 4 +7050 1354 9 +7757 7808 12 +960 9371 11 +7960 184 15 +9811 5812 2 +5521 6581 4 +6406 1700 6 +7406 5638 5 +4960 3158 16 +3586 4728 6 +9677 2457 4 +2233 6544 13 +5618 3407 8 +8307 908 4 +2364 1516 7 +2885 8606 8 +5928 1092 9 +5026 4359 14 +2682 3323 13 +2315 6224 2 +8461 6000 12 +9703 3624 2 +655 4187 10 +1467 3400 12 +9890 2772 14 +2913 811 8 +6705 41 11 +5356 6061 6 +3922 4851 7 +4829 475 12 +5681 7014 7 +1758 745 9 +4847 1651 9 +7553 7148 17 +8012 5280 8 +1785 4976 5 +8390 5930 1 +5722 2851 8 +734 2922 8 +9813 6408 5 +1433 9076 14 +3244 8602 3 +4990 9971 5 +8698 7412 9 +7284 7244 13 +4008 1540 13 +2602 9167 2 +868 3848 9 +4546 4964 4 +798 8601 6 +5566 7254 10 +2627 2632 3 +9741 6931 13 +4631 120 5 +9777 2823 12 +8434 2769 13 +6828 467 4 +3605 2618 4 +1978 3312 9 +2569 6988 14 +5239 7929 10 +925 1408 3 +406 3123 9 +6958 1247 17 +3457 8436 10 +4548 1380 14 +4877 3704 9 +8969 5981 7 +6513 6449 9 +6165 3729 7 +4214 5371 14 +6638 1179 6 +7006 8537 12 +9348 7090 6 +4587 7083 10 +6263 5044 8 +6073 9711 8 +5195 9350 8 +1517 7438 6 +6173 3679 10 +9181 7181 10 +176 3931 7 +6086 7685 6 +2603 5069 9 +5125 2346 7 +8612 5516 5 +6088 3096 10 +7988 9913 7 +1466 7435 8 +5374 4468 9 +941 7048 8 +2495 4145 10 +7963 1267 16 +3946 6185 6 +897 1718 12 +8808 4310 5 +3027 7826 6 +8099 2845 10 +1933 7619 9 +654 7916 6 +8241 2826 14 +6865 8828 10 +3246 6805 15 +2288 5368 3 +341 3201 8 +6273 9504 10 +4914 1131 11 +6755 7590 11 +4389 5018 10 +5453 276 2 +6327 9423 17 +4065 765 4 +8064 6181 7 +1914 9529 14 +127 8423 11 +1541 8989 12 +3374 4111 17 +6366 5054 10 +706 8371 18 +7881 5276 7 +5215 8251 3 +7248 6794 6 +9427 7186 9 +1728 4854 4 +7713 1663 16 +7769 7375 16 +2095 1223 7 +1910 702 12 +4517 7029 7 +744 777 3 +7256 3278 13 +8548 854 0 +1602 7583 11 +3521 696 9 +7096 9948 4 +8654 3812 10 +161 6596 12 +1306 2074 8 +5246 5972 12 +574 1203 3 +4365 2439 2 +7053 1362 7 +7662 1701 8 +9635 7416 9 +872 8779 14 +8071 1912 12 +682 1138 3 +1209 4730 13 +1830 8183 9 +7536 4937 0 +590 5987 3 +9362 8920 18 +373 1755 13 +1754 881 11 +6477 118 17 +9602 6571 10 +501 4913 16 +7504 9567 14 +7150 2756 4 +4189 3432 5 +3260 2326 2 +6157 1859 17 +1463 5989 9 +1798 6141 4 +4775 4205 4 +8605 2359 7 +7125 8676 9 +3177 7484 6 +4149 4046 6 +1897 5794 2 +7341 3866 5 +8270 2084 9 +1762 2690 3 +9108 234 14 +344 185 17 +3772 9753 2 +5815 891 5 +2483 1918 6 +9093 5313 16 +7229 4283 11 +1430 8787 9 +5366 7004 10 +1099 1577 12 +8694 523 4 +5266 5658 7 +3843 6182 2 +7315 6766 5 +3372 8314 9 +9118 8086 3 +4752 8823 14 +9649 9520 2 +55 1590 0 +3258 8539 7 +9465 9721 12 +804 2992 3 +5003 2681 11 +2351 5857 11 +2347 1453 7 +3748 8218 13 +8073 6652 7 +6145 5985 13 +8458 8582 4 +7870 4042 9 +8649 5776 9 +5277 3081 9 +8740 2707 1 +4479 270 12 +5247 9435 10 +5242 4414 13 +3854 7223 8 +5710 628 11 +7709 4514 10 +4652 581 4 +7384 4062 9 +1050 6543 6 +91 2362 14 +8416 6973 7 +9972 7602 9 +2908 3888 18 +1235 6365 8 +9006 3940 2 +2317 3455 8 +7483 1847 7 +9732 1404 8 +9071 5851 5 +3619 2110 7 +7040 4705 5 +2226 7779 5 +3379 9160 6 +493 657 6 +6727 1088 12 +1646 6444 1 +9032 9577 4 +2193 5580 16 +106 1624 9 +8804 3572 13 +8747 5270 5 +5541 2721 8 +5807 832 12 +2143 1042 10 +1288 4341 2 +3334 4674 6 +1018 9864 15 +9279 6945 15 +7759 9413 11 +6879 6665 8 +4335 6525 8 +5870 6216 5 +3212 6064 4 +5418 1748 3 +5950 6102 12 +9731 3143 8 +3124 3396 3 +5116 2454 12 +9863 451 11 +6881 6918 9 +7453 5668 6 +2742 526 12 +9025 9340 8 +9385 6174 11 +2502 4 7 +1030 4993 8 +6717 4324 10 +9541 1072 7 +4390 4881 15 +7120 2903 7 +3965 9943 6 +2018 6585 5 +9607 8008 13 +6309 5768 6 +7645 6723 11 +9858 4079 6 +641 6373 12 +188 489 1 +4401 6546 2 +5900 6019 8 +170 3495 8 +2703 6092 1 +9309 2684 5 +1155 415 9 +7399 8027 9 +1459 8400 6 +4848 7448 7 +7343 6640 13 +3697 5662 11 +9918 2380 18 +4838 653 8 +1424 8249 9 +9637 4018 10 +5220 1688 15 +9735 9363 7 +6288 9894 7 +8379 8504 12 +4103 9668 11 +3969 4542 7 +5085 570 11 +5529 2642 7 +5699 9641 2 +684 4445 9 +7696 4097 7 +9002 1948 6 +4347 6605 12 +9001 2722 7 +4711 9392 11 +2458 6686 14 +815 2876 9 +547 6521 4 +3831 397 14 +6967 6999 13 +1648 392 15 +5949 3553 16 +314 6480 3 +9166 2382 15 +8354 1915 5 +3295 186 7 +8360 9027 4 +9382 2540 10 +7409 534 12 +7837 19 13 +217 7658 7 +7408 8578 9 +3698 7209 6 +9128 9059 7 +7604 5674 5 +4334 8840 6 +1240 6556 2 +5343 6533 9 +8788 1703 12 +9633 8130 3 +5720 8327 9 +7339 7247 9 +7790 4824 1 +3625 6914 10 +1462 3999 11 +6972 5327 9 +279 142 4 +780 3817 5 +8731 7597 8 +4768 9332 7 +5462 6075 11 +9265 1535 1 +8888 289 11 +4662 2054 18 +9617 3963 8 +9240 5557 8 +5695 4455 8 +2108 1070 7 +6032 5192 11 +8916 4984 4 +6757 7690 4 +4844 5096 9 +304 9236 4 +2989 6434 10 +2539 5616 10 +8418 9212 8 +821 3307 6 +1343 1800 15 +560 9034 13 +6815 8562 11 +887 670 7 +2065 9074 12 +3062 5423 16 +4505 9914 12 +7194 5022 13 +5284 6206 9 +5982 7788 13 +6502 5166 2 +3429 45 9 +8884 9968 13 +8382 8903 3 +1736 934 6 +7599 6961 1 +9654 7894 5 +586 4820 8 +8620 9068 15 +2156 2418 7 +1841 1395 2 +7955 7577 7 +7999 1332 14 +5910 9934 13 +5376 2484 15 +822 615 8 +9578 1731 5 +9936 7643 13 +5290 2050 8 +5043 2691 8 +3635 3516 16 +3902 297 5 +3750 5558 11 +203 764 3 +7652 5620 16 +7304 7401 8 +2965 6463 7 +3406 8837 15 +6832 7146 13 +8533 8044 12 +9963 4039 8 +9646 2987 9 +2752 2910 4 +578 3417 3 +9926 9192 11 +4641 2561 8 +1469 711 8 +1793 135 10 +3844 9412 6 +2521 865 10 +9833 466 14 +6265 7180 11 +8459 938 4 +5145 7822 7 +7948 6655 14 +1436 5610 13 +4301 128 17 +7731 520 12 +8083 2231 12 +4170 4689 6 +8116 6057 8 +9586 3450 7 +9738 6007 11 +2397 2356 3 +7367 919 2 +3880 9156 11 +7973 7324 6 +5199 446 12 +3986 2857 11 +4112 2200 10 +8948 1206 12 +9568 2198 8 +6783 3488 7 +5804 9394 11 +9136 2860 14 +1238 171 8 +9402 2274 17 +3073 758 10 +8511 2311 6 +9925 8953 12 +9872 6431 5 +8964 4746 12 +7758 643 8 +1693 1428 14 +362 2671 9 +914 2491 10 +4198 4070 9 +5111 1976 8 +6874 5302 12 +8730 4037 6 +7765 6692 17 +7427 6438 7 +5360 4439 14 +3816 2869 16 +8179 6014 2 +4247 4433 9 +9521 6380 9 +550 7697 16 +4461 9752 7 +612 5169 6 +2832 9425 5 +9775 1289 11 +7047 6275 12 +3330 3140 2 +4772 1554 13 +6215 7499 14 +6970 4957 0 +5262 1279 13 +2783 1060 12 +133 2934 16 +2011 2641 3 +6423 5561 6 +9120 9492 5 +3221 4425 12 +6303 4280 18 +5859 7198 9 +482 6822 11 +8442 9996 7 +2149 7296 15 +6636 3435 5 +5079 8117 0 +4978 7451 13 +9632 5271 7 +3009 8716 10 +6376 4731 17 +5484 7423 10 +7671 3800 6 +4919 7270 10 +6152 7152 10 +2655 9822 5 +9146 4330 11 +311 7430 5 +9973 2229 12 +1443 7730 11 +824 3648 2 +9933 752 6 +9601 6857 3 +7852 3683 13 +5370 5765 9 +4966 7084 7 +92 9431 12 +9657 5952 1 +2167 4586 10 +7458 3997 10 +6129 7191 1 +9794 803 10 +452 1080 6 +6315 630 15 +5387 3314 9 +1283 3598 8 +5172 1831 13 +5240 4163 16 +9591 1465 5 +3609 753 10 +634 4983 12 +6527 8790 5 +5801 770 10 +9095 3708 8 +5011 9759 16 +6257 1636 9 +2939 9061 10 +2459 7469 4 +2506 6177 3 +918 8572 7 +2265 8638 11 +5617 7132 13 +3423 1034 7 +7746 3487 12 +8831 3452 7 +907 2532 7 +7541 8448 5 +6311 1494 9 +962 1169 14 +2002 5727 12 +4608 8480 15 +6816 4080 8 +7155 264 14 +5493 1358 9 +6706 8383 14 +4011 1842 10 +7578 1299 10 +3615 2114 9 +1743 112 12 +3663 2178 5 +4676 8598 4 +7622 2426 14 +6508 2006 15 +729 2649 6 +1719 6094 14 +3886 4302 11 +9849 9141 9 +4057 7814 9 +5326 3799 14 +7183 6052 3 +9106 2530 7 +8545 2003 6 +6405 323 8 +4477 9981 6 +2093 4077 9 +8248 8462 11 +4784 3351 8 +3355 4590 6 +9598 7637 8 +2853 1501 12 +9842 3959 6 +9379 194 2 +5064 1490 10 +3163 9175 11 +168 2658 5 +244 2775 7 +9680 7400 5 +6042 5926 9 +6662 2165 11 +8621 5187 5 +1600 8519 10 +1911 2437 7 +1578 2945 9 +7813 7549 15 +8792 2468 12 +3241 5143 12 +8910 8802 7 +8963 8288 17 +5421 335 9 +2055 6198 9 +7613 2152 14 +8946 8472 7 +7271 2174 5 +8668 9322 14 +5051 358 7 +9005 8435 4 +1770 9101 14 +5365 9866 11 +9063 3534 7 +6497 2894 2 +4684 9458 9 +2408 6512 12 +1585 2236 2 +5923 1560 12 +5339 817 11 +2909 2789 6 +8029 8734 7 +7021 154 8 +7470 4956 11 +3063 7350 9 +4123 7615 16 +7058 1497 13 +4207 4629 16 +4105 1257 2 +9058 7616 15 +3462 6661 6 +145 1518 3 +8980 6352 3 +9964 8281 6 +3632 3696 9 +2481 5968 11 +1165 8993 13 +3157 968 9 +9763 3167 3 +2127 4696 5 +7971 3393 13 +3281 9387 1 +6363 812 12 +7393 886 11 +9257 9227 10 +3392 8040 15 +286 1594 15 +4645 9026 14 +2693 6953 7 +8057 3015 3 +2321 1009 6 +1296 8054 10 +9901 1000 18 +1141 2638 7 +6456 2172 9 +1797 1304 12 +9653 2284 4 +8635 9941 13 +2827 686 4 +1447 6770 5 +5546 1241 16 +5254 9204 8 +8401 3242 7 +9433 9565 12 +251 9273 1 +9597 6187 8 +1117 5053 9 +9404 9532 15 +6006 5805 16 +3923 7845 14 +9717 1001 3 +1867 3577 3 +4047 4783 13 +3792 4765 10 +7160 989 11 +1781 8701 12 +8860 1352 5 +2259 2688 10 +1951 1620 8 +3913 6840 5 +9173 6814 14 +1726 6909 5 +2698 263 12 +1341 4144 3 +9164 9705 8 +849 3924 12 +2758 5017 17 +7264 4904 8 +6552 4243 10 +257 8103 14 +4573 6974 6 +4525 576 4 +3151 1191 2 +3211 7859 6 +22 2551 9 +6517 2031 6 +6823 8021 5 +4535 8556 11 +3862 4526 5 +1181 7371 14 +5443 1966 7 +9730 1414 15 +6396 5324 11 +4895 5345 12 +6398 1422 4 +4462 131 6 +542 1557 12 +8302 3653 11 +9585 1723 0 +1188 1401 7 +6095 7309 13 +979 8642 10 +7726 2811 4 +9689 1523 9 +651 2088 16 +5158 4300 8 +1041 5253 5 +3709 6114 9 +2699 9162 2 +7715 229 12 +5499 207 6 +6562 5016 10 +4486 9623 10 +7876 261 7 +4381 5790 14 +1837 1244 10 +4962 8011 6 +6998 3893 14 +7474 6779 5 +1389 7418 9 +8486 987 15 +7419 5458 13 +1146 7672 10 +4794 3839 9 +95 5126 8 +1658 9886 8 +7524 7850 13 +3722 5530 12 +4279 3533 6 +7785 1752 7 +9325 2955 3 +6964 9148 11 +8493 8035 6 +3224 7460 15 +7283 4686 10 +5449 5315 11 +9803 1582 3 +8320 1598 7 +2060 2291 5 +8409 4351 2 +582 4538 9 +9342 1365 5 +6641 3026 9 +7262 7910 5 +1090 7792 15 +8736 4221 11 +4237 2124 13 +345 3549 4 +6159 8282 8 +3759 6199 17 +2452 862 14 +2828 9469 11 +1492 5211 10 +6436 9679 15 +4257 9949 15 +484 8287 10 +9778 7131 9 +453 8412 8 +9600 1308 14 +2191 4592 11 +5704 3788 14 +4328 7584 8 +3072 3901 14 +1610 3631 5 +9787 5703 6 +3528 8742 10 +8405 3189 15 +9461 2035 11 +4113 7996 7 +5766 9518 4 +1513 2144 9 +650 8970 9 +860 2211 14 +8238 9552 2 +3657 2476 11 +2419 136 8 +8809 7275 7 +8559 4670 2 +6296 1709 17 +1614 7267 11 +2222 3933 5 +422 5369 9 +3459 1251 11 +4809 2664 7 +2086 4393 6 +1359 4167 9 +3538 336 18 +4603 4766 6 +1698 1521 11 +3929 8372 14 +9912 5993 1 +7313 2998 12 +2573 6754 7 +4270 6225 5 +3580 3691 12 +1773 6980 10 +9116 1588 11 +8453 1861 6 +6720 9216 10 +3592 1872 13 +9008 3416 9 +6268 3567 12 +3067 4901 15 +4892 2926 14 +5743 444 8 +7303 622 5 +7345 4559 4 +340 3454 6 +8491 3235 5 +9494 7411 7 +5015 8608 16 +1917 8552 9 +3397 756 11 +2154 5479 4 +7355 6469 9 +6229 1732 15 +5687 7342 3 +1959 4322 7 +5244 9450 8 +3271 721 0 +2413 1606 12 +8919 839 16 +7108 5245 8 +6161 6111 10 +6538 6425 12 +8592 7294 7 +5996 6255 12 +6884 662 14 +2834 6845 15 +8689 3431 12 +223 1305 8 +4206 5936 6 +9335 779 6 +2788 7522 7 +1668 7095 18 +6895 5679 10 +6608 312 9 +810 4084 10 +7292 8187 4 +7234 321 10 +6153 5209 16 +8956 5902 8 +1708 4934 6 +5050 8636 10 +7625 7382 9 +8859 6609 7 +9551 6937 4 +5121 3620 10 +8304 2273 8 +3803 3070 1 +722 2033 4 +4267 742 1 +3724 9193 4 +6043 1486 14 +5798 7974 12 +7114 8384 16 +9899 5984 8 +2914 7259 11 +983 7508 10 +8819 6873 4 +5173 5567 8 +4242 6627 12 +5406 148 1 +9760 7515 10 +2643 5337 6 +3910 2266 4 +2109 2997 4 +5734 8618 11 +4954 3559 10 +2420 2770 12 +3504 1121 9 +2659 3261 7 +9411 1827 12 +8359 1902 5 +9603 1037 3 +6880 8938 6 +7214 9421 5 +3539 660 10 +9572 9486 15 +8560 6291 7 +4161 4100 4 +9188 1290 12 +8547 8664 12 +7725 2794 1 +4265 3415 6 +5120 2272 10 +704 629 2 +2264 4869 5 +4657 9009 10 +3841 8258 12 +7033 2097 10 +5664 6214 13 +9354 5055 12 +4864 3944 4 +5788 4742 13 +4890 1262 10 +9951 695 1 +7953 418 2 +8264 7648 11 +4210 1482 15 +5233 3677 9 +5585 7403 8 +5359 3101 10 +7517 5990 1 +1856 6195 7 +5048 7640 16 +3434 4648 10 +9647 1547 4 +4323 2444 6 +8147 8797 12 +5403 5887 15 +9339 2410 0 +7865 4690 14 +6602 8362 3 +1987 9724 0 +4048 6091 9 +3911 9317 6 +7018 4501 8 +6715 4271 9 +4068 3210 8 +4274 4127 4 +7188 9562 15 +1983 4564 4 +9440 8707 2 +9220 9846 1 +6170 4743 15 +1337 2373 6 +5775 8343 8 +6601 5039 5 +5758 1647 13 +808 3604 10 +2027 6965 6 +6971 9485 1 +6413 3229 5 +1569 4215 13 +4739 1626 6 +8358 5258 10 +4085 4786 10 +5705 5190 16 +9318 5460 3 +715 1101 10 +8285 8534 16 +5601 2814 13 +5341 8768 7 +8487 4308 6 +2318 7269 0 +3398 4879 14 +2186 5298 10 +5542 9950 4 +6235 6393 18 +7377 3233 13 +1890 8259 11 +4996 5955 6 +3380 7753 7 +3801 9798 8 +7227 391 17 +558 8093 18 +6994 450 9 +7141 2276 8 +208 9629 10 +7659 8780 8 +8253 1988 3 +8000 4740 7 +6960 507 10 +8551 9017 4 +3870 6251 4 +3194 2995 10 +7581 8037 9 +680 746 13 +7087 6968 1 +4862 9119 5 +2028 9522 14 +801 5480 8 +2270 2040 12 +1152 6069 16 +305 4049 7 +3821 5089 11 +5549 2678 11 +2407 7836 11 +512 518 7 +7513 9843 16 +7683 6888 11 +9329 9814 10 +1339 3996 9 +8772 2734 1 +2936 8781 13 +4863 7784 9 +9271 7389 14 +819 5921 5 +1878 7121 16 +5878 3764 3 +4353 885 6 +7374 5409 7 +9507 8936 11 +8558 1149 8 +8571 677 8 +2583 7692 6 +5633 9198 14 +4843 761 17 +9182 9328 10 +9915 7803 12 +985 7512 8 +5603 1106 6 +441 2254 2 +7794 3203 3 +906 5545 11 +4512 1478 12 +3085 3071 5 +2297 9868 15 +9882 1425 11 +9613 6835 2 +2328 757 4 +8433 3757 14 +981 79 7 +7493 617 14 +4885 2061 3 +5761 7101 6 +4020 7521 13 +3 409 1 +1640 7075 12 +4582 9781 9 +6208 5287 14 +1031 9474 5 +9262 5234 14 +9821 8812 7 +9604 8615 5 +5107 1242 11 +5682 1973 13 +2566 339 9 +8316 8142 12 +4102 183 7 +9502 8052 11 +7286 9749 12 +8974 3342 13 +7595 2480 10 +6604 3225 8 +2704 6890 5 +6390 5028 8 +6139 6484 4 +5103 4400 15 +6584 683 5 +688 8492 8 +8134 7934 8 +9149 9766 10 +7673 5651 6 +1849 513 6 +2404 5642 5 +3451 6470 7 +6476 5721 5 +7739 8815 9 +9999 8644 12 +1110 8209 15 +3502 9840 11 +2170 1200 14 +1882 7909 10 +8097 4811 15 +4312 9004 5 +6920 620 2 +6068 8152 6 +4779 5898 12 +3387 4813 13 +2668 5093 6 +3311 6906 7 +5754 3338 18 +9311 1355 7 +7810 8078 1 +524 310 6 +4616 8041 15 +2118 8377 12 +850 3474 3 +146 8056 8 +6982 2512 11 +8494 2463 10 +5596 2455 3 +3401 4635 3 +5861 1378 12 +4655 5056 6 +2695 8744 13 +2029 6737 9 +8968 1065 8 +2187 6140 7 +5184 6926 14 +2225 6507 17 +9190 7742 6 +2519 7776 9 +8913 7711 7 +7542 4438 9 +6184 5171 9 +1765 3273 9 +4176 7552 10 +9365 6928 4 +5514 5825 5 +3517 3269 10 +6573 316 11 +1672 7138 6 +1334 2301 12 +607 637 9 +3701 5014 4 +464 1924 7 +8309 5729 12 +6612 5150 11 +1735 6058 5 +7984 9737 5 +4900 3824 10 +8630 9246 7 +7714 2357 4 +2130 152 9 +7054 6093 4 +7443 3977 9 +1177 8921 2 +1282 5856 18 +7476 5033 3 +9446 2901 6 +52 9295 6 +4219 1869 14 +1844 988 2 +7392 7386 9 +4226 204 6 +9758 7878 11 +1233 7892 14 +5719 8393 12 +2441 4306 6 +7472 7388 7 +2735 413 16 +5472 5151 7 +1767 9344 8 +8351 5543 8 +829 2296 6 +9283 8951 3 +9661 4551 8 +2982 9121 2 +4125 4203 12 +6639 5267 8 +301 8366 12 +3875 8373 4 +1386 5783 8 +583 1666 11 +5819 3712 12 +209 5777 17 +4171 6314 6 +9883 549 7 +9064 2073 9 +2898 2635 6 +7630 9060 5 +7420 9895 9 +3656 3300 10 +4942 8952 8 +8333 5147 6 +4473 901 4 +56 2933 10 +826 7754 8 +3000 3117 11 +7009 7981 7 +156 2363 10 +5594 8076 10 +4407 119 11 +4998 449 7 +1536 5882 9 +7310 9453 10 +9133 4524 6 +7346 147 8 +6024 8625 10 +7143 3277 15 +5154 9773 14 +6852 2440 11 +6416 5678 17 +5152 4058 13 +7300 5322 11 +8988 5826 7 +6412 1939 3 +6592 4581 12 +2868 4143 10 +9953 6616 8 +2085 7939 8 +5156 3818 1 +594 3136 13 +2497 8891 16 +7847 2120 5 +3714 7717 7 +8805 5077 14 +3530 5282 6 +9546 1357 11 +1527 4116 9 +9174 6741 7 +7700 8749 16 +9898 9467 10 +9673 8386 13 +7263 5159 4 +164 9375 14 +3236 6644 8 +1234 8066 17 +9947 7311 7 +8928 3223 11 +6191 5820 2 +1756 9206 17 +9645 1635 6 +4576 2715 8 +1405 7550 9 +7056 2434 5 +7491 1979 9 +1375 7594 11 +3892 1657 9 +8836 7840 6 +5202 9663 4 +3020 325 4 +7818 4724 12 +4574 9378 2 +4345 7051 8 +1412 3627 10 +6954 5 8 +9510 2692 11 +6516 68 7 +5598 6067 10 +3752 479 13 +8210 9288 16 +8485 1109 6 +2966 5869 18 +2531 7872 8 +5226 1931 13 +5474 174 6 +6313 9671 9 +9993 2100 5 +2044 5401 8 +7667 4001 10 +8269 8300 12 +4715 4673 16 +1275 6399 10 +7608 8120 4 +1250 6503 13 +3294 1010 11 +7897 4245 15 +9955 4955 8 +220 1215 10 +7926 1729 9 +5554 8374 9 +4284 8454 15 +5896 8965 9 +4814 46 7 +2629 6482 4 +9755 2718 14 +1049 8088 3 +5213 2796 12 +6554 3961 8 +2396 2448 11 +6947 1751 4 +2647 5495 11 +7575 5265 8 +7022 1888 8 +603 5417 10 +2411 9593 2 +9035 6849 7 +2125 1612 8 +9808 6446 11 +1164 7378 15 +4593 7074 9 +5348 3641 5 +2194 4063 12 +4622 7890 7 +1024 3826 9 +9099 3419 5 +5097 8337 10 +5362 1863 6 +4601 5146 11 +3207 1284 5 +2741 805 10 +9896 3303 8 +1186 8907 9 +3827 7287 14 +6774 672 7 +6983 2376 4 +5724 8987 8 +6501 4275 5 +3802 9383 8 +2798 2021 10 +1168 4427 7 +7610 9145 5 +4286 6077 5 +425 2368 8 +1270 8410 12 +9733 3768 9 +8924 4857 3 +3384 8106 9 +7885 4663 13 +4927 2787 3 +5874 6293 5 +218 5278 13 +7482 9930 2 +4394 2352 12 +1104 1566 14 +7913 421 2 +8982 4130 12 +4861 1865 11 +5828 4553 9 +5434 3725 14 +483 490 5 +1510 7975 11 +7627 179 10 +1962 8617 10 +6066 977 5 +1159 5570 9 +4933 4909 6 +8931 7607 7 +7118 7426 18 +5524 4410 10 +1475 7851 12 +5712 8579 12 +2716 125 12 +3033 5793 12 +2579 3253 9 +8267 6691 15 +8675 4995 6 +2393 7091 9 +1860 625 11 +4450 3476 4 +3367 3343 10 +6316 2895 15 +5693 4488 12 +7081 6535 12 +7465 5249 0 +5726 2988 5 +6688 1710 8 +6768 8813 12 +4456 9067 9 +6307 445 13 +8402 5436 9 +5933 6241 1 +6429 9874 2 +1621 9837 7 +7689 390 10 +4556 5965 9 +2950 3248 4 +2525 3483 14 +3930 9873 5 +8479 2251 13 +2517 5268 11 +1555 8755 5 +4604 7258 6 +9320 6933 10 +2049 1294 9 +5938 9599 7 +8789 2663 9 +6134 191 7 +4467 8104 13 +6875 8111 3 +4109 3810 8 +5508 2745 6 +4666 8468 2 +6418 6203 4 +9583 1944 5 +676 1984 9 +1054 7299 2 +4781 9614 10 +4022 6302 15 +8666 1607 4 +3660 7327 2 +3531 6427 7 +4949 8357 9 +8039 4225 11 +9104 4156 6 +5635 7650 10 +6457 1207 10 +8726 6579 12 +2831 8680 15 +6493 351 13 +6409 8018 15 +3678 4931 6 +9243 8960 12 +1512 1772 16 +1231 4799 9 +7729 5626 4 +7676 7089 14 +5303 4482 8 +6574 7563 3 +9805 4780 4 +5264 6281 4 +7944 9003 2 +6166 8877 17 +3373 3332 11 +5846 2353 10 +7854 5296 15 +4585 5478 17 +7135 8890 8 +81 1972 6 +9888 9180 7 +6475 2340 8 +8456 6539 15 +8336 5430 6 +411 784 11 +5714 2059 14 +9452 8914 9 +2394 1144 9 +23 8034 10 +2778 3526 13 +7518 5238 9 +8204 1153 10 +6270 8188 12 +9533 5967 13 +7034 5225 11 +2169 8923 6 +1432 9113 9 +1930 1002 4 +1145 1802 6 +8284 6040 7 +4437 8501 3 +8897 8198 11 +1077 4540 11 +9251 9207 12 +9417 7525 11 +2574 6432 8 +1981 3956 10 +991 436 11 +9170 2014 18 +3736 1896 12 +8132 708 9 +3029 4376 12 +2104 781 15 +8851 6989 2 +7656 8291 9 +6048 6017 14 +2252 9473 11 +1005 6 13 +4197 3445 4 +626 6213 10 +74 4706 10 +533 561 4 +7567 6355 7 +719 4695 8 +3905 26 15 +8121 8915 6 +519 2790 9 +6541 3861 11 +7905 3016 3 +275 6847 10 +4825 9958 15 +3687 8546 15 +1171 8 12 +4029 3556 9 +5929 1226 7 +4575 9976 10 +2235 5427 3 +1817 2374 4 +1107 1229 17 +8702 9566 12 +6104 8098 6 +1574 7708 3 +4200 3409 10 +3022 4598 9 +7251 5961 13 +3216 3551 12 +3035 7677 9 +6734 1871 6 +375 6267 12 +1481 949 10 +2801 6169 10 +6295 1673 5 +14 4193 7 +6560 7035 17 +7109 5250 10 +3871 4396 12 +4007 3365 13 +5647 4227 8 +724 3738 9 +99 5328 12 +5021 284 14 +318 557 9 +1195 2221 1 +8471 2224 8 +1205 5591 9 +5562 2560 5 +7127 5665 14 +3651 252 10 +9965 6458 10 +7203 2686 5 +6649 4261 13 +7431 1998 12 +3422 313 9 +3418 1813 15 +692 8100 6 +3052 9098 4 +80 6279 9 +8177 1895 8 +2285 679 14 +9472 6237 15 +8784 796 11 +4441 8738 6 +7102 4723 6 +7531 9838 6 +3626 2219 16 +2398 7516 1 +2477 5133 9 +4417 2133 11 +1690 794 3 +8616 5848 5 +6395 9105 8 +1866 4266 9 +2295 4184 9 +3723 6096 12 +2705 7545 9 +8538 4316 10 +1660 2400 12 +6097 2592 0 +7151 4021 3 +3876 4916 6 +1011 5005 2 +6775 4095 14 +4898 1170 14 +419 2471 7 +7475 4194 9 +673 8640 18 +5431 6796 3 +5531 9495 7 +1604 8853 7 +143 3693 10 +9854 6383 8 +159 7088 11 +158 2523 10 +2859 3196 9 +7665 1134 4 +9906 5901 10 +2062 5439 10 +7902 9419 14 +9304 831 3 +6951 6672 4 +2780 9062 4 +3849 3813 8 +1457 1543 7 +1829 9305 4 +6131 9316 8 +910 1925 11 +4420 4567 9 +1269 8677 8 +3402 6537 11 +1312 9909 15 +1122 7272 10 +347 2522 8 +3257 4233 7 +4424 4842 13 +1439 8509 10 +6654 9056 11 +3166 9547 9 +3133 355 12 +6022 1190 14 +4509 5973 6 +2762 2134 11 +3639 611 1 +8019 3178 2 +7459 389 18 +2800 9727 11 +7277 9457 8 +2584 2150 14 +2427 6733 4 +895 5200 4 +5606 7351 8 +1451 3719 9 +7295 3797 11 +5611 687 11 +4518 6120 13 +2064 7603 13 +6530 3321 10 +7334 1926 10 +6464 7307 15 +129 8622 11 +8388 4941 2 +6510 210 4 +597 2572 2 +6278 1328 8 +9051 6578 9 +2609 3565 11 +3783 6178 17 +6987 9517 15 +6646 5839 4 +5320 9310 0 +8420 1504 1 +6637 4168 5 +9 5122 16 +1814 4395 13 +6013 2943 5 +7954 8825 8 +9241 8655 4 +1228 6780 13 +1315 4963 12 +2001 9287 14 +2184 9226 8 +2076 2862 11 +3796 4618 8 +7086 5505 0 +8871 5124 7 +6156 308 6 +6256 8277 7 +2805 9235 8 +426 1295 10 +8395 8172 4 +7991 9786 14 +6072 8631 7 +1714 9610 18 +2884 1211 4 +3912 5632 7 +3877 1376 10 +417 6855 16 +7831 3388 7 +5917 7354 1 +9229 96 4 +2867 1947 3 +9066 4253 14 +8830 5363 9 +4834 5482 7 +505 4727 6 +4430 8186 6 +5117 9954 8 +9962 3279 8 +9016 9437 6 +4873 8226 6 +3640 3874 9 +9356 9560 8 +6204 8280 7 +9834 1713 8 +5155 1291 3 +2829 326 7 +6707 2761 6 +7361 7694 12 +5299 9481 14 +3477 4778 10 +4276 8498 5 +3773 7008 11 +8577 6039 13 +175 34 14 +499 4050 7 +9660 3060 16 +998 4531 8 +7447 2580 7 +6326 2760 16 +319 3202 12 +1403 2576 2 +2893 189 4 +5336 4446 9 +8229 1353 9 +2606 285 7 +5799 2230 5 +6615 2298 14 +4845 298 10 +5715 3050 7 +9367 8137 0 +6337 2423 10 +48 2122 6 +1722 4704 8 +1048 8193 13 +9155 5176 9 +6361 6417 11 +370 4405 16 +9592 5060 13 +3084 2425 15 +5413 1387 14 +139 2954 4 +4764 4680 7 +6950 8786 10 +5382 3378 10 +9263 7030 15 +9621 7618 1 +2015 7468 2 +3985 8329 13 +723 3510 2 +8047 9729 7 +4762 3348 13 +4647 25 3 +115 443 4 +2815 8656 6 +2765 666 7 +6403 1368 9 +9161 1013 17 +7280 7187 6 +9142 8095 13 +4709 1638 2 +5675 529 9 +6957 6284 7 +6287 9487 2 +7898 9961 6 +1192 8042 15 +153 5723 7 +9110 8290 11 +2946 9230 1 +6728 7666 13 +8528 4248 2 +8024 5372 13 +4649 7407 9 +5845 896 7 +5076 8028 13 +1163 8151 11 +2488 5713 6 +7716 162 7 +7078 2482 13 +1232 6657 17 +6301 6452 13 +4333 4977 10 +5475 4897 8 +1254 9085 6 +1834 4737 9 +1458 7347 15 +7862 9754 13 +4440 7398 8 +9690 4835 16 +4374 9462 13 +2220 9801 7 +33 7928 5 +3364 5877 14 +9301 3568 5 +8570 3465 7 +5512 8710 12 +7487 3102 14 +2282 1495 8 +4296 1174 4 +3103 4140 15 +2492 1082 5 +462 1796 6 +2465 7745 14 +6071 7722 10 +29 5483 1 +6993 1938 6 +5564 3234 8 +4902 6750 5 +5467 7235 8 +743 8758 7 +7631 2850 11 +7241 9685 10 +7628 7243 12 +3092 3247 5 +5994 2687 6 +3356 3860 12 +6021 4925 7 +502 9869 12 +866 5259 5 +3485 1263 13 +4682 1961 17 +2994 1485 8 +3350 9137 6 +2978 8139 5 +2553 6854 5 +3716 9855 11 +9294 6163 7 +2343 65 5 +8810 8995 3 +3623 2083 8 +9747 8338 5 +7038 6674 13 +506 9692 10 +4549 9862 8 +6076 5708 7 +7057 3306 5 +4946 7498 9 +9848 3045 4 +8510 1499 15 +6274 2102 10 +9151 4803 11 +996 3732 10 +3798 8984 16 +3322 3610 4 +6388 4821 13 +606 4650 8 +6761 9464 5 +7833 3908 10 +2269 5019 5 +2877 9682 5 +494 4463 12 +4930 7328 3 +4894 8674 6 +632 5547 10 +212 3132 10 +8161 7274 7 +5030 6100 10 +1147 5025 5 +4146 4035 1 +2331 1078 10 +4733 1545 10 +3647 486 15 +3131 9022 6 +4412 1617 9 +6590 5501 6 +60 9919 14 +1784 7647 14 +3804 6622 1 +3753 6763 10 +4790 7213 12 +8904 1923 10 +1047 6415 5 +5925 1027 4 +5057 9372 11 +7225 2874 7 +359 2839 14 +2835 7931 13 +6653 3149 16 +852 1272 12 +2446 8929 2 +2067 652 1 +3501 6001 17 +9797 2245 4 +331 593 11 +4502 6407 10 +6258 54 12 +5452 2661 1 +5716 4788 2 +1298 2068 12 +1014 5210 15 +404 8381 4 +4747 1277 16 +3793 2290 11 +7657 3695 4 +9445 9524 6 +6938 783 2 +9687 2022 4 +8009 5392 10 +5506 103 5 +5285 5052 6 +5814 5350 4 +2345 3519 17 +3361 3925 8 +7923 1103 17 +3899 144 9 +1471 473 6 +6877 6631 8 +1954 2157 7 +2969 8332 14 +6767 1667 4 +6027 1819 8 +4640 3747 9 +807 943 9 +8312 7993 7 +8925 9373 10 +3936 5862 12 +2042 8653 5 +1742 5357 9 +1111 4523 12 +7824 1971 12 +8127 7193 3 +649 2415 9 +5623 8659 4 +4370 6813 5 +7976 6882 7 \ No newline at end of file diff --git a/examples/mnist_add/datasets/train_data.txt b/examples/mnist_add/datasets/train_data.txt new file mode 100644 index 0000000..c6e404b --- /dev/null +++ b/examples/mnist_add/datasets/train_data.txt @@ -0,0 +1,30000 @@ +55431 23531 12 +29016 31531 15 +39520 30760 6 +14132 1921 5 +51122 56131 7 +24589 34903 17 +42015 27060 4 +20862 49322 2 +20762 25455 8 +36075 50076 10 +10877 42191 12 +20496 46974 6 +20707 27977 10 +48960 41052 0 +46052 4949 13 +4330 22196 7 +7754 53585 9 +41766 50283 10 +7210 14485 12 +29904 1510 4 +59719 45022 11 +32652 55323 7 +32077 38468 9 +33652 40292 10 +43165 5662 8 +38260 35335 10 +41599 14084 11 +16767 13251 9 +57884 27580 7 +21975 30466 6 +14779 47883 10 +32616 44322 12 +54387 22157 14 +29999 10455 2 +33202 28358 8 +4756 42051 8 +37470 34165 11 +57595 38483 0 +380 57250 5 +43993 43282 17 +22267 36206 4 +43136 44117 12 +28771 12562 9 +43244 43962 8 +48770 42619 10 +1505 43783 11 +8215 44727 6 +46377 54619 7 +45319 59198 9 +35148 11900 9 +28401 876 16 +11831 8152 9 +36564 23159 9 +49279 41231 16 +36728 14193 6 +13403 1254 11 +7184 52226 8 +58365 16547 7 +4753 34892 11 +52062 18747 15 +42681 2335 4 +10150 6116 5 +53398 5112 14 +53823 34531 2 +43104 48943 8 +55059 19222 10 +11267 14419 0 +25647 47245 11 +17850 36152 11 +4856 6719 5 +19473 18670 13 +54518 24110 14 +24407 20806 9 +27272 14869 10 +37955 9926 7 +58147 34391 6 +59488 25007 4 +58412 28974 8 +40268 38791 13 +39714 30410 5 +54078 27720 2 +55633 35139 12 +11332 14571 8 +19028 6250 9 +51157 43626 9 +24402 40544 5 +20857 13423 15 +4505 24970 1 +41108 38902 5 +58994 1743 13 +50799 22331 5 +41603 27018 10 +45088 19433 10 +36639 55569 14 +47960 1718 4 +32301 42581 5 +6866 16757 8 +52193 53268 9 +36481 43355 13 +8518 20644 18 +9932 15815 6 +5562 45663 10 +58904 46134 9 +59200 54914 6 +39232 34716 4 +12434 528 9 +42919 52856 12 +24075 31156 7 +15980 26681 11 +30093 44925 9 +51282 54713 16 +24367 55402 0 +5672 55524 10 +2756 59208 12 +56670 40330 16 +29398 54104 3 +15702 2064 12 +44998 18660 12 +8268 25419 11 +23987 49733 12 +33084 52534 7 +43524 42290 15 +21912 39511 10 +18545 17825 9 +16743 42048 12 +33607 46986 6 +21567 50308 10 +31137 37677 7 +34051 1941 3 +1962 5139 12 +5501 15712 11 +56562 34058 9 +23828 55459 5 +38774 15136 6 +32197 51562 12 +38358 53658 11 +47994 23287 4 +42346 2726 7 +59977 35952 11 +38858 9793 15 +44883 8417 6 +48920 51375 0 +16214 38461 4 +33940 30527 8 +52844 52522 10 +29814 52624 5 +4573 18478 13 +39493 36097 9 +56471 2809 5 +45742 3341 8 +15595 54521 12 +40108 51491 0 +31318 219 6 +12290 6553 12 +4437 3026 10 +56965 13533 6 +41245 3781 9 +29261 34437 17 +24138 45217 10 +51818 45073 2 +53546 25842 5 +34703 1286 15 +1889 18509 6 +50695 516 7 +40064 7737 15 +45318 16097 10 +32481 39710 2 +30639 20598 6 +28801 1628 6 +19459 7175 1 +46322 37941 7 +34746 50391 11 +25412 50529 5 +54197 42727 13 +52292 58731 13 +13952 33742 8 +31887 25471 5 +2036 26631 13 +8836 51672 2 +59611 41069 7 +20316 54074 12 +4855 9887 4 +51053 4908 13 +24477 28891 15 +7573 19720 12 +22760 13495 10 +21370 30232 9 +27987 27197 11 +28934 49699 12 +18272 9181 9 +1991 35461 9 +20011 44066 10 +4070 51759 4 +50186 4039 15 +9758 3786 18 +45040 774 9 +12487 42012 7 +2795 23322 12 +16980 1561 7 +10770 39984 6 +13254 9355 4 +41030 57634 0 +31054 48788 9 +36966 51517 8 +24207 49688 11 +16847 29023 9 +28645 18193 18 +56910 5927 1 +39922 17516 10 +443 53792 2 +39638 38564 9 +54255 11419 10 +26623 28880 7 +36359 29044 4 +32971 40800 17 +56738 17915 8 +56618 32493 9 +45373 56552 7 +26269 5777 8 +49130 39096 18 +11421 4628 10 +50026 45981 4 +56208 26780 2 +32913 58485 10 +59011 30905 4 +3746 34606 11 +29130 35040 9 +47957 4282 8 +5044 56016 10 +42587 31708 10 +48884 23695 5 +50754 38921 16 +56589 28371 8 +13943 35671 4 +11502 7837 2 +39205 49947 14 +24522 51792 11 +21857 17266 4 +10815 32631 13 +14180 43984 6 +23336 34161 12 +28810 51523 13 +2314 45726 7 +29890 43579 13 +27407 43490 1 +27638 16458 1 +46569 42394 11 +1642 50860 14 +2044 28036 10 +49307 6130 10 +55249 43594 9 +18645 47559 7 +34398 11414 13 +10236 28091 6 +16939 29986 10 +58766 16211 7 +40500 33048 8 +51871 15075 6 +18564 20078 2 +23186 16666 12 +9283 4319 8 +15072 4135 13 +26556 5657 8 +48682 37973 9 +52359 43785 10 +42470 13292 15 +25254 9275 17 +42613 34214 9 +32945 17780 12 +52373 17396 8 +23546 26930 5 +3160 11124 4 +26843 23753 0 +49241 12651 14 +36288 43557 4 +9301 14134 6 +24197 2677 18 +7297 46606 10 +1856 28809 8 +14867 33539 13 +11785 36225 14 +50884 37073 14 +46544 3618 12 +53460 55814 9 +54755 28647 0 +4509 24260 9 +41990 34140 7 +56073 11470 8 +37775 27923 13 +40055 14361 6 +36499 8385 9 +46216 43853 11 +25076 4265 6 +20062 49240 15 +18726 27792 5 +58819 42435 1 +34364 10776 5 +26965 13246 9 +19291 55221 16 +15921 34122 6 +44586 29683 13 +34995 56849 6 +13160 15634 7 +39573 38530 7 +53886 56734 8 +26494 33709 7 +10268 24338 4 +15111 34799 6 +25189 47498 9 +32196 53673 13 +28 13189 10 +43914 39733 15 +11321 1742 3 +48365 12591 5 +52680 5273 13 +29187 59407 4 +33439 9411 11 +23429 10551 12 +32770 30666 12 +5769 16047 9 +27386 40221 10 +22903 9236 8 +3050 51960 6 +5806 11298 11 +36157 29663 16 +47621 59158 4 +23568 58991 6 +53553 27556 12 +33871 38224 13 +40612 43175 5 +7027 43039 8 +21517 5727 13 +5381 32845 6 +59268 25426 11 +44181 52620 9 +41734 23462 10 +26920 13744 13 +21656 26423 15 +12247 38693 4 +48376 28658 9 +39147 10852 3 +9394 44811 9 +34772 46331 12 +6306 39830 10 +56089 35010 4 +3225 35833 11 +50551 11050 7 +34150 17962 15 +21417 13149 11 +23446 32333 9 +51235 12043 8 +38397 1255 10 +8488 43649 8 +30183 19765 4 +9322 54996 3 +11580 5958 12 +2298 41266 1 +53637 54754 4 +31655 47707 13 +11656 15497 8 +13282 7271 6 +46320 36704 10 +14755 50665 14 +39498 59287 9 +6023 25654 5 +21041 17837 10 +18712 1952 2 +42079 29307 13 +3571 13639 11 +32460 15804 12 +37242 15657 18 +15487 50387 14 +7659 43706 5 +59858 2583 10 +8150 23058 8 +26627 51180 11 +38118 58216 15 +20847 31117 5 +45020 38227 11 +6460 53007 15 +50613 57608 6 +35657 38343 5 +26475 1052 1 +21589 1195 8 +17735 45332 9 +7620 20608 9 +39066 47956 5 +26476 57863 4 +35545 13389 11 +19808 151 8 +51405 46650 13 +58723 36636 11 +38824 35835 3 +16250 35243 9 +23603 23082 9 +33536 45262 6 +6186 12883 2 +1080 34556 4 +59093 40265 10 +59711 39674 12 +57495 1318 9 +742 41721 6 +35219 41457 11 +17851 49796 3 +21190 33540 10 +50906 8286 10 +6758 11602 8 +50514 27805 8 +24913 855 4 +28225 30260 11 +24885 15665 8 +9161 11428 1 +77 3419 6 +35493 36028 9 +57681 30092 17 +24508 15124 18 +38816 30605 0 +39507 48485 5 +19229 55709 9 +37411 43221 5 +28241 54502 8 +32824 59764 9 +49564 30904 15 +15968 25856 7 +38402 841 10 +23616 15232 5 +9391 39627 6 +9027 3692 7 +42342 11873 5 +12450 12321 6 +33767 43059 5 +51739 49119 8 +58575 48705 5 +798 32076 15 +52176 43324 16 +37091 4357 14 +41236 37916 11 +38733 18024 11 +59761 19482 10 +3755 215 4 +49062 52222 9 +40504 8952 3 +50744 27604 3 +52953 18800 3 +46561 10453 7 +52862 33471 11 +10583 42412 3 +6360 53372 9 +59471 31626 9 +10831 7128 5 +57212 25698 17 +5613 54326 6 +45759 56839 12 +49517 22372 10 +4971 25499 7 +40596 54944 15 +22381 1394 8 +19041 35279 10 +4694 4522 7 +37818 23869 12 +30718 8045 15 +19781 14163 11 +15910 381 2 +9695 49735 2 +35085 15074 16 +31956 41637 3 +8520 22154 11 +25586 18498 10 +48552 8638 14 +22698 34861 13 +58232 20976 13 +31453 40963 9 +52376 59558 1 +58158 32520 8 +25171 48095 13 +39722 56715 5 +48950 50438 12 +24267 15122 8 +41600 31704 9 +51260 48959 7 +16928 3253 7 +10720 50609 12 +44360 12332 12 +36912 17848 12 +33255 34397 17 +58980 52377 8 +26870 53933 17 +58413 35189 18 +23218 52340 12 +54566 30894 6 +57208 3269 4 +8609 15126 7 +35338 38249 16 +2491 47916 13 +56780 23406 11 +42415 3509 8 +42914 8442 12 +28290 4680 11 +14506 9127 16 +30395 28000 10 +50645 24912 10 +21349 10318 7 +22733 14665 11 +54447 19910 13 +29171 35017 10 +2928 3374 14 +9821 4569 5 +25820 50150 4 +14346 40702 5 +48029 11345 7 +2260 5375 2 +27213 23517 7 +40379 23707 10 +53568 50401 8 +3436 18539 11 +50289 39559 14 +46068 41924 12 +54728 18752 3 +6 29617 5 +38623 17432 10 +40828 23482 9 +16563 48208 7 +35816 24392 6 +11724 57990 11 +51401 1956 3 +45734 4919 7 +4989 12578 10 +31116 21400 7 +5353 33654 1 +21409 27303 7 +43114 35284 15 +57647 58873 5 +47269 801 8 +53738 42502 5 +10701 32344 10 +6054 54001 10 +49588 57896 10 +27545 24060 10 +1169 10751 14 +48329 26449 11 +54892 10640 10 +22946 37035 6 +44610 56372 9 +33284 38799 15 +27155 18459 13 +40501 27515 15 +5115 51740 5 +45080 42519 10 +46743 11489 6 +9803 51870 3 +2384 8947 14 +54778 17112 12 +24939 6953 9 +318 49844 11 +58014 11709 4 +41098 28717 9 +12092 20236 9 +56789 29248 10 +7038 13020 6 +41392 2278 1 +35440 27095 12 +25073 25508 9 +14847 56285 9 +51988 24176 10 +26495 13096 10 +52369 40669 6 +34491 57791 15 +42634 11662 8 +29978 34968 17 +13677 18683 9 +58678 42040 4 +8091 15276 13 +28947 55226 12 +27123 57275 7 +39661 1692 5 +34283 23497 5 +50025 56005 3 +26588 56579 6 +52843 13296 10 +33143 6291 7 +42178 34421 3 +47313 2452 8 +20322 57261 1 +1943 42650 15 +3023 21305 5 +17565 20257 6 +44736 46476 10 +29656 13562 16 +43926 32668 16 +54487 264 12 +8208 31033 5 +59237 15315 11 +36571 7081 6 +53819 32588 5 +17844 24470 5 +32949 28812 9 +59680 11901 1 +59686 27172 4 +12594 51423 12 +38109 39708 5 +22782 7372 9 +45816 5450 5 +40463 40410 11 +56524 51570 9 +30523 10415 11 +1990 54762 8 +9168 26154 6 +7561 5191 3 +11324 53941 7 +26498 6045 6 +19738 1961 5 +34660 59515 7 +11911 25477 8 +41649 18739 4 +47810 9629 12 +18340 13506 7 +42118 22296 18 +21192 39488 11 +12741 44424 2 +3947 17111 14 +35576 33435 10 +27945 56888 14 +55039 13509 11 +39470 4187 4 +45187 48348 12 +42975 9914 6 +58160 7264 9 +24920 3707 12 +1180 52639 4 +16609 7653 9 +56463 58640 9 +59911 55379 6 +38152 23636 10 +54444 3922 8 +13376 25327 4 +11507 51540 6 +42477 29383 6 +57165 32589 12 +55976 50573 9 +37186 35650 8 +54344 30678 8 +54533 47436 7 +12167 48887 8 +29149 22399 13 +51329 10642 13 +20732 51183 16 +7407 51666 10 +4961 17584 10 +14782 26762 16 +56083 6401 7 +38653 56856 7 +34412 5670 9 +1601 6656 5 +52036 12904 8 +29386 29715 1 +30338 48680 11 +2076 32416 10 +12555 8603 4 +23641 9327 11 +5470 32557 0 +597 44620 10 +44902 30766 7 +51781 32613 8 +49016 42200 15 +18993 20744 7 +20659 29967 3 +25740 29929 7 +54718 47182 11 +45239 7956 11 +5775 7916 10 +53058 24322 2 +19389 6161 6 +13552 27037 11 +45603 16941 9 +24798 15397 5 +9624 39563 7 +27822 19039 7 +28191 180 3 +13709 22486 8 +51554 11604 2 +20378 25685 6 +25288 35367 5 +18941 30215 8 +19962 54620 3 +45197 20087 7 +53491 32537 7 +28704 42160 12 +48059 25868 7 +32970 31978 11 +4160 40393 6 +46882 45626 10 +54092 42669 8 +19154 21964 3 +36911 12385 5 +730 46926 7 +19893 49751 15 +6487 2352 7 +30165 14663 14 +13705 24243 11 +17193 25666 9 +22243 1333 18 +53388 54749 10 +22556 12726 9 +43573 30533 5 +8645 25459 10 +13690 24752 4 +7243 14468 2 +35943 13126 0 +5354 36698 9 +45233 22879 6 +9813 46281 10 +19947 56000 7 +54175 45544 4 +52349 57326 8 +59991 46682 8 +36170 46004 6 +44916 38746 5 +24616 58137 8 +56068 18403 14 +8628 33385 7 +19951 46850 9 +19692 39692 17 +6302 9309 8 +54998 7120 8 +35498 46899 6 +4180 24536 5 +34246 4690 9 +55563 9596 15 +49785 57705 5 +57151 37042 5 +30383 2020 14 +17926 51429 11 +32567 58383 12 +42530 50276 12 +43454 40421 5 +15560 8654 18 +58378 58337 9 +49772 251 7 +37215 12783 10 +44023 12615 4 +53534 8236 9 +41351 5168 7 +44647 27702 11 +51819 40891 14 +14025 27384 5 +4226 59837 15 +37000 51085 7 +41511 20430 7 +29567 31778 1 +39522 39967 10 +49871 36387 17 +50538 27610 5 +10876 53588 9 +14556 29430 8 +57507 18401 9 +28720 18875 10 +6454 43583 5 +7983 44064 9 +34373 18566 3 +49443 53147 1 +38117 14814 14 +7308 2780 13 +34787 52125 14 +8494 33047 5 +18656 58558 4 +52695 21063 9 +34037 20885 5 +50735 36748 10 +27006 40886 8 +35066 23874 8 +8730 16360 17 +44129 45245 17 +55075 9822 5 +5642 46915 4 +53638 31759 8 +51152 39981 9 +46538 27493 14 +49605 288 9 +58504 54497 11 +36016 57456 8 +33290 49448 8 +48663 9312 2 +14903 39318 9 +53080 16248 8 +58201 48465 11 +50050 13693 6 +42562 16686 3 +52128 56959 12 +1200 10818 9 +50012 57550 10 +39301 52771 6 +13061 1019 9 +48743 42580 16 +36565 31758 16 +59665 20132 8 +389 32985 11 +46495 47555 8 +10709 25103 7 +23538 38590 4 +12342 45990 12 +8107 50330 10 +31802 31807 4 +13158 17646 10 +38440 42791 3 +17514 58070 13 +26578 46241 4 +42220 17341 6 +57960 33752 2 +993 47748 9 +29286 18945 17 +51205 45141 13 +7719 11784 8 +44539 44455 9 +32922 3752 11 +37962 1204 14 +30536 51653 6 +32642 51242 11 +43090 28234 5 +33172 23489 6 +36119 28992 3 +12953 47397 7 +48662 56091 11 +44138 38448 9 +56975 51877 9 +41601 56841 14 +41341 37676 6 +39179 18651 11 +26515 37821 11 +14391 21437 14 +8292 33721 11 +54060 52380 9 +7326 36431 2 +14639 57195 8 +35974 6626 4 +19633 6086 8 +55115 38805 8 +45856 16960 7 +25370 35114 12 +36978 40914 5 +13262 38465 6 +24209 9966 1 +32050 57185 8 +39268 30454 7 +2827 12055 8 +39715 59151 3 +48232 49989 1 +10742 39735 5 +23774 22312 15 +44349 59993 8 +25386 27843 9 +3157 34975 13 +18585 49281 2 +54725 54169 10 +41321 35387 14 +31115 22560 10 +48648 31579 10 +7895 39689 7 +14586 50098 13 +38976 2497 2 +54554 8379 9 +28293 37299 10 +52495 44704 2 +22948 29038 14 +16303 35743 14 +56902 34904 1 +56097 46295 11 +22634 59239 3 +19668 11862 16 +36849 56793 4 +58425 12324 7 +36963 25846 13 +52284 53856 9 +27364 54560 7 +36204 39678 7 +41483 39725 14 +50502 17329 11 +26684 48189 8 +47529 21166 3 +58099 43848 15 +24398 51976 11 +16175 11975 10 +37765 39614 12 +42092 32514 2 +3765 3466 5 +41983 22642 10 +22567 15418 5 +7500 32728 9 +37480 44063 7 +48301 13426 10 +6907 28552 12 +52339 24828 7 +27392 30744 13 +37572 49563 11 +7618 22239 10 +15957 50498 7 +18895 25266 6 +6047 7789 7 +51967 49350 10 +45660 21526 8 +12530 49198 5 +49620 17681 16 +34277 52845 15 +23343 38768 15 +20215 5041 3 +33941 53607 6 +50411 31183 7 +44247 16031 2 +20509 44179 13 +51014 35563 11 +59335 40329 4 +1720 48254 10 +56642 14588 7 +43799 8705 6 +37674 14985 10 +44423 12049 10 +26507 32428 7 +50722 59069 6 +54113 9055 6 +22836 86 10 +5297 15345 3 +40332 56469 6 +54403 38795 15 +29528 7636 11 +47737 19870 10 +50432 4977 11 +31235 56466 13 +52790 27212 6 +36067 29635 8 +13640 18040 6 +44056 29602 11 +22167 21668 9 +27814 48621 7 +14445 59241 0 +25128 13857 9 +45904 28103 7 +57306 56291 4 +45144 58282 16 +57070 25422 14 +51493 58023 5 +43365 41195 10 +37805 22151 13 +51705 33945 8 +27328 28275 2 +49599 51547 11 +52612 3152 16 +30592 35699 2 +36262 21716 4 +34712 11066 5 +30278 38375 9 +54430 11415 12 +26543 50146 14 +4613 39038 11 +36136 43865 10 +2196 20463 6 +9877 52634 2 +8375 53115 15 +50215 53614 14 +11442 14610 15 +50385 33991 4 +36101 17540 7 +26315 9032 0 +32896 51888 8 +57046 52195 6 +43657 36779 13 +22171 38279 16 +40047 17402 10 +29001 56021 18 +22177 53064 10 +11802 55725 11 +8736 45061 9 +16408 46180 7 +29338 3796 16 +34768 23091 10 +13199 1224 13 +46203 5307 7 +3059 40352 8 +15997 18326 12 +26346 7709 9 +11684 54359 10 +5991 5928 7 +12581 14065 7 +28013 39842 11 +36578 33465 6 +36926 42319 9 +28205 8280 14 +14090 33199 14 +47127 36220 13 +54569 10605 11 +49866 18376 13 +51331 32213 4 +30977 59410 11 +27992 55172 8 +51295 53172 11 +24891 22378 9 +29922 36539 9 +14191 40415 15 +24524 2279 8 +54199 24796 5 +12661 14739 7 +55529 7100 8 +29594 33508 3 +18438 7411 5 +1551 40388 5 +19703 162 16 +9051 32316 18 +1700 35145 9 +58926 5299 9 +8380 36291 10 +28212 41475 7 +49535 24113 4 +41074 56827 6 +44284 13577 7 +44069 42070 11 +13696 7889 13 +26546 51300 11 +58414 26230 5 +21241 39904 9 +71 6585 10 +49302 59875 10 +47663 19927 12 +25079 37825 9 +50399 48810 11 +56165 59625 4 +5754 20701 13 +33760 46911 6 +48094 41636 11 +9052 12419 15 +20222 12891 8 +45838 7486 4 +19972 27039 16 +34222 53627 2 +55321 8487 12 +28388 6495 10 +57598 47827 9 +28784 12492 7 +27351 5624 6 +15020 14721 7 +39753 23368 17 +29516 28738 8 +43006 32227 4 +10276 33847 0 +21043 43184 9 +42141 7088 7 +59117 36800 11 +25187 36572 6 +21319 37699 5 +25568 38916 15 +19787 8829 6 +44036 27506 2 +54697 39199 12 +6152 34524 16 +7771 55767 4 +51652 30144 13 +23853 58653 14 +20385 25561 14 +12519 44765 9 +59547 4266 7 +9592 40120 5 +55043 46352 12 +13192 28598 16 +55310 25778 6 +24190 6485 15 +38829 28680 15 +3303 57012 15 +2178 28330 15 +22804 29965 17 +11936 49935 10 +26849 4884 4 +28692 9819 2 +14751 57751 15 +45452 16142 7 +22342 42657 8 +319 30008 16 +42483 32893 7 +48210 59960 5 +13835 35098 5 +9122 36858 11 +29887 23469 7 +4365 46464 8 +43636 8339 9 +35182 49040 14 +23430 4185 9 +18258 36766 7 +28483 54250 11 +19886 35620 2 +59311 13601 8 +11392 41719 8 +13776 42067 14 +34708 2267 9 +49261 32089 18 +45848 53752 12 +56261 9242 3 +25898 25748 8 +12401 29032 13 +10620 24122 3 +51599 34392 2 +17874 53412 9 +14830 1752 13 +24566 43771 16 +26976 50394 13 +34366 25250 11 +20415 44680 5 +5811 36540 11 +14862 3384 10 +17904 6997 8 +27022 38151 11 +13634 35134 6 +22314 36258 14 +31808 4792 3 +16496 14513 7 +37099 36647 4 +12189 36378 12 +5988 13799 2 +28153 13217 1 +48146 17371 10 +50839 49703 9 +47393 1892 8 +14292 56183 10 +28281 58582 9 +10550 43203 14 +17129 55515 7 +51131 11369 6 +8012 27682 9 +28603 47075 3 +44307 673 7 +17996 29474 6 +53243 4624 0 +42259 26452 14 +2691 31214 12 +58782 23712 3 +22438 49577 14 +21380 6336 4 +6399 26521 11 +5625 21601 9 +42223 25598 7 +7876 10697 7 +30845 26638 11 +17965 28070 7 +52706 59185 9 +59934 49538 1 +9190 16911 7 +23415 11927 12 +47334 2772 8 +34031 4878 12 +22315 35981 11 +39918 54575 11 +16210 3544 14 +44280 40685 12 +2768 1731 14 +16291 25528 12 +10172 42201 7 +14834 50757 5 +35664 46615 13 +55676 47595 11 +15534 36946 6 +56195 47113 8 +48393 54112 8 +26380 8425 12 +31776 55231 7 +14636 59568 11 +31859 7590 9 +45778 56598 6 +43509 21356 12 +43467 2082 4 +391 2378 10 +20983 53984 2 +28147 48406 16 +11635 4745 14 +43343 8467 18 +36838 7623 15 +55864 21834 10 +33719 49159 15 +31369 33340 5 +39902 51541 10 +41790 57286 13 +58304 55774 10 +40473 20714 9 +3123 26420 7 +5079 10360 8 +41568 11824 8 +30936 6902 14 +44738 55485 8 +58953 14770 8 +17323 33694 12 +45086 16816 18 +48167 27542 7 +26626 30145 3 +54991 43448 7 +59446 50647 6 +54705 29730 15 +19339 28408 1 +59809 47638 5 +19487 38203 9 +42113 54371 1 +28833 40166 12 +3472 3135 3 +31395 15544 6 +52765 46654 6 +30628 26513 10 +27723 36736 12 +36982 31102 6 +8259 33369 4 +32080 28963 9 +16501 23137 5 +1684 15956 17 +54890 25167 7 +34605 35686 7 +57781 44892 12 +44800 6939 4 +41176 17348 15 +2614 9456 6 +49774 49054 3 +51139 36480 11 +7139 35396 3 +32066 18427 16 +20166 56415 3 +35909 32816 11 +48336 6919 5 +47837 41631 7 +41459 38182 15 +16110 55849 10 +10226 13083 13 +7843 59569 7 +8848 21545 12 +17890 26278 14 +23459 49978 11 +32622 33937 9 +50677 4904 12 +23179 58620 9 +7674 48664 9 +18817 40157 8 +23609 47583 8 +54790 59736 8 +8732 48615 13 +40687 31312 17 +59746 35781 14 +33025 41169 3 +4788 54540 16 +40418 58905 8 +29879 17691 6 +22950 18931 15 +35408 33181 8 +47232 12497 7 +18600 16367 6 +19887 2573 15 +32193 11919 10 +1519 11029 8 +44626 42777 10 +41347 42083 11 +55847 56853 13 +26178 12152 3 +39569 28131 9 +42543 53295 9 +53074 36987 12 +24362 51745 13 +31556 57475 3 +51047 1015 4 +34383 51579 5 +50878 59295 16 +15238 17936 14 +47109 20255 11 +54337 24052 12 +22978 4769 9 +44479 59621 6 +10167 28456 0 +47300 13679 12 +47270 53936 2 +50770 37903 6 +7917 9772 7 +11800 29184 13 +8758 57883 13 +15483 36626 7 +3650 6505 8 +21859 10826 12 +44569 48162 9 +27915 54327 10 +20959 57003 2 +30375 54073 6 +55181 36451 5 +20797 40364 11 +4261 13105 11 +42971 46252 11 +11308 1087 6 +39966 48474 7 +46892 20783 2 +45823 24172 7 +39275 37057 14 +17299 13974 5 +29598 6009 5 +40913 56127 9 +33475 10557 11 +53087 16327 3 +860 20150 3 +37262 47019 5 +19552 52927 2 +33027 18054 6 +14710 352 15 +17916 11826 6 +37190 6289 3 +28568 44787 8 +20739 42660 7 +12985 19729 11 +47305 30311 9 +6616 35351 14 +2266 30293 5 +49891 8999 16 +2918 30062 4 +24487 3767 7 +37140 31820 11 +38757 29251 8 +43925 19904 13 +55753 7613 7 +41850 26821 5 +42549 51584 16 +17886 31985 13 +12928 26085 16 +57799 35469 9 +42709 25417 12 +32335 96 8 +31351 9476 9 +13768 26848 15 +33240 45345 12 +47713 34573 8 +38505 51825 14 +30711 56563 5 +26375 5226 12 +5767 26731 0 +40578 32863 4 +49215 25990 12 +24580 57903 2 +57879 22289 11 +35765 53006 11 +13759 42542 15 +49465 11068 17 +22118 7048 16 +42813 5173 11 +92 35273 13 +1042 8404 16 +11556 31582 6 +52418 20557 8 +53185 22283 9 +50449 55574 3 +19916 40216 13 +16660 48977 2 +33177 46790 8 +55376 22696 7 +49037 1610 10 +34774 33482 7 +33139 42600 9 +6498 21058 13 +28611 5782 8 +23320 48182 12 +56519 24874 10 +21342 28068 3 +15509 38092 10 +19727 47998 5 +14981 6684 4 +37128 12126 11 +31825 28424 12 +22907 35579 11 +20591 41838 4 +13732 16169 12 +29670 53698 12 +34976 53274 9 +39935 41137 0 +29723 10243 12 +55688 58586 4 +16238 3569 2 +3449 3927 15 +43023 13554 15 +33933 59272 0 +15175 51409 8 +59714 35003 6 +24408 6791 5 +59341 40873 16 +23369 59756 10 +29653 16898 9 +59951 7941 5 +32287 31971 6 +34041 47106 11 +46282 56769 16 +26900 14207 10 +58553 15626 8 +56774 42340 15 +17410 53018 10 +24905 29111 15 +50642 38526 3 +11241 10871 13 +45945 23696 7 +19850 46870 10 +37924 6025 12 +46493 27394 7 +42232 30211 7 +30843 10020 9 +40558 22754 15 +29743 30500 18 +29144 45089 10 +956 17238 9 +6061 5069 13 +58604 57709 9 +11360 16835 2 +49797 27224 10 +278 57795 5 +39899 10127 4 +48045 35301 5 +55723 31325 11 +36374 22655 11 +4775 32318 3 +40445 44074 11 +42556 52774 7 +35354 47945 5 +39730 287 9 +30396 35526 13 +23407 37119 10 +43155 37520 12 +25493 34353 16 +16244 29006 5 +36535 24535 2 +29917 22242 10 +38636 34788 5 +8330 49096 10 +59706 43760 4 +15157 1831 2 +57819 3424 7 +13364 58562 15 +24901 52400 6 +46694 8333 6 +646 10109 11 +51753 58536 6 +6911 4042 14 +55689 38123 7 +53417 2079 8 +13610 20596 7 +719 56115 11 +39327 14881 5 +16488 4211 16 +33651 5608 7 +55834 43394 11 +27055 14712 4 +46100 27390 15 +11525 23848 15 +24625 17618 2 +15268 6096 14 +57708 37666 10 +59242 37485 7 +27340 40010 11 +13415 6157 8 +49336 21394 1 +36554 56201 16 +21290 48125 13 +14076 6935 7 +34669 7909 16 +32366 26919 17 +51698 3017 4 +57811 14243 7 +2747 57804 4 +24157 46670 12 +6191 29579 9 +53393 6638 15 +55358 26152 2 +30831 46027 5 +27278 28105 8 +14085 33003 5 +18906 27406 8 +14061 26727 6 +52541 49158 8 +45634 23530 7 +33147 20230 13 +30736 38972 11 +4590 649 11 +919 55795 6 +59238 15643 16 +39851 40595 11 +18940 54415 5 +30381 39414 5 +33302 2705 3 +10716 27940 13 +59097 30886 12 +41620 37210 2 +723 17030 8 +31189 48953 11 +39130 33813 14 +18557 15184 11 +43176 29545 10 +19959 20935 10 +8523 59378 12 +33735 26730 15 +4572 11163 8 +57523 12549 5 +43204 10827 6 +11348 27200 6 +18490 44339 5 +4565 43823 8 +30045 13005 6 +50382 33963 7 +2591 30251 10 +23795 16574 6 +34601 37274 8 +46292 18522 3 +8357 8659 14 +37683 27259 9 +8225 31993 7 +7110 20481 9 +28422 40761 14 +24947 43780 14 +4299 28724 6 +37966 40865 9 +7967 57638 14 +53271 13619 13 +6232 15381 10 +56158 22825 3 +11754 57852 5 +17252 19125 12 +14189 59938 15 +28537 6972 2 +54308 8437 15 +30558 19095 13 +58863 17533 13 +57400 58935 2 +6055 16944 3 +29161 42506 10 +18623 21173 9 +3621 4312 14 +4106 20669 7 +6295 23315 9 +21264 12272 14 +5679 22478 4 +51655 7084 4 +2285 38687 11 +7087 9625 11 +14106 17562 14 +32955 38443 16 +57759 56635 9 +21531 39812 7 +13597 53820 12 +38812 15048 5 +9740 56017 14 +44491 18881 0 +36616 20199 11 +58029 53706 13 +37216 39182 9 +44913 10671 14 +1522 18862 12 +18953 30709 1 +42998 38335 8 +8224 29560 6 +48986 19089 7 +38137 35902 11 +14429 37521 7 +33390 24966 1 +27101 12777 14 +46235 34307 7 +2717 53476 5 +42907 22714 7 +39707 16608 2 +34967 34724 5 +29033 43789 3 +27085 27091 9 +33232 28486 9 +50993 41068 5 +55442 52252 8 +1854 36479 11 +44352 39374 9 +3610 3130 1 +46109 53809 14 +29108 45119 18 +37895 29544 7 +47622 34952 10 +1306 18335 15 +23011 11468 9 +58793 21746 7 +16747 52112 10 +558 50494 8 +39814 37045 4 +5923 361 6 +52840 47689 5 +45320 55561 7 +41199 7541 18 +38546 1882 14 +35508 26769 5 +682 14201 11 +33207 18884 14 +31028 45393 10 +35108 57473 2 +6890 222 8 +2287 49066 11 +14396 7101 16 +23090 139 13 +31630 44458 16 +53806 27290 8 +4372 12566 12 +55780 32398 12 +38271 29264 13 +55239 49377 12 +57369 51643 12 +18406 42720 8 +44965 280 16 +55266 7548 7 +55024 3519 13 +23984 42049 6 +44648 11511 4 +35420 24733 8 +49070 53720 4 +32312 40568 7 +56731 2220 12 +670 11266 10 +16013 46092 5 +59100 46297 10 +58719 51473 6 +31720 5762 5 +42628 34847 9 +33100 42378 13 +22350 47603 9 +1853 12276 8 +55483 34622 9 +40977 27226 18 +40801 23657 3 +27191 13229 14 +20639 7010 15 +53932 28669 4 +13609 16891 6 +8633 49766 9 +39032 25137 2 +6605 13181 5 +30635 37179 9 +7957 8373 7 +54339 26984 4 +46497 12461 11 +20221 29013 9 +44311 33370 8 +44152 34552 8 +8626 7298 15 +35861 38825 8 +48557 9505 3 +36815 44251 7 +53921 10122 4 +42707 5283 10 +25023 25756 12 +41500 25854 8 +27453 26217 11 +15203 10718 9 +53612 23593 13 +12682 32551 9 +46444 45480 12 +33782 42176 7 +57123 11401 10 +3320 12501 5 +56173 28785 12 +36993 59475 11 +22631 18456 8 +41663 51585 8 +9832 23120 8 +44459 50741 10 +5220 19343 10 +4605 19901 7 +52256 5529 9 +36146 2644 6 +13298 22852 5 +22173 25615 8 +11524 39311 8 +9894 48350 2 +45550 49043 10 +33421 21871 10 +44797 18263 2 +28713 16720 10 +7535 28104 11 +11418 47983 6 +419 21240 16 +19542 21189 8 +310 20920 10 +27075 6835 10 +13273 10539 10 +3999 31224 10 +50415 49557 7 +41533 40351 11 +27855 34074 2 +44335 11386 6 +2839 44495 2 +8176 38417 3 +51007 51971 15 +13821 7979 13 +27261 3372 10 +21347 6850 15 +49298 58790 14 +4079 7103 11 +10682 33600 8 +5407 33337 15 +31743 28921 11 +34414 7904 17 +25382 34384 7 +19725 53474 11 +41006 54002 9 +37230 47346 6 +18016 41534 1 +14740 48720 5 +1276 52623 9 +57407 44778 9 +52164 31814 7 +5194 47281 5 +31436 55552 8 +26641 59523 7 +27401 54006 8 +37616 25349 6 +54352 2581 8 +31797 28235 7 +45873 18793 16 +2505 41043 5 +59362 53288 13 +22800 40155 4 +351 24717 1 +10132 41915 13 +32120 30116 15 +27899 30682 7 +32545 33928 7 +51937 27135 5 +44084 47496 12 +54473 19250 7 +22471 45737 17 +37769 11283 11 +24651 59582 7 +42926 8655 9 +24490 23064 13 +37459 57292 15 +2065 34739 11 +6322 4740 11 +12906 7940 11 +14315 11302 14 +55813 48941 15 +28467 28400 12 +9566 25621 8 +54556 23359 6 +8859 40071 9 +35751 6313 14 +54969 23786 6 +47893 31719 10 +6538 33677 0 +56318 1223 8 +29926 13163 4 +20115 35136 13 +8687 24452 3 +46636 8281 11 +39587 43290 10 +30672 23128 16 +40515 273 11 +24291 35571 9 +50073 59533 13 +49739 32386 5 +24860 9881 9 +13418 10193 13 +20996 25958 12 +28329 2385 18 +31132 33261 11 +20160 24896 2 +34632 25051 1 +53763 9538 14 +147 29869 6 +58182 32468 10 +620 44764 8 +16040 11036 8 +25210 15478 9 +13456 5152 12 +14741 28797 13 +46242 7922 12 +28094 12850 8 +19617 53254 10 +17636 40817 6 +27696 56095 9 +45545 59395 11 +23797 29954 9 +16456 20361 3 +20077 9240 3 +19932 18507 8 +44484 14218 13 +46558 40799 15 +16809 47279 8 +144 7900 13 +41272 30895 9 +37852 48379 9 +4001 20554 12 +29876 30964 10 +54603 21653 7 +21969 42448 9 +13957 3208 5 +43471 50337 10 +2235 37611 13 +46581 17522 11 +28778 27701 6 +46189 31642 7 +53141 9175 5 +27097 56915 11 +43691 49334 10 +10439 15917 2 +27678 17841 10 +32860 26408 9 +25793 56871 8 +1571 7160 1 +11860 45350 2 +24955 6432 13 +29078 50918 10 +55434 27926 14 +42674 3777 0 +7577 41522 11 +42557 32067 18 +49360 31819 10 +45872 50646 6 +37626 26933 10 +22616 8057 12 +58196 59071 2 +51113 17772 14 +46051 32307 10 +31947 56946 6 +18315 48150 15 +37535 48789 15 +54805 42573 9 +6715 33005 8 +48173 40968 2 +14401 27736 5 +10113 54191 5 +23507 33491 10 +4220 57787 5 +37639 24591 8 +2165 31446 8 +43808 54392 1 +23194 22208 2 +9462 13884 6 +22835 53394 4 +22134 19968 8 +42162 41489 8 +47579 46845 6 +5569 9954 14 +1950 25000 12 +20180 2718 4 +53219 56828 7 +8391 41044 15 +54121 9145 4 +37447 13059 6 +31405 37864 14 +37297 26802 8 +48879 16336 9 +37296 17473 3 +43561 6440 3 +55857 21399 10 +15582 31619 6 +57004 17474 7 +21123 31718 8 +16739 38884 8 +38295 461 9 +25619 48337 3 +3202 23842 9 +21160 21147 9 +5904 37058 6 +2704 13627 9 +51109 11782 11 +18370 40191 8 +12768 27628 9 +39791 14001 5 +42964 51110 6 +18025 37175 13 +52417 55417 5 +17450 43542 11 +41036 29757 7 +32586 11896 5 +40029 49837 8 +18502 22091 13 +37538 15565 15 +14911 45924 12 +33510 2236 7 +15668 46916 5 +44054 29352 5 +18669 25048 13 +6826 11366 12 +5865 42248 12 +15756 30072 9 +23627 53300 4 +32415 7643 12 +31684 28230 7 +10944 27655 15 +17737 18105 15 +54173 42576 9 +6066 36384 12 +36169 58842 9 +50817 38695 12 +46879 42911 8 +51826 27444 2 +51127 17748 1 +5350 14397 13 +9826 52544 8 +47914 31492 2 +45482 46863 7 +2799 12542 4 +19018 39006 3 +41981 3825 12 +52886 45469 14 +32232 638 5 +26567 15755 13 +28157 36910 9 +858 45841 17 +1367 22926 1 +43869 56076 10 +2541 33716 5 +833 48010 8 +32671 9079 13 +11175 58806 10 +38779 43872 2 +23633 56638 12 +11394 47025 8 +31335 39567 11 +13275 15684 3 +40588 3727 11 +38380 26431 2 +38880 53794 14 +4415 23211 5 +36287 44029 3 +17755 4975 9 +27362 1781 9 +20142 42140 9 +586 269 8 +51283 51778 11 +41614 23582 8 +38603 13708 9 +52512 4110 8 +45163 35437 11 +9674 56316 5 +29973 25055 12 +42207 34265 13 +31389 247 9 +3444 36265 3 +43537 23088 3 +37342 57966 10 +50809 49720 3 +45435 14096 12 +59684 1668 5 +36964 23890 15 +37645 18543 4 +4283 44986 5 +52662 52345 10 +11405 34509 6 +10631 13065 12 +44543 37185 11 +26866 36021 6 +37076 27314 9 +22928 42099 5 +28002 42539 2 +59731 17378 6 +23723 49945 18 +10543 17209 6 +19882 11261 10 +3246 5563 7 +40233 34375 16 +29795 46210 7 +38839 246 2 +26240 45430 7 +43329 23976 17 +26286 16195 9 +10619 31586 14 +18152 6141 14 +13050 41961 10 +17714 48560 15 +32624 36791 10 +38769 24461 2 +30258 46710 5 +46737 26398 12 +29299 58597 9 +52122 1954 7 +27788 34596 4 +44578 1268 0 +12021 37686 5 +49851 45193 8 +35681 52075 1 +9823 19844 7 +20869 59136 8 +41223 47219 11 +41657 57534 12 +45352 45392 16 +40225 38766 4 +46794 53900 14 +39065 13885 13 +51772 38684 2 +13206 37641 9 +55090 17069 14 +30053 11935 6 +11570 3899 10 +51733 548 2 +44981 54043 8 +44948 641 18 +35262 31756 5 +38173 6114 6 +53923 17819 8 +20135 12127 10 +33902 23665 2 +28350 2807 2 +13961 31627 4 +10212 489 4 +1607 58985 2 +7225 22210 9 +14865 4232 10 +14392 53084 5 +21729 24384 11 +4927 1702 13 +59852 31643 11 +13778 53740 6 +55040 29405 2 +30598 26972 14 +18144 23738 15 +12358 18530 11 +6724 35639 4 +29098 34625 2 +827 12069 18 +562 16632 13 +13688 44407 13 +7207 5513 6 +13804 4214 8 +17173 1487 6 +45467 25818 0 +4047 30068 1 +45533 13121 9 +42392 57293 10 +17357 11184 8 +20738 43756 7 +17783 56647 14 +28557 10440 7 +38449 14766 9 +20224 35001 3 +38933 25433 6 +11783 18349 10 +28057 51276 11 +27735 32472 1 +14916 27382 9 +28142 56048 7 +32884 63 8 +19535 16590 6 +41537 48421 3 +49532 36544 13 +7660 43580 11 +32064 4671 18 +35175 31649 5 +12571 3598 16 +54909 58330 13 +13815 1978 12 +38355 6658 3 +1578 22915 9 +5032 57989 10 +28957 11559 3 +37744 29980 9 +38762 32595 7 +39506 51013 14 +36880 9675 9 +1058 25259 7 +52872 37220 1 +45242 30632 15 +59228 31942 12 +10299 5559 15 +3893 52555 12 +49795 47780 4 +35534 58041 10 +2333 13437 9 +53230 36900 11 +48214 36831 6 +23704 57389 11 +16301 41621 16 +52933 47633 15 +11745 4503 18 +57105 46718 3 +50763 41602 12 +56711 40537 6 +41055 57947 6 +27829 31835 7 +11537 32886 2 +55946 16811 11 +57512 58614 14 +39691 53846 5 +11606 53906 9 +33610 22546 16 +46370 10307 9 +27100 59534 9 +30139 24464 13 +5336 5834 2 +46162 54806 4 +25704 58123 2 +16190 56345 9 +21073 43200 5 +50304 8846 15 +26829 54597 7 +53899 45042 1 +11953 2008 5 +47509 14116 15 +42253 3146 15 +55275 35652 2 +34387 5693 9 +25790 31520 9 +13156 1078 2 +22858 52966 10 +13516 5859 4 +12509 36235 10 +4582 10820 17 +52736 5877 13 +33347 39574 9 +57773 25236 16 +31943 58279 4 +32843 6346 5 +35300 54331 13 +22726 36837 6 +28541 4602 8 +1787 8946 4 +50595 6349 8 +2543 43291 11 +27868 42328 9 +53047 56667 14 +23208 5527 5 +56295 38742 16 +11166 3012 7 +50943 41051 7 +22114 53929 9 +48438 2642 0 +49962 14138 9 +25441 5825 8 +36350 6834 5 +46593 36951 9 +36664 57169 5 +44120 24978 1 +34618 43877 9 +53994 30291 2 +34275 55415 5 +57099 48500 10 +8315 54184 14 +56993 35653 4 +52629 56202 5 +32634 36874 2 +19204 38473 8 +56228 7574 4 +2759 46408 9 +44400 19106 14 +11972 23960 7 +22731 50889 6 +49752 31268 15 +24428 39386 10 +4107 39282 6 +52254 44121 1 +48574 1973 4 +59864 36656 9 +53036 14070 9 +34737 10675 1 +7412 40289 5 +52305 36808 9 +34567 41610 13 +43826 52768 8 +51000 46039 7 +29648 30312 11 +27522 3016 7 +15318 32997 4 +8810 53105 4 +34512 52936 4 +27862 58053 7 +56479 5131 5 +15407 24427 2 +3069 42348 10 +40867 6796 10 +29625 48917 10 +25971 11490 10 +40005 39895 12 +53782 25387 16 +30303 55112 7 +44566 4017 12 +3294 17771 7 +53825 39054 2 +12606 50229 8 +37804 59444 8 +10341 14168 14 +7867 36278 12 +25744 58893 9 +11830 6006 17 +54032 36411 13 +27579 6986 15 +8187 45438 1 +19670 57616 13 +40197 14877 9 +21387 53503 7 +26044 421 6 +28374 18516 4 +34416 8842 4 +38234 14080 8 +47912 19276 8 +10054 57924 8 +9502 55203 9 +34906 53042 6 +51050 54952 8 +23527 30285 8 +30731 16995 5 +45501 44444 8 +117 39261 9 +48262 41667 6 +52749 50998 10 +25671 30329 5 +49048 24889 13 +4490 3638 8 +20119 39265 7 +34600 53024 12 +30478 10374 17 +31402 13845 11 +49098 487 11 +42005 25335 5 +38628 57221 5 +23773 39128 4 +34360 27991 12 +52906 2532 1 +35538 17711 2 +46195 26875 7 +3035 37106 8 +19543 38499 1 +19805 42258 15 +23132 19591 9 +20521 7267 8 +25206 31540 8 +35854 40915 9 +45128 42138 2 +50238 42807 11 +38391 24861 8 +10309 11086 9 +43372 20479 13 +5873 19760 5 +46430 55316 2 +18191 58124 13 +37869 22451 4 +31239 10533 14 +7935 26147 11 +50170 15091 16 +42486 45406 3 +23074 52923 16 +22863 48322 3 +19752 21761 12 +24268 21960 0 +15844 59152 11 +27047 56810 4 +1077 10206 6 +9199 55452 8 +8531 39490 7 +47120 32145 16 +23778 37894 8 +33625 48812 14 +41078 44323 15 +57315 33225 11 +1251 34779 15 +33691 53026 4 +3989 28608 7 +8121 34209 14 +49409 36631 4 +35150 53173 7 +53624 30193 10 +9507 13540 8 +30277 3552 12 +19516 15475 3 +3881 6803 8 +25377 58692 12 +12796 51618 0 +127 37269 4 +35427 35385 7 +48970 41050 10 +58823 172 13 +18323 37890 5 +1672 14598 11 +22164 36241 1 +44149 23599 6 +6479 51308 9 +46847 57373 8 +27504 13036 11 +49380 28247 9 +54599 54015 14 +59643 24974 4 +16243 37320 5 +18807 7714 13 +9119 23438 14 +29972 9396 5 +41370 24773 10 +52770 58508 17 +1175 3932 8 +69 55104 3 +52439 17761 5 +1024 22513 12 +21586 38280 5 +25543 27671 10 +24136 32303 6 +32478 9390 17 +53965 45481 12 +15777 30773 4 +19806 508 10 +14323 22207 1 +47366 54441 12 +6483 53310 5 +48767 5836 11 +28210 44601 9 +29444 53076 6 +34347 4517 15 +47463 43683 10 +41142 4743 13 +23963 9774 13 +21979 8746 9 +56123 21464 8 +23316 38772 11 +9741 33710 11 +22531 43233 9 +22346 41186 12 +37183 12475 9 +12536 37011 4 +58256 25692 8 +13645 19277 8 +29889 20617 7 +13230 37154 1 +29024 16273 10 +25844 21310 12 +28970 33293 1 +51545 50353 9 +24652 4646 9 +53365 49422 11 +52187 45369 10 +48738 1405 3 +27389 20486 9 +36107 58037 4 +30005 11001 13 +18195 39693 16 +20969 56357 15 +43330 44538 10 +19485 5503 9 +15288 2701 7 +33330 56398 3 +14251 49646 9 +38254 11210 13 +24234 16736 14 +52919 8709 7 +44268 19301 5 +6868 22600 11 +21003 53153 12 +25618 5222 8 +45969 16437 16 +49706 25300 8 +54203 8182 12 +35795 58523 6 +34570 23736 9 +50578 10897 10 +11482 33336 17 +28314 51044 7 +767 27821 4 +13636 13378 9 +44460 35777 7 +50911 32169 3 +8965 47791 7 +45928 38070 6 +43624 4149 1 +3843 19255 16 +53589 3726 13 +16109 19478 6 +24099 52916 4 +58685 53309 6 +50505 42247 14 +44249 30274 4 +34251 17232 1 +14590 32164 11 +36869 46087 9 +37810 31376 17 +33500 43076 6 +46151 37494 14 +51080 24079 13 +11281 40987 1 +46170 49638 9 +41490 28585 6 +28671 28396 9 +44501 50486 9 +30155 58001 10 +14176 34770 12 +58987 55727 4 +58208 17107 13 +57927 11940 15 +9898 50836 11 +3032 41832 10 +7785 31491 12 +12507 39029 5 +52105 13395 9 +4575 7324 8 +15240 37860 4 +58961 28190 13 +58650 53406 10 +5365 15047 12 +16737 14464 6 +7850 43510 6 +53428 15501 13 +26326 27659 2 +32791 37524 9 +37361 5565 10 +12431 59726 14 +27157 29396 8 +54545 17899 11 +52628 491 1 +40493 26909 7 +3673 50952 1 +16296 56270 12 +25188 3891 12 +51043 36063 13 +5990 51640 9 +1021 18266 4 +15640 46518 9 +44967 42237 6 +8539 30252 10 +44067 38429 7 +44331 36137 14 +52405 51557 14 +31205 57860 14 +59123 45270 16 +23602 8140 14 +6717 24518 16 +43186 7358 0 +31229 39820 6 +54555 20791 12 +53440 59733 15 +51532 3061 11 +53339 2855 2 +21136 9488 4 +54782 52987 5 +45136 14530 2 +29554 23505 7 +27995 34659 11 +28379 9350 8 +5604 50014 4 +40188 15510 2 +51074 28817 13 +49676 17384 13 +6766 50423 3 +11871 52587 13 +30577 53595 9 +13017 14333 11 +12016 27544 5 +46075 14922 12 +25134 39831 14 +43091 38890 15 +2761 36726 12 +48187 3605 1 +17264 3454 11 +4379 47057 12 +40576 27496 10 +53827 50963 13 +52435 54063 8 +41332 41845 5 +24449 31725 9 +55886 36628 5 +45289 32264 5 +57237 33686 10 +12265 24981 11 +43684 13142 4 +6576 42712 11 +40458 18306 13 +18270 1067 9 +55250 13169 15 +2632 38072 7 +46800 47230 11 +10833 54919 14 +3442 7393 8 +4827 30050 14 +52215 50038 13 +49020 44109 4 +23255 17824 10 +13478 16782 9 +12734 16905 1 +58891 31114 5 +40919 4945 11 +38251 12706 9 +36598 58832 11 +3118 18675 8 +41110 46028 5 +16607 33587 3 +22761 22113 3 +17334 45235 0 +20148 4898 9 +48503 37932 17 +25083 26949 18 +33934 10215 13 +44049 49899 3 +23944 228 12 +15700 57088 5 +35783 36792 14 +44345 27812 10 +16673 43393 12 +15606 12856 8 +21403 37739 0 +44643 3960 13 +41039 55013 15 +44625 44656 5 +12259 34776 6 +23168 11750 14 +43453 3836 11 +37621 39633 15 +36437 37273 9 +53077 11214 1 +17679 36366 13 +21540 25632 11 +53286 27950 15 +49329 15030 5 +59968 17911 8 +31078 55109 9 +55618 25984 16 +55248 37728 6 +39281 16695 7 +16566 3697 8 +22423 14551 11 +58242 41935 17 +15228 18976 11 +43619 9689 5 +58911 28949 11 +36908 29522 2 +4248 25065 1 +5008 53331 15 +32349 13684 15 +47321 45610 5 +28799 54064 13 +40898 46452 4 +6910 36434 6 +19234 55214 4 +49219 57607 10 +21593 9452 8 +31419 3474 10 +24214 22327 10 +50314 30012 9 +9725 46933 3 +1682 57526 7 +3339 12862 7 +4700 24179 14 +12219 59676 5 +21406 19950 11 +24775 13906 12 +53354 22221 6 +56260 58538 5 +2212 40766 10 +43380 3106 3 +2758 8956 9 +984 54132 8 +20231 59171 7 +48057 43173 13 +21346 2596 9 +45699 34204 6 +31882 33781 8 +50296 47064 11 +21965 19397 10 +50930 35903 5 +55492 56924 14 +40636 20577 3 +29994 52589 4 +36002 58068 8 +27675 46031 10 +38352 7858 6 +36805 47794 8 +24900 36320 9 +10428 39371 16 +34030 39833 12 +36889 26622 13 +19538 52973 12 +38221 59524 9 +24032 14518 4 +12772 8834 9 +47547 43311 15 +35878 11503 11 +27533 7963 14 +26455 54920 15 +22791 27699 12 +53735 18883 18 +49928 11610 11 +12529 49444 16 +21142 5548 8 +38688 22854 16 +50601 24600 13 +33117 9613 13 +22292 21188 16 +33544 53298 9 +56626 42400 9 +56936 50602 11 +22718 2168 13 +47165 7395 11 +16971 12674 10 +57217 44047 9 +43024 14879 8 +4812 9386 7 +19224 35144 12 +10811 19872 5 +21868 34850 10 +10230 11019 0 +15364 53201 13 +55118 57611 16 +14169 31656 9 +50518 31631 3 +1388 14932 10 +1243 59557 1 +55964 19507 5 +52278 28021 3 +48919 32016 13 +35892 52920 13 +36222 58539 13 +14632 13173 15 +21759 4142 0 +46221 13278 12 +15379 9192 6 +8607 35767 16 +41423 49258 12 +57559 39200 10 +42016 15298 11 +21837 51794 9 +21245 4073 12 +39620 16945 3 +5160 12216 9 +19166 52848 8 +17659 21228 10 +37475 41026 13 +13534 30883 10 +58760 26618 7 +44344 37632 2 +9606 55414 12 +45789 17535 3 +681 36652 14 +22857 4947 8 +48757 43763 9 +58520 53196 16 +13566 57198 13 +9577 48084 10 +33541 44886 3 +47027 58035 3 +57696 52002 9 +376 44136 6 +3832 23207 12 +5540 559 15 +35613 54644 10 +32689 25190 8 +6978 25331 8 +44949 43726 13 +21642 18577 12 +42567 43058 13 +8199 4288 9 +5018 1725 6 +32849 52870 12 +17235 56771 17 +37309 37613 5 +37730 4235 6 +36855 21469 3 +59005 38852 9 +2443 59653 4 +41944 1244 4 +47782 23607 9 +29322 44237 15 +28197 12412 10 +37166 33909 8 +36069 19544 8 +15160 4091 9 +29327 12647 7 +27252 50604 10 +44239 20497 9 +34984 48974 8 +32026 25949 8 +15602 15675 9 +10258 28359 8 +48484 30128 15 +47049 22648 16 +38120 5888 2 +41183 39629 13 +5728 36386 11 +19645 42215 3 +59255 10630 12 +31589 45953 7 +22751 12304 9 +58142 41400 11 +3492 5383 5 +1992 30117 5 +46000 1532 9 +15256 55130 7 +40975 57253 8 +28910 57574 2 +7409 53466 5 +28869 20863 7 +114 33001 5 +31108 40508 12 +49569 46040 2 +38163 47221 8 +39503 20542 2 +15269 8925 10 +50414 2779 18 +54275 2891 13 +42511 48138 13 +54205 12829 11 +37898 58920 16 +10681 56713 18 +34833 17986 3 +58841 52302 9 +37681 42490 15 +13758 29448 7 +50849 46096 13 +3382 29064 11 +38167 54479 4 +14098 54887 8 +30597 6390 14 +26162 8833 9 +51796 22608 16 +20894 12858 6 +12477 5518 9 +29908 33016 9 +38490 14249 10 +32317 16884 11 +54039 48113 14 +6659 15356 9 +43944 44521 12 +10546 44695 13 +42741 36890 8 +44823 2409 4 +870 22798 2 +37307 30283 6 +8905 53603 8 +16344 40322 3 +57592 6391 4 +10060 29204 9 +17520 33613 3 +47821 5628 11 +33423 11538 16 +43299 7229 15 +13680 29737 17 +32974 53118 1 +16798 11337 9 +53444 1049 0 +6780 26051 9 +12577 1452 5 +699 18198 7 +14803 59293 10 +4643 36608 8 +16033 49611 7 +22048 41813 9 +57231 57342 7 +56812 11573 12 +35946 17791 8 +39760 28435 13 +4549 32494 4 +15905 54259 3 +50462 54800 9 +4860 22813 16 +13515 6927 13 +26396 46392 12 +56795 26465 6 +7892 20349 0 +13824 11061 4 +56098 1819 6 +22610 34334 10 +52772 49108 12 +8686 19528 9 +49714 35871 16 +52972 24897 5 +58166 46081 13 +26905 28418 4 +21748 41424 10 +12712 2551 13 +24237 56468 11 +58008 8679 2 +33784 56124 9 +24658 35827 10 +12425 5752 14 +26107 17600 8 +46646 52757 8 +57159 56900 16 +54409 59577 8 +54759 59428 8 +56300 5118 13 +14356 21800 0 +22343 26441 8 +29237 31414 11 +1939 1105 17 +58964 10861 2 +29966 6922 15 +46683 12648 8 +48290 26775 9 +18071 25961 5 +41634 37044 6 +30927 11121 12 +22337 17136 7 +51844 27687 13 +48509 1623 17 +30824 51145 12 +31507 36489 16 +24202 47076 13 +46883 35263 12 +58696 1326 16 +32677 34804 9 +10053 19386 9 +11362 27651 4 +19136 1796 1 +49749 45065 12 +4567 49242 11 +53734 45513 0 +11122 26351 15 +52763 56641 15 +1208 33076 13 +26777 6011 11 +37239 4550 6 +825 3440 1 +52532 45054 11 +31472 43971 6 +45389 45307 3 +23977 36202 7 +15115 57788 11 +56063 53987 8 +46441 29658 7 +8878 35531 5 +337 27072 15 +48989 57935 7 +48533 2341 9 +58185 2732 11 +11057 38668 5 +51509 15248 9 +14840 40111 8 +6908 32163 16 +11412 24445 1 +12063 31434 8 +4168 42754 7 +105 43851 3 +14599 27560 8 +50520 57603 8 +27846 12840 18 +51908 42574 5 +57519 22570 13 +11320 50317 3 +26856 36167 7 +54931 58797 5 +21789 35960 7 +8253 47745 9 +38187 30362 9 +50393 14989 14 +48916 194 5 +27283 37990 6 +38470 38792 14 +46511 34285 12 +18488 4501 9 +25567 53613 4 +6084 52955 9 +13135 33075 2 +22886 54831 2 +21454 31307 1 +31093 19802 3 +12613 35994 6 +43819 43854 6 +17389 39448 12 +24936 12916 11 +5134 4821 5 +55601 2952 1 +33130 45205 10 +33158 37914 2 +35780 22124 6 +535 26111 5 +35804 38447 16 +9393 44420 7 +45793 47016 14 +39209 21784 6 +43957 14722 5 +40658 49630 9 +17640 10996 10 +52563 25655 12 +56044 3184 10 +1119 16560 9 +55693 14853 8 +34631 5249 4 +21042 5485 8 +16645 30064 5 +7030 21153 6 +18084 490 5 +46818 12058 4 +17350 9798 13 +25443 56494 10 +29190 1593 8 +7907 10527 8 +23030 35102 8 +58587 35701 5 +43190 15819 12 +29588 18529 13 +57161 58175 5 +38796 5473 13 +44145 9436 11 +23898 48285 14 +59783 41221 8 +24714 34083 2 +44222 40845 7 +11719 48831 4 +52638 1305 18 +16570 31153 12 +26621 23212 14 +45901 30823 15 +18809 27880 9 +14544 37344 13 +39643 24553 16 +3516 13041 9 +18137 12375 1 +13850 59945 9 +11963 6757 5 +33479 56161 7 +14357 21637 5 +4784 24801 14 +36859 39222 10 +4086 2878 3 +43924 24217 12 +19063 21109 4 +977 31896 15 +56200 10547 12 +48906 33999 17 +17778 35187 8 +42805 49768 3 +29255 52699 14 +57663 54935 15 +3602 11048 7 +13881 52716 6 +46901 16908 10 +1704 55099 9 +38307 42952 6 +25077 3924 8 +21585 23060 6 +52409 5925 5 +36999 29368 5 +6150 10626 10 +23560 26445 9 +46266 47953 2 +17168 25222 9 +35869 2789 9 +33824 11952 10 +24372 33378 7 +41819 1877 0 +22069 31932 4 +4649 13972 4 +6238 32684 14 +9194 13781 8 +58999 7871 12 +4804 19847 2 +29031 4018 15 +3640 10982 8 +18826 5860 12 +26258 28122 0 +47204 3645 14 +28631 39949 9 +50946 46458 8 +30109 30924 14 +29517 27715 5 +33808 42654 7 +45655 27534 6 +59254 53787 8 +17092 3369 9 +49211 21489 9 +41363 40547 16 +34052 29358 13 +3520 41253 14 +40313 36916 7 +13279 52288 5 +50140 19570 11 +3052 35229 8 +54960 495 7 +55170 19829 5 +28110 31635 3 +31525 10449 17 +38973 18907 10 +17922 13150 11 +15193 43154 11 +30658 3916 11 +31366 15673 7 +34314 20028 9 +59449 33348 16 +4218 10476 2 +36294 38793 8 +39085 16020 14 +46935 37238 8 +41405 13687 16 +30471 38048 10 +29106 22275 7 +33880 32419 7 +23204 57279 5 +55998 55792 12 +36720 24241 17 +21371 3803 8 +31344 43051 13 +19143 49063 16 +33499 14101 13 +1903 41063 14 +42623 33516 1 +13785 43881 5 +42586 7183 1 +21447 43470 10 +27858 12147 12 +28581 34004 9 +26284 5291 10 +2822 54629 6 +9781 50803 7 +42691 9824 13 +25050 52227 7 +52621 26442 7 +18900 3889 3 +46905 19065 8 +10032 46917 6 +5458 12336 4 +52842 43463 7 +25046 42431 14 +50259 35359 3 +59098 25333 10 +20668 41316 11 +41310 4701 9 +24883 10273 9 +25476 49744 10 +41451 47552 2 +18768 52944 9 +24015 3396 2 +46762 56676 13 +44504 52067 11 +35693 54590 17 +42878 21119 12 +23331 21844 6 +2612 11081 9 +39835 59929 7 +32354 13078 10 +23922 9857 6 +18283 46745 8 +40256 37942 5 +10980 1264 8 +36405 36619 9 +57847 17515 8 +35532 33644 11 +2353 57491 0 +40550 4310 3 +29102 29619 2 +15664 39496 14 +17468 57227 8 +36004 6229 10 +8874 42459 13 +55656 16127 4 +48933 56804 7 +58105 59548 10 +17021 37857 16 +43003 35574 4 +28278 11718 4 +13347 30301 1 +41705 14628 13 +23745 15178 14 +40749 32099 9 +10708 19016 10 +49810 34064 3 +35754 59992 16 +8647 56125 9 +27510 22820 11 +28213 7738 14 +35977 50700 7 +4563 14659 5 +19440 50160 7 +39375 37604 7 +59789 44211 7 +4713 49168 2 +31111 29009 13 +31486 3218 12 +24074 49439 14 +46590 15593 15 +10633 33186 7 +36601 30600 8 +1654 44699 9 +51619 32578 10 +39518 7811 11 +14583 57081 11 +17670 3065 13 +9180 47870 12 +27210 37327 15 +5671 51041 6 +26720 10006 9 +38630 28384 9 +40345 1460 6 +6111 53961 15 +7301 35953 12 +17055 56012 11 +16838 49166 4 +8438 26030 9 +969 59909 15 +47915 46407 6 +57 37878 16 +55478 1576 5 +30633 55692 11 +56271 2172 14 +54834 9766 5 +52720 26144 1 +45277 56556 1 +8036 1737 12 +40955 50255 11 +57876 45353 6 +30378 39320 15 +25758 48001 8 +41167 41156 6 +54791 52632 12 +58063 25973 7 +41708 16260 5 +23978 41311 16 +12297 33189 12 +32214 39766 11 +1372 13303 6 +54968 17031 8 +22137 45259 3 +36952 5369 12 +48610 58801 9 +26067 7388 11 +16049 42189 8 +19075 30567 4 +13647 55862 5 +40304 40566 8 +17333 43968 11 +51661 6339 13 +54288 13642 11 +32056 10854 8 +21783 2379 10 +11566 51782 9 +38126 27662 1 +35190 3584 11 +26041 40710 14 +8142 28481 16 +27274 55274 9 +24008 871 8 +6242 48982 10 +38726 32274 5 +45218 52286 9 +30436 37487 5 +41493 11765 8 +56117 59085 2 +53121 5544 11 +31095 49653 9 +11999 35096 11 +24259 9072 8 +49343 40885 9 +55173 12767 10 +4692 24024 8 +2608 8263 9 +50466 37254 10 +27223 11735 3 +51654 9913 12 +47660 42736 16 +8184 22102 8 +37149 12387 8 +38663 24317 5 +57858 41604 14 +22511 28736 1 +5703 34536 7 +24575 19982 5 +44531 54432 2 +15028 53942 5 +58361 26068 10 +27606 40078 11 +15443 7996 4 +6633 52367 11 +31242 25617 7 +8302 9416 14 +21332 13739 15 +41465 27831 1 +7232 24165 11 +1674 35313 10 +43612 37080 8 +21806 21638 7 +50992 25643 16 +51542 59799 6 +57692 38554 8 +15919 17859 16 +24086 43129 1 +28285 42021 8 +57648 16119 8 +9481 38196 10 +14451 24674 7 +49317 53081 7 +47588 38341 12 +13465 50193 15 +3531 35747 12 +8615 13231 1 +41019 13556 2 +14058 43882 14 +5415 33264 8 +30403 18412 5 +29673 34894 7 +48816 22194 14 +6832 41833 9 +31076 22161 11 +16889 59128 14 +24379 34110 16 +36332 43831 7 +16162 28458 10 +26798 39055 4 +23895 21475 10 +37900 46044 9 +19537 1364 18 +2499 909 5 +34577 59394 11 +40779 26985 5 +40678 21326 9 +48223 6418 13 +42351 32394 6 +23432 37584 9 +51770 6458 10 +54266 58475 17 +30801 32990 10 +23884 20081 6 +33888 12831 12 +35537 27317 11 +16026 48114 14 +23576 55828 5 +47580 36134 8 +29605 16302 11 +39356 5330 6 +6507 13034 16 +39129 42452 10 +40937 41084 16 +48967 40374 11 +53380 10140 0 +38737 36060 9 +48330 33775 11 +3994 19196 12 +38943 6325 7 +46146 57945 15 +31448 7925 4 +28301 3527 13 +55670 7392 8 +34361 50334 6 +21148 48450 14 +40834 7148 13 +55730 12939 15 +9308 37356 15 +21655 16532 6 +57295 48573 10 +10686 35350 6 +3256 12716 9 +1656 58544 10 +14202 5855 7 +54561 50427 11 +37983 39468 10 +54706 59738 4 +7202 43615 7 +38949 12374 0 +53993 2518 8 +43411 49820 6 +36044 31551 18 +56141 47287 11 +20753 42057 15 +43587 58191 11 +53691 53040 4 +20116 41531 12 +47034 25730 14 +50659 26179 18 +56722 46893 8 +30349 11494 6 +52752 34312 15 +57671 59181 11 +16404 31548 5 +24002 20237 9 +13600 59740 14 +12440 54107 12 +11278 7642 9 +17523 33417 5 +33014 4706 11 +36279 15879 12 +57096 10457 9 +15958 29283 2 +9818 57515 14 +14948 47119 2 +23879 19443 5 +47030 37607 9 +16750 8656 13 +34997 51387 13 +2035 52430 14 +41895 43645 16 +12670 28794 1 +32852 58590 3 +33031 33820 12 +48419 43528 7 +47551 27971 8 +52141 10119 6 +46471 44956 10 +40018 52399 12 +54302 4965 12 +17551 19216 9 +25885 42710 5 +10864 4823 10 +37703 2418 5 +38215 53382 10 +9134 21242 8 +26192 16642 4 +50120 59545 2 +51516 23962 12 +48564 42467 13 +15169 10043 9 +32786 17534 3 +13222 47772 9 +34394 583 9 +43734 26714 16 +29115 53217 13 +41706 39588 18 +2252 47303 10 +58390 23789 12 +19416 33131 14 +38856 44090 8 +15121 22708 4 +46439 43201 8 +45902 19696 12 +49086 55571 9 +22583 45146 0 +45997 11575 8 +32992 13622 11 +29457 53389 11 +38216 1007 5 +54789 37742 8 +38801 59265 13 +20325 19228 11 +35801 27078 6 +57492 11639 3 +59895 1908 9 +7290 23454 8 +53245 55356 7 +45310 897 11 +12717 50760 14 +9100 12781 11 +31457 1114 13 +56846 10058 8 +37748 36711 10 +8909 34109 9 +47616 12782 8 +7490 8627 12 +13416 6085 13 +11777 48216 7 +28033 47319 13 +12133 4439 5 +27612 9808 10 +57045 8177 9 +7401 31724 9 +3125 7056 13 +34977 29803 6 +26801 36567 16 +54744 21524 8 +50785 42571 14 +11016 56720 9 +7111 27795 9 +29722 3063 17 +10310 40502 7 +7944 12366 2 +38859 19179 3 +19973 58565 12 +7962 36521 14 +1996 17906 8 +51379 28699 10 +54115 50804 7 +59679 26654 12 +51538 22222 7 +38382 51230 11 +52517 51223 12 +18913 1350 10 +58265 1104 5 +58751 26358 10 +24396 24479 16 +34613 30255 1 +35632 59214 9 +57162 13193 3 +29109 22233 11 +54581 53251 14 +34316 56249 8 +54227 14298 7 +51636 26551 13 +25678 2349 13 +55905 55850 14 +12619 25244 9 +1086 4011 11 +40309 34810 8 +14534 33879 11 +34956 10412 11 +7927 47427 14 +22988 20756 7 +3438 48321 11 +29778 33296 11 +24546 13352 5 +6464 47074 14 +7151 3543 14 +28385 38094 2 +36286 11853 13 +35899 12596 13 +20899 15943 5 +13950 33543 8 +57441 16679 3 +12110 2922 9 +20485 44419 6 +16256 1232 13 +53043 5690 8 +6643 43702 12 +18439 38437 15 +14815 53778 18 +55928 16225 11 +31466 28270 7 +7163 4477 12 +16332 46954 7 +59871 35226 10 +10926 26214 10 +2170 40691 11 +34855 5894 12 +48984 20279 5 +56058 725 15 +27265 11092 14 +55097 39213 9 +29434 36956 2 +30181 43550 18 +1832 33765 15 +38001 55967 7 +54 4985 9 +18626 15504 3 +38061 37901 2 +36477 33563 13 +56416 8508 11 +50346 51918 11 +59869 19821 3 +13588 9080 11 +540 12533 3 +1188 38004 9 +51463 5683 6 +56289 12915 6 +18787 15577 11 +32387 22606 6 +7143 43450 2 +41004 47024 4 +13877 41242 8 +43602 3022 9 +30307 32143 13 +10049 56111 12 +29659 23498 6 +5103 39770 5 +58018 56800 13 +46494 49847 6 +30369 16369 9 +52378 24829 11 +44750 717 4 +35334 59856 14 +18304 20788 10 +38467 55769 11 +10622 30705 7 +2636 56858 16 +19515 40590 16 +53950 20794 13 +6395 38713 2 +29819 11615 2 +57980 27343 7 +26058 55931 1 +58656 39138 4 +16288 36485 9 +6317 9089 6 +38022 3316 15 +2941 25673 9 +52812 47327 3 +39341 34166 5 +36729 1967 9 +1159 19963 8 +37374 22818 7 +51214 15935 10 +27520 13267 6 +30206 28535 10 +36138 32438 2 +45117 5916 16 +44855 4335 12 +20622 55580 3 +46138 13060 8 +29385 24551 6 +5890 52473 6 +13782 56359 8 +54108 10170 7 +2334 21889 5 +38181 24800 7 +13427 25516 8 +41047 12204 8 +39102 39709 7 +52704 16258 10 +18702 34569 15 +39196 862 3 +48193 56411 9 +36118 20422 12 +10816 19300 15 +23486 22232 6 +8907 11794 3 +12685 3068 7 +39258 7830 6 +42494 48843 10 +19913 8877 9 +47705 7950 13 +52431 913 9 +27195 15180 12 +45255 44125 1 +22286 21506 8 +57902 27388 6 +12157 17355 8 +25705 34479 9 +20498 46679 8 +11113 43507 8 +28661 2874 11 +35776 2017 10 +8464 3590 1 +14270 21451 10 +57760 40408 8 +23402 27597 14 +53519 26180 10 +31834 15198 9 +38426 11663 10 +13559 37471 15 +5114 36897 5 +16698 33892 10 +45162 50736 2 +3630 59000 14 +37421 19777 14 +23345 25563 15 +1555 49865 15 +55488 34960 9 +54142 49290 13 +8198 49434 9 +43973 49217 14 +11543 11447 5 +52674 24722 8 +39530 13863 13 +6359 57612 8 +48468 20451 4 +2510 14525 8 +55167 4782 11 +52190 12753 14 +1399 49988 18 +14726 10909 9 +31041 29315 10 +59737 8875 6 +49643 12740 4 +23705 50403 5 +4955 21154 5 +42931 48313 7 +46099 44812 16 +41759 34235 8 +44310 40659 13 +39811 57135 4 +20494 37451 12 +46041 8515 0 +58780 9354 5 +1744 49550 10 +3311 32124 13 +39015 48018 9 +30503 4449 5 +430 30397 14 +1068 37026 12 +57412 8274 1 +42892 58694 7 +59372 46724 7 +52024 31945 5 +28843 4154 7 +10880 39716 4 +12250 38493 17 +55523 883 8 +48427 46687 8 +1063 20525 12 +11837 8414 8 +36456 38264 10 +23277 6876 6 +49539 17493 10 +37372 24571 11 +13522 9752 11 +41184 29180 16 +40159 46409 8 +58579 45684 12 +7135 34666 7 +39778 41777 9 +55489 2131 10 +57184 28515 7 +28859 52567 10 +54230 32477 11 +42275 30239 8 +45493 44943 15 +59627 26462 17 +30892 29445 17 +53722 33889 7 +12142 14545 0 +14939 10166 13 +13040 26158 12 +9028 55712 10 +10584 4525 16 +48684 37441 9 +46907 15455 17 +35884 20515 6 +51027 54187 9 +16777 48231 2 +23059 55736 5 +19740 39532 16 +55986 48416 7 +58335 49862 8 +5368 28158 7 +56045 41330 7 +23164 59137 10 +34533 13760 14 +29086 31687 6 +14148 15587 3 +28220 49209 14 +4083 17401 14 +676 20960 3 +78 36905 7 +15723 50060 13 +1667 47248 11 +48517 44794 3 +56991 34323 3 +36996 9084 11 +36055 34498 4 +36305 39176 11 +11978 21572 10 +1523 46517 9 +31898 56353 6 +12691 51738 2 +37355 16241 4 +29381 46478 16 +15493 16036 9 +49363 39284 4 +6800 52718 15 +34490 6134 13 +50563 18510 11 +36276 42792 3 +21386 42472 9 +22661 59865 10 +52861 13632 15 +5420 18820 11 +15344 24823 8 +12273 5546 14 +12517 26302 11 +47324 52873 4 +46912 52682 6 +9962 58336 5 +47833 26650 15 +59724 30194 8 +42976 17317 16 +23230 31688 9 +24595 52805 14 +4337 10438 15 +4688 33419 8 +59461 10665 8 +49931 59347 12 +55023 12235 8 +57262 32148 4 +55287 34985 5 +24033 43506 9 +11364 40130 6 +5664 18894 16 +53096 47137 12 +19394 37396 9 +14938 3388 6 +11710 47011 9 +58153 59528 7 +31676 23639 17 +506 56051 15 +48545 1307 11 +22358 31894 3 +1975 34547 10 +169 38195 2 +5749 6873 9 +49348 45046 8 +22683 34063 16 +43185 37813 17 +52997 38698 12 +54452 5571 5 +5460 37590 13 +11513 38844 11 +51361 6258 11 +20370 37160 12 +27574 1946 14 +11520 56760 12 +45078 48817 4 +21782 9968 14 +33978 44499 4 +11809 28233 9 +53082 10387 17 +45384 28258 5 +59808 36605 11 +43923 600 16 +6180 22061 14 +47801 57071 9 +23780 44533 12 +58389 55281 5 +38211 34645 3 +5636 8861 11 +30393 53635 12 +51056 31578 5 +32797 25341 10 +27204 7717 12 +26944 35312 5 +44766 42661 8 +40768 13646 16 +52881 17419 15 +26990 34081 7 +26597 13513 7 +17099 34598 14 +13553 40376 6 +32201 49909 16 +19667 2442 17 +46888 55961 8 +45030 33159 12 +3787 47289 9 +20299 29775 9 +43144 25390 5 +14789 18761 4 +37168 55510 16 +27456 16997 2 +11144 13510 5 +21205 55604 14 +59053 46610 7 +46918 12062 10 +48809 20357 6 +32013 16840 10 +32353 27403 6 +35582 45815 7 +5886 40282 6 +35135 53996 17 +20026 8090 16 +54793 40300 18 +42193 33983 5 +32821 34236 17 +33551 29772 4 +24167 37136 17 +53306 23434 10 +36836 43668 10 +25644 28311 8 +5372 19455 16 +1173 25227 10 +45402 28940 8 +6148 37437 5 +10782 55195 12 +29094 49292 4 +21890 34480 0 +7647 17038 12 +57767 37282 14 +26064 40100 12 +19998 29776 10 +20320 25421 7 +37889 44060 3 +23442 59648 15 +17632 1056 12 +10885 59197 1 +20372 51396 12 +2173 28325 7 +21707 522 12 +42713 13956 2 +13839 28038 4 +6755 20821 8 +28464 11038 10 +50848 32267 8 +47144 56193 16 +41856 45648 2 +22790 10628 15 +52649 18524 3 +36597 53615 4 +7041 46351 9 +58312 25850 12 +5896 58094 15 +29992 49504 5 +52482 56904 12 +18892 51481 6 +44000 53730 8 +45682 58564 3 +39168 12894 11 +45414 7671 3 +24643 30656 17 +55846 7124 5 +44350 17834 4 +24898 59616 10 +2002 34374 14 +22923 14158 8 +49194 886 6 +55330 27217 14 +42376 27852 11 +47890 9834 2 +24679 2143 13 +19622 58714 7 +8461 38676 12 +36325 27486 11 +10468 53202 10 +55853 39204 12 +53513 59610 14 +59517 177 2 +20155 46397 15 +38011 11085 11 +47646 55192 13 +4343 1709 0 +19635 7517 11 +34844 22587 12 +31538 45059 10 +50797 7646 3 +30710 25635 6 +26533 12805 3 +42977 26106 11 +35345 33562 6 +47904 8858 3 +27083 16088 13 +27942 30504 12 +10059 4735 12 +16850 50027 10 +42318 28751 17 +43236 19817 7 +29668 12299 3 +22653 55367 14 +8808 45998 5 +503 36264 9 +49287 52986 9 +42687 43031 8 +28826 18636 7 +34559 6341 2 +8536 30956 14 +41188 9776 8 +31293 2647 12 +9232 6133 5 +48483 28896 2 +3232 55008 12 +19483 8777 10 +50298 18475 9 +40369 58205 9 +55027 21944 1 +38459 43025 6 +2033 59034 17 +11051 59943 5 +957 18759 0 +42969 54272 2 +49049 15259 10 +44901 53408 7 +39793 48243 16 +18831 52658 8 +52475 53521 8 +30743 40818 11 +58381 48729 5 +42531 2164 5 +44081 45176 18 +9633 29362 15 +13140 18107 8 +24270 36327 12 +10806 9801 5 +24993 8779 5 +48836 48080 16 +26444 36158 8 +5701 32727 9 +11980 6357 5 +16326 17826 9 +5942 59007 5 +26951 53294 9 +13668 35669 10 +55098 52506 13 +56226 4024 8 +10263 50241 13 +37541 2463 12 +33360 55456 15 +34521 35314 6 +7780 30123 2 +6210 37682 13 +29059 56101 2 +18965 35924 11 +13742 47228 11 +36943 16703 7 +15326 36205 16 +11359 32436 11 +53683 59192 15 +21765 9846 10 +54990 47273 10 +46660 869 9 +15925 14171 7 +57625 18220 9 +25830 29014 10 +36829 15464 5 +27993 19733 16 +24182 25667 11 +2177 31273 9 +15492 15101 3 +42627 38038 8 +59279 48882 16 +21742 43951 3 +43673 16683 8 +18659 43539 2 +57583 13366 2 +15566 25261 2 +40602 22183 11 +33796 20814 7 +9050 54057 17 +41988 41863 9 +17475 44173 17 +11637 46948 6 +58178 56515 11 +35324 3919 10 +6126 49477 5 +18399 48245 14 +30606 34455 9 +27230 34163 18 +43590 31675 11 +59866 56542 13 +27484 59280 8 +42003 43966 17 +1984 43075 16 +8953 33497 15 +38977 37180 13 +39395 31220 14 +41562 23154 9 +35621 43083 1 +21897 43488 14 +17817 50164 13 +9994 49528 0 +52866 16213 12 +18642 16749 8 +3573 43273 4 +20384 20445 10 +52882 53791 2 +40022 40970 12 +46125 27014 4 +32023 21501 12 +49177 2477 16 +3943 14367 3 +12802 34356 2 +22750 46287 13 +48522 10799 13 +4029 13518 9 +32939 28172 9 +29471 58742 9 +29379 28753 18 +56540 21096 6 +5611 1069 4 +58244 41905 11 +47275 21760 2 +40538 27495 10 +58698 35418 13 +44392 8832 11 +22503 781 9 +7389 46514 13 +9369 35636 15 +34926 49060 7 +58349 51026 11 +17137 6204 4 +37844 33452 8 +43600 8595 5 +26327 17553 11 +23199 14483 9 +57164 25127 11 +55541 19491 0 +58489 8257 2 +53364 38806 5 +39482 20977 12 +43531 35249 7 +3112 14979 7 +35478 15592 11 +47797 53507 5 +12936 24942 17 +44433 50262 11 +36898 55399 3 +25145 50371 10 +35556 47825 14 +17689 166 13 +48556 34919 7 +48442 267 13 +11798 43767 13 +48486 3524 10 +13154 18019 2 +55629 26262 2 +2782 43697 6 +2098 55110 3 +57290 6581 15 +40386 17089 9 +56917 53422 8 +47564 333 14 +39247 48273 10 +25367 31881 17 +26289 56970 12 +48112 15162 10 +51833 35793 1 +57630 13028 13 +22819 21639 14 +6427 6943 11 +4349 39127 9 +7144 30331 5 +43754 39409 13 +36654 9347 16 +55682 5189 1 +52068 35304 6 +13336 17063 8 +56530 22416 7 +29003 54336 17 +9447 17295 12 +43868 18954 7 +22234 45817 3 +58 11611 12 +52171 20715 10 +17502 9830 7 +59226 10265 6 +32543 57125 7 +22070 42041 5 +56649 57533 12 +47671 59667 11 +51485 51068 13 +36197 582 7 +45919 31066 12 +21428 3353 11 +15 20900 16 +52327 25978 3 +13766 10517 11 +46033 21548 9 +27710 20689 18 +38572 18371 1 +27861 53016 4 +55548 55951 9 +45223 34493 9 +5205 35157 8 +1886 57511 11 +28871 22547 12 +7096 31990 7 +14430 1714 2 +22968 27725 13 +39668 57978 15 +8381 31136 6 +57646 14377 13 +57311 3313 7 +43198 35989 7 +24135 25622 6 +2043 17470 11 +5006 16116 6 +48914 51804 13 +22540 20203 12 +28900 28120 10 +24841 34295 8 +31213 43703 13 +4448 20233 14 +5384 9509 8 +21660 43313 10 +38008 4921 5 +43098 6293 8 +17208 31984 7 +12612 12074 13 +41824 49144 5 +52717 45142 14 +10779 53433 8 +37906 38463 10 +10052 54967 3 +22809 24334 10 +27652 4026 11 +59050 24504 14 +45330 31209 5 +54366 9684 10 +19690 988 13 +10277 27267 9 +26973 11916 13 +13030 23220 7 +40684 54312 16 +22059 7368 13 +53283 24014 5 +18307 57599 4 +3479 55596 1 +38899 45241 15 +45056 38353 17 +58869 13195 14 +49628 51391 9 +28554 10316 11 +25570 57547 9 +24784 40389 3 +39177 55386 7 +24852 2949 15 +10295 6453 13 +38692 52959 4 +6467 40604 4 +58210 2290 8 +5310 6508 14 +13269 15084 15 +33526 16994 10 +15328 43220 4 +32270 41853 16 +554 13865 11 +40459 28694 9 +54723 21850 10 +7846 21882 4 +51134 18495 13 +6452 45837 6 +11435 4581 10 +24649 2806 2 +51236 17188 9 +33493 35746 4 +50185 59751 8 +56420 35410 5 +19396 5161 12 +28113 55660 15 +15105 26190 2 +216 33520 6 +33546 33308 17 +34789 14619 5 +54716 42430 8 +56244 37114 9 +6089 20890 8 +47312 13141 11 +24391 9547 8 +21608 56689 1 +8096 35779 2 +56296 52826 13 +46890 56348 14 +31228 47297 14 +57578 47042 6 +23493 56262 6 +55324 26978 6 +25101 26288 4 +58446 2011 11 +43298 40903 7 +24029 56928 14 +52111 13187 10 +49001 20693 9 +55644 22398 2 +27964 48265 17 +35745 4329 16 +48895 59827 1 +20500 349 10 +29836 50986 8 +48636 56943 12 +38548 11529 14 +55555 49263 8 +24105 44576 8 +4574 17103 5 +27271 56192 6 +17692 26481 14 +11890 31173 8 +52326 24842 5 +54900 25927 2 +22011 7487 7 +52175 52595 10 +3567 33294 6 +54669 48472 15 +18242 26994 10 +20343 10560 10 +33987 23409 6 +57111 5539 13 +41555 5049 8 +34080 50426 8 +36957 46529 9 +17037 36585 11 +30988 38421 13 +21734 10447 13 +34760 15543 9 +27222 57670 13 +27080 48501 11 +59803 56403 10 +35112 45148 6 +4156 47708 7 +32837 11642 15 +46519 58426 17 +38198 59282 3 +58370 22705 7 +24514 33846 8 +1414 7610 13 +24838 6247 2 +8778 18397 4 +43103 8830 6 +43553 3823 8 +28299 20805 6 +30358 11025 14 +15438 21559 4 +56646 2687 12 +44921 24977 8 +3551 35668 8 +37276 54653 7 +56213 38272 3 +46507 57059 5 +58323 32958 3 +48669 40427 7 +13659 42715 14 +57082 59859 4 +48655 26285 14 +45761 16582 9 +16876 2256 5 +17782 17363 10 +2024 19121 11 +9999 58700 10 +18449 48407 15 +22549 58324 12 +45715 40780 7 +45602 54299 7 +28702 26010 13 +29883 18308 5 +45058 57832 8 +49583 29069 1 +168 24192 11 +14228 30756 10 +37880 43418 2 +14267 18207 3 +157 40555 6 +12443 15596 11 +21187 41498 8 +43829 27617 5 +17172 31226 14 +43005 18395 9 +49459 29792 4 +54519 26252 8 +24760 1745 16 +13487 34308 3 +8298 2693 6 +11240 15912 9 +12804 20301 12 +50499 37714 8 +1132 1866 7 +5761 32341 11 +20602 18891 2 +54675 50976 4 +59202 40478 8 +52527 50463 5 +2081 13369 1 +40013 33639 11 +48460 31561 11 +24404 56080 12 +16261 14355 11 +49508 47631 11 +31914 27425 15 +50907 45564 5 +15280 15730 7 +49777 47015 11 +1579 36024 10 +31503 49269 5 +51161 7109 13 +9662 40826 7 +13805 7350 13 +16481 23003 10 +25765 9125 13 +21057 11933 4 +48731 27765 9 +46582 6536 4 +5342 47491 4 +46224 22384 8 +3420 18613 10 +32504 343 17 +31060 15188 14 +4504 28226 9 +4377 21070 10 +35201 47993 5 +14003 36298 11 +49128 51154 12 +57080 3722 10 +4952 13656 7 +55095 18565 1 +51817 1680 5 +10367 2472 4 +29152 29253 7 +2966 46526 11 +48975 46391 2 +45432 5863 9 +50013 43591 13 +45032 29800 4 +5043 56281 12 +13404 16254 6 +45522 55917 0 +43651 53925 9 +50380 44546 10 +28740 50607 8 +31487 19941 4 +26970 44754 15 +32452 56745 13 +35911 8757 8 +55397 51184 9 +56574 27979 7 +26884 36372 12 +33393 46080 15 +43525 25825 9 +23188 13051 12 +6383 24804 15 +33545 37679 10 +12758 32118 12 +23825 14829 12 +15333 947 13 +32033 46880 10 +33251 41222 9 +12283 8179 10 +30860 22448 8 +8223 12184 0 +12108 32251 12 +568 30004 8 +17768 22106 9 +54306 56100 4 +46411 47654 9 +32719 32403 13 +21078 12214 3 +10615 36845 9 +39422 38666 1 +39603 35161 3 +21999 42454 6 +51132 13102 10 +27543 19846 12 +42115 52459 7 +44172 47271 8 +9329 17492 4 +16852 48934 9 +2544 43323 6 +52493 14731 14 +7533 8171 9 +12076 10111 14 +49867 8516 14 +28662 13449 9 +29592 24288 8 +30509 40192 9 +22636 50881 5 +31543 27565 13 +32407 16218 9 +23140 36005 10 +16887 14348 8 +44760 34495 7 +27980 59821 8 +50336 57854 12 +14984 29138 15 +43390 30652 14 +29447 49218 1 +14500 9738 11 +29815 14555 7 +26979 50351 7 +39348 25100 11 +23425 47905 4 +53021 53679 14 +12722 57839 8 +41671 53155 12 +54398 51729 9 +56199 21948 12 +31871 32325 7 +21485 57769 6 +8496 55451 16 +21595 51926 13 +3577 37376 15 +4542 15545 9 +34649 50139 9 +35258 3550 7 +39372 12668 13 +2576 17152 16 +36547 19086 14 +19064 43063 10 +6886 47104 14 +45499 59950 9 +7763 1757 17 +14786 43735 9 +29414 19280 3 +7848 19256 14 +30259 47110 14 +40077 12078 8 +55526 25585 11 +31997 661 11 +59930 35266 4 +28589 18484 9 +55166 28340 11 +59963 57370 11 +6812 26717 5 +59654 54874 14 +13283 49035 2 +16778 38489 17 +57049 48075 9 +34522 5909 8 +55869 5033 7 +32847 38790 5 +8312 4770 12 +9077 12427 11 +6645 56427 2 +27316 476 5 +19565 49591 9 +50822 27502 12 +27738 49038 9 +15718 58847 11 +50370 21194 9 +59566 45920 11 +20405 24811 9 +17293 46089 1 +2888 54562 9 +33157 11334 7 +23523 27955 5 +55035 9605 8 +17625 54972 3 +5308 5007 7 +12534 41089 14 +3047 13618 11 +23361 17664 6 +58339 30703 11 +49839 20174 2 +46873 48093 14 +20126 52912 15 +16591 45529 9 +16349 1033 8 +25821 41210 10 +58746 8026 13 +14473 57073 6 +3808 51213 12 +26287 34056 9 +20332 59304 8 +36747 22889 14 +45527 55278 9 +53872 17925 9 +4871 11224 15 +56834 59422 3 +29615 28082 8 +26718 41823 3 +9143 3875 12 +39765 12143 12 +30576 10654 12 +11279 57817 4 +19533 7014 7 +41873 59108 12 +1148 13605 9 +40262 58942 8 +10600 40390 6 +13852 7260 9 +21395 15045 11 +55762 52797 8 +52827 9146 12 +37410 17308 7 +34463 52499 9 +32266 32785 7 +31893 36724 15 +31514 58772 5 +8519 2396 5 +38192 8138 13 +26865 8020 15 +31767 13749 14 +56152 43704 10 +10970 48696 11 +32372 5635 6 +32314 40524 10 +22241 35130 15 +44953 33774 7 +18723 56043 8 +58238 45977 3 +31301 26616 13 +43148 4219 14 +48443 55369 11 +31851 28322 9 +20253 22165 15 +26404 16372 8 +45214 25220 6 +56040 44467 10 +48661 44253 5 +442 46277 6 +6731 49566 11 +8652 37794 5 +47419 37464 3 +33874 13621 6 +56409 36252 13 +11844 513 7 +39721 41382 5 +22724 4887 16 +21849 39328 5 +41312 54193 12 +31744 24053 18 +44201 56458 15 +16429 48170 7 +59186 5409 10 +59496 16333 14 +58975 1675 10 +21052 17221 7 +40772 42774 13 +44624 51744 13 +58852 32732 18 +57393 6620 8 +51129 34895 5 +6994 33342 10 +50476 57677 4 +32456 15281 11 +4331 14124 9 +25884 13257 6 +27341 37737 16 +28167 30409 1 +49260 16045 1 +16129 45725 7 +8144 33163 7 +33437 23453 11 +4359 1897 3 +50208 40494 0 +39702 35032 16 +4030 43793 14 +48837 13293 5 +20213 7213 10 +2151 18469 8 +19038 42966 3 +30833 16025 5 +7863 17719 11 +18964 42999 4 +58453 18910 14 +18525 39645 11 +37749 1101 13 +53157 49171 4 +37390 42843 12 +16493 57690 10 +5578 21870 12 +6491 25059 15 +2006 25113 15 +20244 57865 1 +11082 57910 3 +18549 4579 6 +57660 42742 7 +41617 47550 8 +29796 28513 8 +59159 6347 7 +53799 37967 4 +36293 47477 10 +54454 37659 3 +18202 49445 9 +58681 18629 15 +43603 25787 7 +30806 23358 14 +949 46748 0 +28619 54655 9 +2087 57265 6 +20991 7469 8 +45857 15100 4 +52468 1478 10 +53911 29043 9 +41060 37868 14 +25539 12313 5 +12635 12979 7 +27974 48255 17 +28334 12880 1 +45421 23127 16 +50084 14313 16 +38592 21388 12 +33102 53608 8 +43937 58612 8 +52465 49094 13 +59229 11399 10 +33582 22250 14 +40618 3247 3 +48543 1099 5 +14536 25319 14 +9795 55901 5 +59360 20522 9 +49698 11020 14 +47995 38266 10 +3170 54448 7 +10356 36243 10 +38776 8251 9 +41714 17311 13 +14220 32495 7 +28089 52207 6 +21226 3536 8 +48252 54348 10 +49966 13910 5 +32914 16231 9 +31989 46501 10 +24388 42997 7 +5296 46082 10 +3593 25896 4 +52552 14746 1 +52204 55599 9 +30063 47171 8 +33726 18955 7 +32718 19031 3 +53747 52253 6 +9663 52673 3 +19013 56967 7 +15763 32952 1 +25396 9463 4 +41474 10347 3 +40125 56517 10 +37509 18594 11 +43407 10197 10 +5704 26572 14 +51368 25799 12 +1210 34599 12 +3053 14037 15 +6685 8473 4 +39646 6787 10 +27895 41548 6 +44030 59487 0 +43979 56916 12 +27622 16630 9 +23812 35542 6 +25861 9170 8 +38442 12806 15 +52173 7597 7 +8314 38153 15 +49339 22470 6 +31544 39516 8 +46283 32384 11 +2121 52220 1 +32471 16432 16 +40392 18228 13 +53407 23416 6 +3073 51136 9 +19088 19983 8 +37295 32192 7 +44994 52388 6 +59225 46286 2 +43918 56383 12 +4262 1043 10 +49761 25197 10 +54897 41661 8 +19776 35874 4 +31356 41699 8 +26037 44982 14 +16998 12373 13 +16364 32018 13 +57155 6697 12 +35840 10652 9 +22393 44430 14 +22579 44131 3 +52519 42204 11 +7970 59114 7 +11407 6095 11 +53808 17328 7 +42056 57633 5 +33598 13212 8 +27925 7684 13 +48857 54207 13 +13864 13131 11 +13325 28874 8 +12185 8470 9 +54122 50862 5 +32029 39659 16 +58499 42570 15 +13887 34982 16 +9880 22345 5 +26652 40641 10 +1126 4888 14 +12268 10494 12 +3760 41701 18 +16678 38237 11 +34100 3150 9 +35485 2071 4 +4801 10930 11 +21757 15354 18 +11159 48898 9 +33872 32590 6 +40423 48145 7 +39850 36997 1 +6569 53071 13 +18276 10281 13 +45785 29435 13 +20311 57664 15 +38999 36182 17 +23284 36923 10 +48911 57202 0 +31515 13288 6 +34257 44675 8 +49711 23112 14 +55859 43678 15 +22506 20836 9 +54984 48567 14 +17056 6921 10 +49117 56578 6 +7221 17014 7 +57999 53127 8 +20504 24093 12 +23604 48347 10 +23072 32533 11 +27027 21635 8 +11829 26568 14 +12585 49597 8 +35368 50882 18 +46009 37228 13 +20751 19926 2 +23875 45013 6 +53721 35920 15 +47919 29527 11 +6235 8587 8 +35965 30737 8 +6535 1269 15 +42717 3965 4 +4873 33429 14 +33304 55131 16 +40441 5145 6 +2283 22335 4 +40128 47979 12 +58316 29509 7 +55134 30327 10 +58296 9974 11 +52006 52964 9 +48854 7880 10 +51943 2216 17 +50875 20047 3 +14563 16675 9 +8159 40714 1 +1839 12351 1 +50680 709 3 +46629 2137 17 +10164 35680 6 +57725 29707 13 +3800 14883 4 +20292 9076 8 +13899 7932 8 +4238 42690 4 +31467 7947 11 +14749 56145 8 +44730 35979 3 +58173 15526 9 +49728 28559 7 +35810 55289 12 +5676 32019 8 +4702 53636 14 +12705 31998 7 +477 30984 5 +25932 3674 3 +46457 11510 14 +57334 35076 9 +33287 41760 14 +29918 17506 8 +33069 16685 5 +8862 13736 9 +54242 29661 10 +8117 7812 13 +25112 27338 7 +17642 31839 6 +36509 36558 2 +8738 13452 3 +26778 33836 14 +55468 55291 16 +31225 28625 18 +22770 44782 6 +3067 16463 9 +2709 13745 3 +17421 19988 11 +17997 17042 4 +8733 31617 8 +6046 47856 12 +26490 16237 8 +50030 18260 11 +12988 45767 9 +39333 14788 13 +18755 25283 14 +4808 3222 7 +11882 41939 6 +1250 48225 18 +15900 26359 16 +35481 12 3 +15696 14992 2 +19251 44139 14 +54161 16438 6 +17948 7760 10 +1706 25020 8 +9449 35900 3 +47938 56645 8 +40853 28319 11 +12300 40735 4 +47573 36253 7 +3030 25326 17 +17764 2798 15 +45009 38810 11 +14482 11565 7 +11757 19921 10 +39942 53656 11 +43252 6404 11 +50951 47070 5 +52751 1282 12 +32861 53413 12 +52392 40872 8 +21972 55125 10 +28779 52018 17 +34042 57868 11 +49259 48272 17 +43866 47869 7 +16264 59400 12 +37454 30651 3 +19624 4661 13 +29262 52804 0 +50029 17207 12 +20719 20654 1 +11473 45556 8 +52398 43724 16 +9850 1122 11 +36363 6282 9 +49500 3792 8 +5031 55458 9 +21054 57586 1 +57504 46284 16 +58345 39216 8 +55297 45183 9 +25464 8566 15 +27145 55989 8 +36848 16594 7 +36255 48994 6 +32892 35932 9 +5187 51162 7 +20675 2829 7 +20970 4293 4 +57317 908 8 +20506 2158 7 +22955 57741 15 +38596 50987 10 +54481 17671 12 +51075 4418 9 +2617 16174 5 +9415 30111 9 +4677 12918 9 +39486 58301 10 +44762 4203 9 +49910 22192 7 +59937 9022 14 +34542 10765 4 +35154 24617 7 +21478 53212 7 +48258 4526 9 +12495 1420 14 +49314 50791 10 +31541 52159 12 +36948 38764 1 +40495 24878 11 +46634 34620 5 +24252 56718 11 +31690 1618 14 +50458 21827 18 +19574 16764 10 +38228 20085 0 +31801 51231 13 +29553 18129 17 +4899 39429 9 +18512 14982 13 +20823 3565 8 +51555 44039 10 +48773 1537 12 +48025 38920 7 +41097 17180 10 +44303 2590 9 +31244 460 18 +52078 8822 7 +4809 4974 12 +38036 29446 2 +56669 15257 11 +58821 45189 2 +56833 23242 7 +49077 11948 3 +2879 5235 3 +49819 20926 5 +6014 32245 8 +17981 52366 7 +33165 18869 10 +30366 7798 10 +5805 39914 6 +1895 47828 14 +16989 42515 10 +46504 47054 9 +47522 8837 11 +20456 14835 3 +55508 15828 2 +3648 12170 11 +50332 18157 10 +14286 42269 10 +2016 45409 14 +27802 2042 8 +56119 33441 6 +33459 49233 13 +2359 21309 8 +50356 55455 6 +33877 25571 5 +23047 38725 11 +16478 11607 8 +51025 59915 11 +58882 59375 3 +56684 38105 10 +15325 56438 7 +32593 49667 10 +35160 51987 12 +662 3517 8 +16674 15199 8 +51525 34870 14 +27870 47895 6 +30507 6754 9 +40103 46425 3 +51371 27000 10 +26145 35789 17 +30178 46248 13 +32165 3973 4 +16380 39993 14 +13633 5827 13 +609 20671 3 +39031 29784 4 +49533 10893 7 +18164 42711 6 +47447 54740 7 +42611 34795 8 +41385 19114 6 +47133 30777 14 +9261 24899 8 +2637 10817 14 +32156 30420 7 +41572 48106 14 +47664 23039 11 +58756 27815 14 +48701 32830 7 +49894 6013 6 +59145 18163 7 +10373 9033 10 +1875 1482 14 +27528 53355 1 +15835 37029 10 +5733 37158 2 +56691 49286 5 +38 55182 15 +25714 54177 1 +23862 51290 1 +22864 28203 2 +18783 8368 2 +19646 35321 7 +6038 38015 9 +4479 58131 15 +54466 29095 12 +53853 37419 13 +44040 33556 1 +23611 34297 17 +6193 29634 12 +29862 51717 3 +50010 51337 9 +7299 2555 12 +27196 23392 9 +36555 32553 11 +37946 624 7 +13348 10763 12 +16621 31691 12 +18360 24808 2 +50080 52342 5 +39099 4588 3 +371 5468 7 +15273 6071 8 +48815 6947 7 +35356 56699 6 +16278 1093 1 +56018 35223 16 +21809 34593 4 +46101 9683 15 +6641 31283 12 +52199 24077 12 +46209 32252 13 +35662 40193 11 +7334 45015 10 +519 6652 4 +39891 21692 6 +26360 46878 10 +48578 58647 0 +23545 42370 8 +17574 17096 11 +39238 24146 14 +54886 21203 1 +31738 6532 12 +26676 59705 6 +45706 24232 13 +20093 29096 11 +15470 16633 7 +19837 15776 2 +15279 47673 8 +33228 3473 1 +809 34260 7 +15948 38320 9 +52177 45027 9 +42202 16979 15 +37491 7116 3 +42987 6781 13 +2673 55653 9 +6751 28008 13 +8670 18083 7 +46396 3717 14 +20018 31420 2 +14535 59863 7 +52671 55280 12 +30719 13871 11 +42402 1722 3 +1083 16586 14 +6056 33281 4 +39963 47566 8 +49634 24570 10 +31278 38566 9 +17650 41468 9 +9775 29582 8 +8040 37507 6 +1500 15530 9 +58697 46466 11 +9357 8383 10 +42130 18310 16 +12913 15362 8 +19660 50713 6 +31994 16295 9 +17460 7855 3 +34215 47000 1 +19723 48493 12 +16790 59334 11 +12199 39612 6 +50310 51586 7 +7720 58257 4 +33570 53086 13 +14899 56880 8 +14048 44480 10 +40644 25712 10 +25658 37808 10 +9680 50577 4 +15743 12732 3 +16650 22880 9 +39427 40791 12 +16039 37709 17 +18827 37467 3 +30717 58509 12 +24640 40455 9 +56770 18899 5 +59184 56826 13 +8074 34941 13 +38403 43142 3 +28053 22697 11 +18518 36127 7 +35244 23390 2 +48462 38293 11 +43568 13448 8 +24621 46190 10 +25534 16474 4 +33268 55612 10 +18691 41499 11 +38464 20839 7 +55495 134 2 +23443 33761 11 +10134 56615 4 +40904 27700 5 +1893 20094 9 +18008 8678 5 +58052 56867 9 +2751 41890 9 +6277 17795 8 +11658 42683 6 +36224 7104 14 +44279 7746 5 +22010 15317 8 +6042 52102 15 +40846 3930 6 +52354 18170 7 +12545 5014 10 +1936 19860 10 +26708 25139 10 +51381 36300 11 +15213 44619 7 +22716 653 11 +44976 3084 14 +21997 39472 18 +52885 51504 8 +7251 30813 12 +56980 14778 7 +26078 3788 15 +8322 47592 12 +43160 42810 9 +38756 28421 12 +36505 5315 5 +18043 53023 16 +40465 48212 9 +5723 18451 8 +33288 15557 6 +2351 10286 14 +57724 17728 10 +2301 6215 5 +49852 21921 2 +45884 9912 10 +31815 21625 4 +23296 8856 13 +16669 50995 11 +42535 31748 5 +38579 21788 2 +12818 42874 6 +13299 1852 4 +12059 9868 12 +39180 35846 12 +21845 37597 7 +19155 7728 6 +52249 9321 11 +48196 44512 7 +59391 38045 7 +38593 29639 9 +58877 59603 13 +58294 43327 12 +25975 49472 15 +6510 6090 13 +23714 3019 5 +21251 53729 11 +57771 23852 4 +34296 57213 6 +39274 42274 10 +35674 45951 12 +27549 17241 3 +44397 46049 9 +15694 24393 0 +58183 32539 6 +26837 57555 16 +44305 39076 4 +50357 14682 1 +36257 26447 3 +58845 39225 10 +11013 28814 2 +42441 33180 5 +36184 19826 14 +33984 28376 11 +35355 9042 15 +41314 48248 7 +52949 33886 10 +59493 34112 11 +36001 36422 3 +53391 125 10 +44600 45616 14 +46891 43916 14 +11169 31039 11 +48056 48048 8 +20459 40656 10 +17909 30044 2 +8133 56341 11 +36967 13612 9 +24754 6470 14 +37508 44926 16 +57870 36821 14 +59507 14262 8 +2613 40734 12 +34922 20331 6 +8094 47423 10 +17893 43241 5 +22150 55283 11 +6725 13372 3 +37577 8387 6 +46405 6549 10 +18243 3229 9 +11498 40261 15 +56178 44243 9 +51218 33666 10 +37731 40046 11 +5800 37608 16 +41718 53997 16 +13817 1771 4 +36380 15706 7 +13431 6587 13 +18850 47487 9 +54226 59846 13 +48051 12893 11 +53715 29586 6 +6726 5974 9 +20567 32104 16 +17197 15439 9 +6741 5630 10 +14934 9439 9 +23573 46659 17 +16005 24301 17 +44340 50668 16 +1121 16627 5 +54686 57809 11 +47499 44573 9 +38597 2737 13 +17601 43358 15 +52066 59142 6 +49099 25031 7 +24123 48217 13 +26581 45387 3 +42795 20259 3 +17000 3470 5 +15681 20328 11 +25118 55673 5 +1196 27163 13 +30266 47468 9 +26830 29963 10 +36270 44920 15 +4049 34284 7 +25169 31920 5 +3009 25138 6 +3317 54891 14 +34169 50796 8 +25332 11674 7 +49025 54516 8 +21648 32401 7 +19215 31948 9 +26381 20016 8 +48317 30999 14 +45601 9676 10 +57152 36240 10 +7652 9693 5 +46446 22246 9 +37065 41138 3 +6604 46862 13 +11808 43253 17 +14273 28689 11 +51464 32289 7 +43598 50644 10 +40520 53010 12 +17665 7626 8 +17592 53727 6 +18983 24415 8 +13932 2958 17 +32996 1258 9 +18018 33052 14 +20389 9200 16 +53205 11167 8 +45847 14733 12 +15607 5271 10 +42978 17849 13 +26690 20875 3 +3611 41803 9 +34227 41771 18 +33444 27755 6 +32909 6223 17 +10558 23529 8 +41434 21843 16 +33125 3596 7 +13987 7706 6 +35455 33037 16 +17262 16104 3 +24719 29662 6 +38018 26668 14 +42621 5021 13 +41427 36891 10 +11044 57344 11 +53314 53256 1 +38642 1315 9 +22581 33865 6 +14426 27929 13 +54916 12575 8 +38704 7624 11 +658 12413 13 +18980 18056 5 +4119 28527 8 +57131 49249 6 +59323 23037 8 +50055 25860 7 +48611 27546 8 +31959 16462 11 +42105 47065 9 +6611 55715 3 +7501 11949 6 +58641 28006 12 +52641 9212 8 +43375 59748 13 +37500 28347 9 +41564 59842 15 +56662 59633 9 +39314 20634 11 +20914 36772 11 +45024 45874 13 +21720 54856 6 +33572 2096 12 +900 30298 13 +4104 11466 11 +34937 29426 10 +48049 15297 8 +27122 14409 11 +57636 1266 15 +15214 5212 13 +18692 12786 5 +37727 44072 10 +28184 59433 1 +53045 15758 10 +26642 7065 10 +37718 1736 12 +3331 19109 10 +8237 1403 7 +26272 3342 9 +2284 7761 10 +29155 4178 9 +35009 36195 7 +40 46657 9 +11691 15303 12 +40305 16571 7 +6145 17836 11 +21592 44612 11 +43897 43625 7 +38232 56381 7 +3521 33418 4 +42653 30581 12 +37793 11217 4 +53822 59223 7 +11102 20080 12 +15465 4094 10 +47558 18092 8 +12149 54672 11 +45551 36181 16 +13629 6118 6 +885 39001 16 +27914 453 4 +28995 7813 3 +24233 42101 9 +28447 9482 10 +3603 25646 6 +19588 49895 5 +39239 38773 9 +50909 51756 3 +57357 34907 9 +22780 51587 9 +59055 32518 8 +31885 8272 4 +39454 11160 9 +11309 39723 11 +34332 43595 10 +18572 10798 5 +12073 57752 10 +31794 42798 9 +31280 18515 1 +28152 58507 13 +13586 39246 9 +35598 19527 16 +53048 33640 3 +135 44395 5 +9361 12437 11 +6521 34585 2 +15841 40611 4 +11280 18404 14 +36083 40468 10 +14385 59739 15 +58204 20128 7 +41591 18508 6 +5568 10407 7 +12848 14519 15 +44077 21988 10 +49679 54936 13 +40900 5423 7 +28562 47216 8 +24132 11966 8 +15310 14452 12 +12646 18278 9 +17917 25914 7 +4502 7440 2 +32137 54253 10 +44589 54621 6 +35370 55298 6 +2487 33559 12 +8943 36064 6 +35470 29855 13 +19495 13786 10 +36362 42465 9 +16262 55219 7 +31747 39816 18 +46686 48637 12 +7655 28248 7 +48368 32999 7 +41233 24336 6 +58466 10386 6 +31150 29408 4 +50070 55674 8 +22140 57381 10 +4382 43406 5 +21296 3235 8 +42693 37862 3 +23914 42658 0 +45035 2432 1 +11619 38924 5 +18732 2680 12 +49916 2264 6 +9271 32223 5 +19757 13354 17 +19561 25268 5 +19712 31657 6 +55843 407 12 +51764 4322 7 +25075 16445 13 +44140 36470 7 +18058 59090 14 +6334 58851 11 +15591 11004 7 +56767 50826 8 +32928 48186 11 +40609 50086 12 +25239 5653 14 +35318 5434 9 +51062 34291 8 +43692 53990 12 +24856 52156 6 +31141 45968 17 +4933 19410 2 +10594 50195 7 +6673 19407 6 +45398 19408 9 +47329 32 9 +54404 52551 6 +19984 28570 11 +44259 19372 15 +14542 22403 8 +40667 5891 13 +8876 52232 14 +51777 48268 0 +13671 18496 10 +53458 12667 7 +4973 51718 14 +11233 52928 3 +38690 56504 12 +45711 1625 5 +50280 44427 14 +45184 54410 8 +46431 9564 12 +50 39941 6 +15535 8384 9 +11032 18023 8 +9570 59356 4 +25556 42620 12 +18812 29456 3 +31565 35964 13 +8350 29265 4 +29849 3072 5 +36814 44803 8 +22495 2393 5 +31279 1229 8 +19405 21629 9 +28436 53576 6 +26459 51934 4 +19742 37537 8 +17098 30921 10 +6428 12833 12 +2659 27284 8 +42572 14711 7 +43153 29629 10 +16565 24845 9 +28620 21049 7 +42192 44889 12 +51892 53511 9 +48673 20967 8 +21280 29631 12 +16753 21649 17 +49075 37595 8 +28228 24669 6 +40023 3417 10 +58839 5090 7 +28967 24689 6 +41433 28129 18 +30926 5062 9 +24853 33850 3 +48209 539 9 +11077 38789 4 +56702 33579 17 +32374 27875 10 +48766 44854 18 +39012 3248 1 +18145 27183 7 +3730 44819 13 +25843 41912 14 +12557 9410 12 +39947 24803 8 +39531 20576 7 +4414 48851 3 +36346 18730 8 +46775 22901 2 +21161 47223 10 +50127 33637 4 +56033 16133 12 +54807 48763 8 +25760 38735 7 +38303 9225 10 +32907 28876 14 +31260 55057 14 +38389 3356 12 +55037 10271 10 +4930 17754 9 +6542 24954 17 +17405 42739 15 +35294 1040 0 +9756 55815 11 +47697 6127 5 +33066 27603 0 +27205 40295 9 +22500 4998 3 +33496 7156 3 +15768 47847 11 +32464 21398 7 +35565 10850 5 +48782 12654 8 +40776 21098 8 +49576 130 3 +6463 15911 9 +19175 45611 9 +33990 32752 15 +53004 8792 7 +49551 2049 12 +33310 18966 8 +11426 8524 10 +59464 46115 11 +29846 18021 18 +29040 51454 5 +31009 39041 8 +56140 27835 3 +4644 55084 8 +17656 7670 14 +31752 15194 15 +30842 42881 6 +29981 13237 11 +22808 3300 10 +36770 2991 16 +2540 44942 11 +39267 13812 11 +8207 36475 6 +42434 22175 11 +31974 34545 10 +54204 52691 6 +46135 21427 9 +54582 28171 7 +6093 12028 12 +39652 39871 4 +7125 8033 15 +58578 42612 6 +37791 48520 9 +40431 7587 7 +41783 52711 17 +37403 45689 11 +10801 36256 5 +38497 22605 0 +48956 25049 2 +58938 10482 12 +52494 25732 8 +50806 45854 12 +4598 30489 11 +16265 24158 7 +24446 45101 8 +17582 34619 3 +39561 51247 3 +9769 8770 14 +50973 56682 11 +7428 28802 10 +28781 42568 9 +31328 50823 11 +19595 21287 12 +11678 40645 14 +30574 5957 4 +8192 26357 5 +37469 17611 2 +3224 53302 9 +50928 27296 6 +51064 10540 8 +45869 25771 15 +42384 3578 10 +47426 35375 11 +42898 45812 3 +40110 21163 12 +10065 40269 14 +25316 2117 14 +5170 44636 12 +34013 35555 16 +18062 35323 12 +15637 3996 1 +33552 32809 6 +34879 35348 6 +51115 56526 12 +56067 38222 8 +22181 24315 14 +18521 3977 17 +30986 35950 11 +56374 23840 8 +15210 45937 10 +40782 18039 5 +58610 16618 1 +41773 10034 6 +15453 11983 8 +13862 22517 10 +48746 43828 11 +41669 41616 1 +37394 35442 11 +23246 36893 13 +22633 10728 9 +50392 8665 14 +29266 53250 12 +8371 47012 10 +51063 51452 10 +7651 7194 10 +25559 49268 6 +6410 39539 8 +31195 42165 9 +46664 9030 5 +42386 30223 6 +48521 35241 9 +11413 19084 10 +8108 52099 2 +11172 34084 0 +19880 43420 7 +41486 16252 13 +23169 46144 5 +34720 11043 6 +38578 57561 14 +45831 30121 12 +58907 55995 8 +21139 20881 10 +34751 56026 15 +53053 42133 6 +55800 35302 5 +51748 384 9 +50347 44489 17 +55721 28133 1 +14018 4785 11 +52221 739 6 +20527 45365 7 +27329 54494 8 +16966 1368 7 +38160 29482 8 +15332 55622 3 +9044 21407 11 +16801 7912 10 +19078 46102 10 +17383 4398 9 +29334 6687 5 +16707 33915 6 +12441 986 7 +34611 6747 5 +56679 36606 16 +29468 40847 2 +11885 12607 17 +1004 43437 10 +49562 30897 12 +49835 19883 3 +35990 28437 8 +28192 6957 17 +34754 7532 2 +33456 47200 2 +11168 18655 13 +13860 51057 6 +56882 11944 8 +24692 412 11 +34043 38875 10 +25514 23980 8 +47662 28830 15 +52155 46831 13 +27511 32799 12 +29521 46036 5 +47605 40854 12 +8773 36645 6 +19449 46869 11 +41898 21334 2 +26569 34994 12 +42397 34203 11 +3980 6716 6 +19195 29148 3 +32467 26885 17 +42469 58946 6 +9669 43222 14 +50431 34267 10 +38652 14053 9 +19261 16346 6 +50287 17184 3 +25899 54322 4 +12779 11861 8 +3756 20340 4 +42992 59401 10 +17797 27134 6 +53057 15448 12 +2601 5933 3 +47604 21340 11 +1779 29712 11 +31529 43143 8 +42317 25097 13 +2854 53090 8 +16459 16639 12 +54827 51865 9 +43976 19014 3 +12113 52120 2 +35918 26461 8 +16857 14174 4 +7201 51167 9 +3318 36442 4 +55082 57814 12 +32937 29105 8 +25802 55589 5 +43133 33346 6 +14509 57043 5 +42965 30746 2 +4412 34582 1 +2288 36160 6 +57069 40856 9 +26325 26958 8 +25727 40117 14 +23408 46213 9 +5393 53818 6 +27800 53277 10 +1323 43796 13 +20919 54767 11 +51203 44180 15 +2615 2402 2 +12458 56434 7 +7565 7872 15 +14412 41651 2 +652 8325 9 +10210 12789 12 +26833 11126 13 +8136 41440 12 +59124 26535 8 +28878 9874 1 +44291 28789 13 +35603 45775 18 +36876 26220 16 +47262 31233 9 +19099 6129 9 +28653 20918 12 +32669 26294 12 +40703 43371 6 +42471 9480 5 +55737 2039 4 +28272 52210 5 +50159 44885 10 +9508 58095 9 +52437 58195 12 +16542 51790 7 +3108 59092 6 +51286 53179 6 +15881 33743 11 +7990 55619 14 +29807 28555 3 +39740 52131 5 +44755 2018 13 +9300 45273 6 +54480 3955 3 +22382 57188 12 +41251 40366 11 +33883 42406 9 +21794 59539 3 +22725 23982 11 +15425 4987 12 +43446 46310 7 +7695 20993 4 +48716 29763 3 +8844 10951 11 +18988 50658 7 +40122 27258 2 +24555 43905 2 +11586 14876 10 +30554 18064 4 +54513 7237 3 +11512 10075 6 +7291 17078 8 +12989 51749 10 +30487 45861 3 +10267 20365 2 +49603 6892 4 +28414 14549 5 +42000 20059 5 +46201 55363 14 +51853 31933 6 +25741 59452 8 +47191 20537 6 +38989 43894 11 +45738 16964 16 +9331 12868 10 +58028 38256 10 +26435 23200 9 +22029 12327 7 +25672 23183 7 +8206 18126 14 +13630 44832 8 +22078 25098 15 +32116 6048 11 +59579 54610 8 +31034 4665 13 +40031 55296 17 +38396 44328 11 +57244 51462 11 +16505 23381 14 +2053 51170 6 +51768 44882 12 +6991 54471 8 +25852 56135 18 +20561 19845 3 +13590 45801 4 +3830 43317 4 +51215 9525 1 +45172 25019 13 +30334 28365 9 +51226 12866 6 +10027 26922 3 +20447 25355 11 +27563 24312 16 +51677 16183 6 +48204 14765 11 +22624 36417 8 +43150 23975 6 +39609 28912 14 +55531 21360 10 +6597 3783 11 +12728 23805 14 +59831 2678 10 +16585 50917 4 +42747 44908 12 +48996 11974 9 +11179 13332 8 +32094 732 6 +18598 45632 18 +16558 15277 6 +18584 47157 6 +51893 57405 11 +46939 13834 11 +47247 19374 5 +39666 1325 10 +42271 56681 11 +13116 9378 10 +50559 41448 7 +45491 6646 10 +5606 21328 8 +46365 31605 14 +33590 31375 4 +37908 14086 7 +57579 57404 9 +27577 25786 8 +25467 59139 10 +42728 11726 17 +15824 18552 16 +23319 9942 3 +21565 6477 14 +47486 4231 12 +16915 4292 11 +33704 2182 11 +31535 52244 1 +15046 49663 9 +33653 47800 16 +43834 14825 3 +53307 15704 4 +19263 21344 12 +16640 51864 8 +27686 46604 4 +35496 15651 4 +8092 11130 8 +26472 40275 14 +5157 23217 14 +30257 56845 15 +32510 28954 4 +48937 34114 9 +22460 20950 7 +14507 42026 6 +55522 22424 10 +12240 36712 9 +58929 7361 8 +15687 36491 12 +15836 49442 11 +35969 4193 4 +3655 12551 5 +29099 257 11 +8197 3547 12 +32761 59861 12 +12349 52597 3 +14976 13614 10 +24170 33861 6 +36901 42107 10 +21700 19080 7 +53643 58421 10 +59008 29390 15 +36360 9730 12 +1949 49757 11 +27924 26692 4 +4252 57138 10 +4915 58320 3 +4242 49492 16 +25937 29805 1 +38734 50656 12 +37090 59841 13 +13555 50811 13 +56872 43009 12 +36165 55641 5 +201 56302 5 +44988 26410 7 +21607 49985 6 +7331 25434 8 +54129 15627 10 +54307 47069 9 +26255 1793 15 +10889 59296 8 +4774 5174 8 +43732 8749 6 +1366 17730 10 +25114 9136 5 +54316 8549 11 +33958 55006 4 +35715 39982 12 +41662 56031 6 +8938 54260 10 +3378 17191 6 +58019 24326 5 +3698 55513 5 +37855 41234 5 +18222 43777 9 +47135 2748 2 +13194 3785 18 +19160 10687 7 +6231 37658 5 +3397 11388 12 +35260 2736 7 +44103 46985 9 +23863 9163 14 +46193 8128 14 +19869 56007 8 +1218 59048 10 +33960 44757 13 +57756 26598 7 +54576 10559 9 +50219 41523 6 +13132 6065 4 +43687 59914 11 +6596 29250 4 +45005 47416 10 +3634 9931 9 +20859 17408 14 +41552 43315 4 +17471 48811 5 +11917 6330 1 +9222 40491 6 +40535 58616 14 +21955 19471 12 +25213 366 9 +38040 39227 11 +33800 57396 4 +39764 9012 9 +41859 39455 7 +7471 37050 6 +27629 58867 5 +38131 8605 7 +4234 50996 11 +20454 56085 10 +52169 34805 5 +37386 19012 2 +53252 19393 12 +15019 20053 11 +5959 9220 15 +10240 49102 4 +31490 12971 1 +14773 48370 6 +15938 8277 5 +22042 40190 6 +35133 5036 8 +21771 12494 9 +1732 55309 16 +2794 17657 7 +21679 43397 12 +42131 35467 8 +36929 32820 7 +13406 59696 2 +48753 37085 11 +6801 42054 11 +6403 23261 13 +19664 57656 6 +37974 28115 8 +56395 21423 13 +45984 26582 10 +25742 5652 13 +57276 16768 14 +7379 49579 14 +14328 15081 13 +58979 59122 11 +45236 26312 11 +31385 2719 8 +27653 43958 14 +29998 32480 15 +19705 7503 14 +40011 17622 17 +4465 8597 3 +16530 14415 7 +18715 42508 3 +33040 43555 13 +22126 33962 6 +11500 31526 6 +27049 625 10 +54440 25260 6 +10067 52041 4 +10589 12404 7 +42559 52356 9 +44559 51353 16 +12572 37264 9 +31829 10120 1 +40910 8624 9 +49660 50277 16 +38066 2912 8 +12532 46849 3 +30618 31161 9 +13361 57203 10 +28245 6446 1 +38219 43195 6 +26898 42973 11 +51574 37031 14 +2689 45490 9 +12966 45291 10 +27663 50758 9 +24351 43245 12 +29330 27417 11 +10168 10735 7 +2973 59638 16 +454 4146 5 +5780 20010 6 +38934 32110 13 +5509 1929 7 +11751 40438 9 +1738 53000 9 +1763 26706 9 +13773 42323 5 +3789 50939 11 +29281 34048 12 +54453 55597 11 +9474 14959 1 +8996 51269 10 +39790 53621 8 +8710 22493 6 +36449 52245 3 +6472 14332 14 +47903 55258 12 +48921 24622 11 +5696 29238 15 +31888 28337 16 +46178 31777 5 +12029 48632 15 +48526 7115 8 +4118 3575 14 +15572 56378 11 +36383 19009 6 +35212 49389 1 +28928 33169 12 +37452 12531 9 +45010 12466 14 +3163 27229 10 +36641 28962 12 +34989 895 3 +33051 50605 11 +20600 50554 10 +52798 19663 12 +54224 39432 14 +480 1959 8 +13425 49997 17 +40173 11227 4 +29158 32982 11 +15793 49995 8 +7929 17818 10 +59499 19930 2 +5457 10724 13 +55093 56278 4 +58501 39334 1 +15245 305 10 +3963 19132 3 +21343 45840 0 +451 33359 4 +37877 42374 9 +22887 3566 14 +36413 7665 16 +49053 27177 15 +34880 27385 9 +46583 44219 8 +37497 39801 0 +17279 8545 1 +7217 17029 10 +30624 54747 8 +13637 1239 15 +22797 10968 13 +49514 55421 8 +6007 3483 12 +17288 11951 6 +26004 10959 9 +55697 20604 9 +21067 23235 14 +18413 6924 10 +58651 21322 9 +43899 19571 8 +116 49069 16 +22347 5793 4 +53009 27369 12 +23880 25032 12 +49755 54976 10 +53876 50213 5 +7040 57732 10 +58583 33464 7 +24611 2873 6 +26941 18333 9 +8555 24587 17 +9552 38926 0 +22927 10069 9 +53542 3241 3 +1849 20260 13 +30270 51124 6 +22972 7977 10 +28042 32700 15 +56511 6744 9 +54027 58154 6 +36051 44478 5 +45766 23473 1 +47497 1633 10 +19321 40004 3 +32036 36737 0 +42228 32020 13 +38243 23378 7 +41172 48424 11 +11042 46206 12 +20040 19728 13 +41781 54128 5 +57984 1620 17 +33380 35644 11 +31125 53655 11 +25062 59885 13 +25093 30095 13 +27282 156 3 +14318 35941 14 +50690 16968 14 +7718 52769 11 +49467 54688 10 +40223 4346 7 +32207 34234 4 +28316 35612 3 +46531 19465 8 +52713 34714 3 +38741 5887 11 +35428 11968 9 +32310 59168 7 +23167 423 10 +5521 11072 8 +1161 30591 12 +6195 48601 5 +47462 3027 16 +52163 24884 13 +54951 14164 11 +49792 25308 9 +4316 13883 7 +10437 47865 13 +40803 15569 2 +6693 46844 12 +58098 121 2 +10522 12325 7 +16275 34070 8 +18699 884 12 +45284 27684 2 +20619 54303 7 +43714 19942 6 +25461 14664 8 +18015 14828 14 +55749 34668 11 +54816 44051 4 +14122 29343 10 +145 27102 9 +6345 54152 16 +12567 39502 11 +14245 36612 14 +9718 49207 7 +53518 8939 3 +34423 15551 5 +48215 46894 8 +8504 55893 8 +40425 32508 4 +48026 49957 5 +8365 10950 14 +31367 14856 3 +23909 12118 15 +21749 37828 6 +11209 19920 9 +13925 28705 6 +36705 16916 5 +43314 23276 6 +51513 52317 9 +23247 31476 6 +9948 48775 17 +26748 42089 15 +56323 16416 17 +32505 42243 4 +31710 57682 8 +13253 24843 14 +20813 13175 4 +46642 14548 10 +45909 51469 4 +47857 25605 12 +21244 8712 10 +38879 52101 5 +43781 21764 10 +9043 18055 1 +7244 23520 7 +22588 53824 8 +31877 30716 6 +52824 12931 9 +13747 7859 7 +46999 7747 6 +11417 49979 9 +36682 27588 5 +27374 42475 8 +34520 15575 7 +5313 44745 5 +21350 42745 6 +39480 58459 10 +27851 15375 11 +17291 53657 8 +11365 41136 11 +36122 54475 9 +46736 2545 9 +58177 51676 13 +23979 42236 8 +34780 5052 4 +30424 6030 8 +30602 59631 7 +17183 50667 8 +42008 36925 7 +39719 51444 7 +53052 36078 7 +40485 44252 16 +46638 51471 10 +4436 50905 11 +6280 1457 15 +57483 36990 12 +35823 11590 5 +9419 29285 6 +24104 29646 5 +25469 17284 13 +1089 4625 10 +35773 13676 10 +8367 55499 7 +53867 50738 12 +24003 41728 7 +48007 12237 17 +30406 55299 10 +23403 23216 4 +39140 14046 10 +56282 53242 9 +8479 21329 10 +49411 8974 8 +31623 40439 15 +9523 47056 9 +25425 23638 12 +37797 18751 7 +3604 21482 8 +58103 8800 5 +55835 16161 15 +4345 16352 9 +7153 43902 5 +37409 11290 12 +24767 37196 5 +7083 53424 4 +29470 52758 8 +26542 18345 7 +38319 41365 9 +17990 271 12 +37938 26207 5 +58358 29927 9 +39142 36254 14 +29898 31477 11 +21010 16528 8 +1865 43258 4 +46881 42186 7 +9338 39885 9 +45871 19093 8 +45494 18281 8 +5399 52799 3 +54251 31090 13 +18398 15150 4 +5025 52693 8 +30159 30973 10 +20194 34588 2 +29487 830 7 +47922 1203 6 +14578 59365 6 +38431 26977 12 +12956 48604 7 +56370 38532 6 +14334 31628 0 +36600 45339 6 +36545 13119 14 +13101 20342 11 +53055 13568 12 +44654 16617 17 +44043 49585 12 +57124 32131 5 +10289 4557 7 +18241 47431 13 +57871 39839 2 +25397 6354 9 +54737 3099 11 +5843 47256 7 +48666 39093 3 +8289 11572 5 +31135 11223 17 +32085 56818 5 +5 31511 3 +4818 41160 12 +42838 55440 12 +20476 24387 14 +4076 25897 12 +1112 34005 14 +31012 8572 8 +24769 6663 8 +48588 46059 12 +27073 11 7 +18012 10024 10 +599 11211 15 +40798 20673 9 +48228 4046 14 +48399 45390 9 +59801 22953 12 +37959 37839 4 +21278 42891 5 +282 8680 17 +30800 26630 2 +49648 25522 10 +49971 6415 3 +1646 38462 5 +19200 43677 6 +56512 48640 2 +12974 779 9 +33176 12693 2 +7468 54124 15 +26169 46623 11 +41916 10234 8 +8356 39676 16 +50209 9473 9 +57127 31438 8 +2619 9990 8 +2187 7114 12 +15785 47548 11 +22491 3302 11 +44650 10093 4 +49296 29192 14 +49695 8084 7 +10922 42350 7 +57471 27062 5 +1230 24377 7 +42251 28252 6 +56464 57426 12 +3559 44565 8 +9890 19142 6 +57181 12435 9 +51919 40038 9 +7810 7071 17 +17822 21919 9 +26980 38327 6 +14623 53 9 +42837 9008 9 +1469 9121 4 +2852 45343 16 +13741 12448 8 +27973 22906 11 +55748 50673 9 +16192 17667 8 +53072 37853 9 +44549 9727 4 +35052 12900 10 +17921 21509 18 +43653 1837 1 +19212 53567 14 +35162 249 3 +23143 2475 16 +17705 4421 15 +10767 42617 14 +2109 46360 5 +6122 55029 1 +58072 22201 9 +27485 13654 9 +56630 56365 6 +30438 38190 6 +12445 153 11 +35992 25402 7 +38147 36317 5 +10000 15421 8 +44225 44440 3 +42732 41716 4 +16734 46095 16 +19363 46334 7 +1064 5219 17 +6541 57243 7 +54114 20048 14 +5640 13535 5 +17398 55802 16 +6311 9493 5 +48893 52792 12 +33116 34078 8 +55123 5149 7 +36312 6675 7 +36128 26511 9 +51156 11932 15 +11404 45955 2 +30175 5388 6 +31912 7376 15 +53083 27885 4 +49560 25555 7 +48362 10938 6 +54534 38645 7 +45502 55138 7 +23016 49017 13 +10653 52119 6 +11225 11741 17 +31180 35303 1 +31728 16090 13 +32753 41707 10 +44409 27044 11 +18979 18837 7 +12560 3853 4 +6259 21413 7 +20929 8410 6 +28768 55198 9 +30668 47093 8 +48541 46619 7 +40138 2203 11 +29513 57021 17 +3535 14343 7 +11319 27441 7 +53129 16399 11 +28847 17123 8 +43419 27009 14 +34500 2533 14 +14366 11755 4 +12125 21130 8 +37086 11690 9 +55240 47264 18 +29323 3337 1 +21375 32853 15 +58662 35464 10 +8881 59191 15 +2218 48804 9 +14352 1529 6 +5718 40699 11 +33010 9548 13 +44329 24724 7 +50552 31142 17 +26467 4510 8 +42948 37094 14 +27562 57241 8 +8927 1400 12 +35401 35666 12 +57153 28538 11 +45918 14863 11 +48019 12719 10 +47808 10913 12 +38699 9083 6 +8683 49787 7 +50383 26412 8 +39039 9475 6 +20684 2546 13 +6575 38345 6 +40033 41579 3 +12813 46515 14 +38174 57201 3 +25858 42078 5 +23940 59088 7 +20645 23514 5 +12266 33616 7 +55141 43910 5 +5376 7974 12 +49089 17847 3 +16942 27798 12 +1969 13895 8 +4395 27443 10 +48969 33992 8 +42908 5395 4 +50661 51558 8 +23080 10040 3 +44301 14185 8 +56484 45572 5 +46970 42276 10 +16126 8166 6 +48425 21946 8 +20314 7238 8 +33972 36875 7 +57106 27595 11 +54628 21462 11 +34224 50774 9 +42114 46798 7 +27616 17246 10 +8625 22195 10 +42175 12708 10 +30102 12624 9 +42783 10523 0 +10138 53044 11 +59880 33875 8 +24847 8321 14 +2244 31048 8 +57421 28908 18 +51689 137 12 +40151 13878 6 +8209 22430 5 +50717 37896 6 +10784 27146 17 +49707 52815 8 +52290 7296 12 +5361 23544 9 +22912 38594 5 +54810 27408 5 +26319 53001 5 +26505 13816 3 +48852 1314 14 +28539 29173 9 +22548 22769 4 +43111 31396 12 +10102 15499 16 +48633 18249 12 +58788 49765 13 +49701 33534 8 +30449 12148 10 +23193 36883 13 +50755 13847 10 +37707 11633 15 +22045 44326 1 +22635 54848 4 +35109 59589 5 +52740 48477 12 +42663 16886 9 +59264 32039 12 +29504 42377 7 +30225 35483 3 +46867 4859 10 +18176 18130 9 +54304 48069 8 +49112 45120 18 +9731 30578 9 +9326 45074 14 +15947 52599 3 +17966 47386 3 +59788 657 4 +5889 50596 14 +37209 18074 4 +26486 6412 14 +12054 47188 16 +26066 48830 5 +15173 47700 14 +25151 685 9 +21477 37103 3 +20773 5935 15 +12081 25224 16 +14301 5814 9 +44744 5301 1 +18885 51119 11 +2776 26224 3 +29324 52406 16 +59519 40952 7 +46427 35511 6 +51625 15108 4 +42680 12109 8 +40719 5035 11 +17263 25343 11 +10962 40395 8 +53012 32188 5 +55831 56495 6 +52007 15642 6 +23398 16575 14 +11297 27035 1 +1081 55292 4 +39939 54531 5 +5490 5227 6 +325 15000 7 +55839 51498 10 +49084 11491 8 +25548 28517 6 +47879 35550 10 +52970 14713 10 +53599 8787 5 +51275 11574 2 +6207 52787 7 +6366 16189 8 +2213 56998 9 +47325 38014 15 +38955 11406 11 +1129 24859 7 +48279 58176 8 +5302 43061 13 +31949 29074 3 +6320 13307 1 +14095 8194 8 +27276 37153 1 +48925 29902 9 +16506 25394 7 +13320 57378 9 +50962 26234 9 +5534 21112 11 +2664 23961 4 +35935 11416 15 +44245 23621 8 +53462 46200 16 +31394 7427 9 +19675 5099 9 +44715 14577 9 +47481 51455 12 +7272 4500 0 +8966 7793 10 +39434 13750 5 +8748 42288 6 +8241 24154 7 +8156 42422 2 +53632 35757 0 +2494 38027 7 +35405 37505 6 +59997 26166 9 +27707 6830 6 +33994 48612 4 +53618 34988 8 +37426 21385 7 +7234 34174 7 +8421 30943 5 +23300 21331 14 +5141 34381 6 +40539 22166 10 +26206 53937 5 +26544 36151 8 +51951 38599 4 +57340 14460 8 +48536 30930 8 +50166 55796 8 +52035 40436 10 +3126 56194 13 +7451 39044 0 +28451 33948 9 +39026 33494 12 +25125 34888 12 +42985 47202 0 +10205 38823 2 +28439 57644 2 +33799 30392 7 +29199 26182 1 +29710 51686 12 +15094 3284 11 +48294 43647 10 +13382 44142 10 +56397 58514 4 +11892 13438 9 +55246 34317 7 +47712 40784 16 +9495 34802 8 +38896 1901 13 +34535 35622 9 +23715 9489 9 +5813 14967 5 +11199 36065 14 +11771 41720 10 +53442 52211 13 +17126 32361 8 +40787 29344 13 +26700 52546 14 +24076 38210 11 +32158 47575 3 +21250 24450 5 +43234 52457 12 +51862 25376 16 +526 15829 2 +10529 56586 14 +51968 49902 9 +1659 45344 14 +41386 1011 3 +20380 10510 14 +17213 47033 5 +52878 2712 10 +47539 40925 6 +42032 47401 12 +58005 46136 8 +57855 4407 2 +52664 36784 9 +58580 14858 2 +30014 46169 7 +9757 39440 8 +33794 46415 10 +36604 786 10 +36980 36221 14 +33135 44385 9 +59894 42872 15 +9686 5715 10 +37957 29758 8 +49087 57889 3 +44847 46006 3 +10144 38588 11 +10932 10656 12 +54875 30097 1 +24994 14921 13 +11397 9014 13 +21450 619 4 +58338 49827 15 +17278 28240 16 +18596 7576 6 +13545 39447 9 +37771 38689 13 +24151 23699 6 +39752 15567 7 +6600 34998 14 +16935 43533 14 +24011 31558 7 +31658 27168 9 +19655 33968 8 +40012 28342 16 +59634 26499 16 +33995 45457 14 +21867 40006 13 +40530 27716 9 +57543 23510 8 +30613 18830 5 +17463 53246 12 +24305 35423 9 +2681 22929 8 +38377 53857 3 +5686 30759 13 +55778 5708 10 +35691 33259 3 +4883 41756 3 +14816 39718 18 +28446 1476 12 +18437 18462 9 +25176 38238 7 +15009 19669 1 +27656 24590 9 +47563 51719 10 +37982 17094 11 +11325 33109 12 +21493 46553 4 +21064 24520 2 +14678 50927 17 +20951 21502 3 +46535 16392 1 +59489 2556 8 +22349 35403 3 +7171 25116 12 +32410 7668 10 +41177 42055 12 +22939 4733 8 +51765 30126 8 +2563 38555 7 +26008 19458 11 +45446 20508 16 +36154 52448 7 +48202 21255 6 +58006 28731 11 +14605 53191 15 +51775 44163 13 +54574 7824 10 +12684 30389 6 +4635 18874 15 +19647 19709 6 +26791 34705 9 +42938 56781 3 +52821 31615 5 +24254 8072 13 +58840 9598 11 +28107 57974 6 +5389 15299 12 +27301 17788 7 +20700 26210 9 +58674 47055 9 +214 20295 14 +9616 31951 5 +36445 10491 12 +55042 20575 10 +24572 40770 8 +34187 38560 7 +7181 46223 8 +22779 2749 7 +10581 44111 8 +38545 38165 6 +57957 19345 3 +22479 52436 11 +32839 51771 9 +19248 59896 4 +17918 41542 8 +47577 28411 9 +30031 7355 1 +12854 49784 7 +29229 53988 12 +26072 38133 8 +56269 58172 7 +1974 28672 10 +12279 50315 6 +3735 37694 10 +54591 20052 7 +11463 49730 13 +27569 43404 1 +32179 54695 8 +3586 15597 13 +1055 16184 11 +36368 5051 2 +13451 54182 5 +44439 43054 9 +28760 11763 10 +2094 23503 3 +4544 57147 15 +33220 21598 7 +27056 34979 12 +48542 27413 2 +32649 3277 16 +35476 44733 12 +20057 26613 8 +13232 10520 8 +25270 17269 8 +11098 36274 12 +15383 24582 2 +788 46861 15 +52037 51776 11 +44500 29806 14 +20179 56925 5 +58637 27901 13 +14325 41244 7 +39037 23470 1 +39731 49400 16 +41002 3653 16 +33778 15789 9 +43012 43191 5 +57917 25874 9 +37576 56390 13 +32071 44798 7 +59419 55178 5 +6788 40187 12 +4129 34869 2 +43996 35113 5 +48360 26996 11 +46632 22584 10 +42516 30687 15 +24656 9622 7 +43339 46350 4 +46245 48369 9 +34274 33601 11 +2484 45650 5 +55567 13746 8 +53726 10618 5 +5316 36106 10 +23304 52504 9 +55700 14077 16 +43599 54993 12 +4757 30764 11 +10506 33209 11 +10969 18961 11 +25964 29726 9 +7021 34456 14 +22004 27297 2 +56493 46434 3 +59283 1998 6 +2507 32954 13 +19122 26115 9 +32174 24091 13 +34634 49658 5 +50903 37675 13 +25573 18393 4 +51697 34049 12 +40728 49289 11 +14841 28564 16 +10821 26261 0 +44928 35384 14 +16371 33192 1 +52115 26383 11 +54264 58047 8 +52365 36406 9 +806 52550 11 +23831 37976 17 +44071 25281 11 +3354 59828 11 +6456 20998 12 +50273 21911 3 +52337 12997 11 +38229 5421 6 +29427 46573 11 +54340 34319 9 +21622 15855 8 +2666 31481 13 +30792 9614 13 +40296 34249 12 +697 41326 13 +4920 44846 1 +36924 55247 10 +23865 10927 16 +12182 3841 14 +38877 19604 9 +47516 51848 4 +14026 14953 4 +4249 6450 9 +39635 639 4 +37897 7485 1 +2561 22112 2 +6092 46106 16 +18576 33636 14 +29724 5963 10 +17395 22977 9 +55368 22532 16 +54623 19351 15 +47571 33611 12 +3964 48257 9 +57470 36650 12 +20262 42416 11 +40356 44930 11 +23927 55703 11 +41903 22218 12 +10145 31729 2 +57036 45077 8 +25518 8111 9 +50707 13655 16 +26877 2889 10 +56759 13547 2 +8774 13882 7 +30997 9009 7 +1799 40267 7 +7192 26400 12 +43787 31751 9 +20134 35122 10 +18563 18041 8 +8571 22413 10 +13168 6080 3 +36583 21617 13 +3119 36750 13 +28886 45952 5 +43072 23457 10 +38419 45050 17 +21076 40498 6 +7946 26545 9 +24937 42991 10 +38430 38299 8 +15928 2653 7 +45795 47189 3 +6447 1335 10 +29254 51788 14 +44085 34673 16 +30794 19131 10 +57122 43744 12 +23569 16309 8 +14776 13958 7 +5102 6847 3 +32579 42672 9 +14913 30247 16 +31931 37899 10 +18452 46630 8 +46260 51656 15 +7699 32933 8 +5770 45408 6 +7050 10314 3 +15801 24468 16 +35819 55327 7 +57800 45569 14 +45211 43792 14 +53840 28351 14 +10254 33898 8 +11828 4247 11 +52838 18551 8 +12723 39703 4 +42622 52622 3 +1100 12750 9 +6135 48962 5 +26500 38906 11 +5980 7505 12 +11792 2692 1 +6896 48469 13 +43206 50136 6 +56400 53852 11 +51560 40758 15 +1653 53357 7 +1010 12085 10 +46017 27609 0 +48070 36364 6 +33913 16946 8 +41638 7219 1 +22688 32502 10 +30404 37033 7 +204 119 6 +39212 52644 13 +20803 4518 3 +15521 33406 11 +49426 20392 6 +57836 20679 9 +57867 44431 8 +34313 3243 16 +29090 49657 8 +22604 59674 12 +35893 38472 15 +10480 12964 15 +45282 57886 8 +45271 58386 3 +31700 59479 11 +29400 4057 14 +5787 20039 10 +52732 44221 15 +10454 41308 9 +38591 54542 8 +31773 37751 8 +45198 17810 7 +58654 25043 11 +56482 54877 9 +41816 42507 5 +44224 9722 6 +34876 36899 12 +58119 53183 12 +44088 2716 5 +44223 42547 13 +47986 27717 8 +58310 39859 7 +52830 8849 15 +28715 42410 16 +45102 22174 16 +9116 59278 9 +31855 11986 4 +10442 24494 15 +46112 22141 12 +50461 56356 14 +33729 49885 8 +55428 45354 5 +42986 43455 9 +6192 25970 2 +10582 3272 3 +10761 26493 14 +11887 28630 2 +7278 3502 15 +45814 26917 10 +13791 26549 16 +54011 44289 15 +5130 17617 16 +55497 6985 9 +49056 52909 11 +50901 12162 11 +30832 6275 10 +32392 29019 3 +55390 24639 13 +40850 41390 5 +2525 47741 5 +40522 42785 15 +48995 59251 9 +15454 58091 1 +39499 29469 8 +34658 16742 7 +54509 31425 9 +24699 36141 8 +15737 24422 15 +36707 53816 4 +20790 36511 6 +13082 32182 6 +30603 49244 9 +59753 24280 11 +59074 35488 1 +18285 11226 9 +30484 27751 1 +9714 25945 5 +45516 30039 5 +20531 18785 3 +15179 55952 6 +18133 44128 10 +46744 37225 8 +21336 29525 14 +22617 59431 14 +6255 18411 6 +57783 35458 9 +59826 51086 7 +12508 43218 6 +45060 50233 14 +20870 51906 9 +41249 21953 1 +44368 36919 1 +32563 18474 4 +28516 15624 9 +16315 13828 11 +43046 39404 11 +30861 44900 4 +37553 28373 13 +55505 28336 3 +27254 25172 10 +47147 21234 12 +678 47446 2 +17162 41442 14 +52931 11567 7 +58542 57926 14 +49189 21215 2 +22014 28251 9 +2954 47521 10 +2324 59373 7 +2327 48961 4 +52394 23 4 +34550 16619 4 +22240 14560 7 +39799 56241 5 +47954 26602 13 +39435 2354 6 +18849 50024 6 +17505 17556 7 +12686 33833 4 +53710 13317 7 +3641 26817 9 +38191 17411 7 +34849 23892 2 +21785 12241 9 +35022 41831 11 +56784 24016 10 +26148 3928 11 +35297 36692 6 +45454 28772 8 +5932 9654 9 +21533 8293 2 +39465 14844 10 +14490 186 8 +57748 21605 13 +59040 39378 10 +26593 42045 3 +51921 26840 4 +4746 57230 11 +31599 58125 15 +28096 12962 9 +17649 8416 16 +9590 35623 10 +49531 37046 16 +34653 18072 1 +19383 12832 16 +4917 2808 7 +989 34255 8 +12595 56239 4 +9015 53910 8 +19659 8082 7 +33279 6272 10 +12498 11033 5 +44723 39187 8 +31869 30742 2 +6613 16450 5 +45415 42367 18 +57770 36144 12 +37904 33044 7 +48371 37685 8 +22897 10734 3 +10838 25909 10 +42525 44411 6 +15355 52488 10 +52016 192 2 +49078 19390 9 +55286 13993 4 +51420 41471 10 +237 44257 11 +32953 42125 7 +8098 3608 4 +58900 31455 11 +34386 11913 11 +34432 56736 8 +22441 45222 5 +10023 19716 17 +14567 58269 7 +5285 714 2 +7722 3281 12 +46461 21412 8 +39825 11546 14 +3948 10242 7 +24758 19193 11 +16773 19957 12 +57521 39336 7 +34763 58639 17 +53574 19197 14 +4535 52005 6 +3414 55370 18 +41647 43266 0 +22514 41227 18 +11997 30589 4 +35896 18350 9 +44841 56144 8 +19213 56850 9 +22062 49475 4 +12079 28264 5 +58434 43333 9 +49982 27733 12 +31913 59597 5 +17453 747 6 +35015 9265 12 +19042 8511 16 +10596 49237 12 +18857 51920 13 +26267 4676 4 +49770 15416 4 +8069 38953 9 +19244 55540 11 +2149 29826 10 +59164 4440 17 +3680 33661 2 +49323 46020 17 +48739 57739 11 +11603 46993 13 +17545 48629 9 +35192 42644 1 +6304 4425 16 +45503 636 7 +30199 49246 7 +40983 1907 1 +54903 35208 7 +11697 47265 12 +3453 15853 7 +38476 32784 16 +58286 11285 4 +55405 43922 11 +18833 8716 4 +49420 53953 9 +14154 11138 4 +54526 31157 6 +13098 28861 3 +9529 29845 8 +8791 50571 15 +4928 21158 16 +22402 11787 10 +35194 36468 9 +12048 54431 11 +36522 49932 3 +52296 17596 16 +18942 41372 2 +35772 12087 13 +33094 23614 3 +22774 50429 12 +50990 40424 4 +16350 14378 11 +26356 57659 7 +4889 42578 1 +19203 15903 1 +52152 35092 4 +30332 59043 6 +50598 4321 6 +39479 22933 12 +50652 44639 3 +39078 31618 9 +56655 56601 9 +46367 44769 13 +12622 54857 10 +59877 36789 6 +57379 57008 17 +19993 45104 12 +48278 31921 11 +17131 26757 4 +27593 41153 13 +20061 20020 15 +1027 11753 17 +33375 36856 16 +1662 55553 9 +45948 56410 15 +34096 10664 1 +15064 59939 9 +52261 39410 12 +22784 4754 17 +55237 58027 13 +13589 29183 8 +45825 21138 11 +28735 56652 10 +18431 52570 14 +31447 1191 8 +55726 9198 10 +11034 6561 6 +33295 36185 2 +4155 46114 6 +26584 26390 7 +50237 19318 6 +38370 8291 5 +17880 56024 7 +38063 7216 13 +48595 39586 11 +43449 49280 2 +4239 37968 0 +8163 11889 8 +33473 29164 12 +9501 43614 12 +28857 56169 2 +30944 18664 10 +52240 57667 12 +39017 37519 5 +37568 33979 10 +43975 43514 4 +21090 34443 3 +32984 31982 8 +30616 20049 1 +29213 12969 13 +55907 43942 3 +22409 38819 11 +58748 45672 10 +31929 42081 5 +32154 42484 9 +19580 22336 14 +42734 16074 5 +7250 3141 8 +56155 46147 12 +44510 2753 7 +7869 38946 7 +48888 55190 12 +18253 16799 9 +33567 113 2 +44012 43474 4 +23229 57031 10 +10884 44032 8 +14635 29982 10 +14091 27668 7 +11693 29510 5 +43424 22772 15 +39868 25324 8 +41 22537 12 +10213 34016 6 +6523 37360 10 +18124 19299 4 +37431 31591 16 +51034 47374 14 +7596 34015 8 +43069 18844 15 +23642 26076 9 +58298 40586 3 +33701 52556 18 +9385 26888 6 +45911 6993 12 +1638 43650 14 +13968 21852 6 +8260 9397 10 +3138 59608 8 +31926 37265 14 +54982 49106 2 +43152 34825 11 +40207 34943 5 +23441 22788 3 +17444 19442 7 +35863 39867 11 +33119 57446 6 +52453 31295 11 +52100 18632 8 +10862 25972 15 +30707 57322 1 +268 56015 8 +36149 18571 7 +13485 19364 13 +58634 43738 13 +41954 33841 2 +27476 40840 9 +44530 55945 11 +13077 48643 2 +47830 11539 4 +20787 10173 6 +26705 43228 11 +35912 37829 7 +25889 47060 11 +18166 30090 12 +31733 53032 13 +46420 47824 12 +10303 720 12 +6625 50671 12 +29603 5747 1 +17933 8922 8 +21127 34262 1 +24500 57979 8 +53429 36464 9 +46950 33602 12 +34208 23288 11 +5167 58214 7 +56706 29473 11 +12598 43170 2 +17312 14827 14 +3421 35809 14 +12453 37663 7 +24299 43183 12 +44448 44737 7 +53708 44255 8 +2336 13711 2 +3443 31919 9 +20426 52255 8 +48079 44078 7 +7874 39570 0 +2231 42860 17 +48652 13944 16 +56859 59396 11 +667 10890 7 +43116 48504 11 +30441 33557 9 +42647 26709 13 +44536 28084 13 +29736 13100 2 +28106 41325 14 +35203 50099 16 +16544 58033 3 +26296 30353 6 +56360 32041 6 +19137 45956 16 +24271 52181 10 +12335 59112 9 +16909 10573 9 +56254 34231 15 +35910 34415 13 +52144 8796 6 +20469 47518 10 +41881 15710 3 +5737 12331 10 +47878 4469 11 +13424 31822 17 +56848 30387 8 +38067 21828 12 +44553 8917 9 +34196 24189 9 +34731 26233 9 +19252 52134 7 +48351 6607 8 +13686 57271 11 +56639 14704 3 +8349 40723 4 +49729 30345 9 +14157 21848 0 +53224 33870 9 +15118 49586 8 +42927 46877 5 +38863 3702 8 +20163 39754 10 +10685 18701 14 +45875 12197 9 +52524 40868 12 +58026 36117 3 +38083 52368 4 +2877 55400 2 +22375 29133 1 +28558 29713 7 +22703 38748 17 +10797 57055 13 +8279 15715 5 +51173 24848 4 +40228 43682 6 +12213 46886 6 +35235 20533 2 +15057 40181 3 +23583 935 12 +59765 55313 18 +36828 15738 16 +59120 37159 9 +43939 44473 1 +17758 53099 17 +14107 27427 3 +6662 5045 12 +24851 2367 13 +7036 36426 5 +49047 7701 9 +29610 43097 11 +21927 50547 9 +25205 18987 9 +37199 18262 8 +5182 59974 9 +53340 24854 7 +45959 10056 8 +11191 56835 8 +54544 27228 6 +18673 5211 10 +38235 34002 6 +43271 9087 10 +37427 26772 3 +44094 36730 7 +59318 47126 12 +18968 37388 9 +6860 10598 7 +26707 9705 8 +23897 13056 10 +9779 49487 11 +31075 58546 8 +16885 22117 10 +30565 50549 5 +31796 53865 15 +19900 32315 1 +56937 47282 10 +16573 49340 7 +31128 23533 15 +40343 8955 12 +32983 11451 9 +56104 3155 3 +41559 37263 9 +59790 16730 13 +29731 40842 13 +53875 36574 9 +4159 22348 7 +15633 50989 4 +17579 28832 4 +43280 41041 3 +37152 37430 17 +6439 12592 12 +44394 26109 10 +29942 5345 6 +25780 26689 4 +55257 16108 5 +7220 53068 12 +50781 43541 13 +39538 9157 16 +22502 22437 10 +4982 5868 17 +21093 41666 11 +16606 9428 9 +41122 17561 4 +31584 47753 13 +952 2482 8 +36549 15270 2 +30106 19227 7 +8041 23912 9 +3213 50451 5 +51349 11562 5 +24335 22512 6 +7365 54830 15 +20394 49561 1 +42219 42429 4 +22238 44708 10 +8978 48123 6 +16379 42018 11 +45557 23666 7 +22535 32364 11 +982 24421 6 +44572 41749 11 +52951 37289 7 +51617 32004 10 +23248 17559 15 +5023 31256 10 +31342 12070 10 +18372 2683 9 +2161 20448 9 +52483 4863 15 +15389 43938 9 +29613 30055 5 +44888 41001 8 +39915 21935 12 +52318 8250 0 +11165 39369 10 +11220 20541 13 +15892 59620 9 +22186 31378 9 +25575 355 5 +58051 14574 1 +41982 37988 13 +51006 56688 1 +16334 51378 7 +54757 46426 15 +45377 17674 12 +18408 34538 7 +707 51018 13 +28262 9054 11 +6951 57062 9 +57009 12935 7 +43782 30433 10 +17059 447 7 +39761 25304 3 +45510 19825 11 +37615 29892 7 +48109 43352 9 +51830 12738 17 +37592 38631 9 +51885 46887 9 +41418 24881 3 +31227 37359 13 +52213 30120 7 +28788 2370 17 +21062 31537 8 +46484 35885 2 +59049 31383 15 +30940 46896 1 +1878 34758 17 +32411 46851 8 +10873 34470 8 +309 11234 4 +44822 38077 10 +34099 13497 0 +8575 56886 5 +16083 35607 6 +43032 56422 6 +59567 59532 15 +23867 32383 9 +19201 7024 13 +15857 55686 4 +24289 10076 7 +31332 58359 6 +26216 23693 8 +6705 43219 10 +54784 3371 10 +14013 36467 8 +44992 15547 12 +54753 59466 9 +56084 15059 8 +28242 42594 7 +6887 8637 11 +51785 9720 8 +53663 1092 14 +1749 2571 13 +45867 21726 5 +31200 51834 9 +20364 31843 9 +28024 27614 2 +17832 25662 7 +5982 11294 11 +41563 20344 12 +30168 30287 10 +35472 40993 9 +21258 23158 12 +52648 53264 3 +14074 1795 10 +45606 39717 8 +56809 24067 8 +21674 9414 7 +4445 45246 4 +42897 16079 14 +35183 7821 5 +32501 36950 7 +8935 31021 3 +53321 51520 1 +13913 49998 4 +2735 31654 9 +928 36746 13 +51097 6601 8 +16860 54548 17 +58853 44146 6 +10094 59133 14 +54061 49214 1 +55162 332 11 +30074 59203 6 +26283 26895 14 +3402 59313 3 +29568 54955 15 +16444 55383 12 +11200 44954 8 +9562 29626 3 +7465 21563 5 +51417 36767 8 +24877 18381 6 +36520 20688 1 +17883 4364 7 +16006 10232 18 +5394 17908 11 +16410 44357 15 +41763 49403 15 +55662 37838 4 +35544 44453 8 +10500 39508 7 +26926 45425 11 +15226 4071 5 +51008 14718 15 +13363 33803 8 +25359 26337 12 +37324 55416 12 +40753 59882 10 +56976 31789 10 +13016 39082 15 +21159 57798 7 +1894 34897 8 +8801 29382 14 +34679 25759 5 +49497 42151 6 +16044 57158 14 +49868 11779 3 +2937 16748 4 +25411 45149 7 +24319 33190 10 +27836 22247 2 +17615 19605 15 +58224 49977 12 +35836 52161 9 +43773 25754 3 +28791 9424 10 +2171 8941 5 +46218 20035 14 +56114 11905 7 +48728 27982 4 +56039 15264 9 +9292 47650 3 +16863 36620 4 +27808 38172 13 +53478 45071 16 +6937 45323 11 +13946 13013 5 +3514 17447 9 +49067 12701 7 +57567 23118 10 +32623 30754 9 +49558 1343 12 +25798 17978 15 +34765 30550 7 +44574 36009 9 +37081 12851 7 +34367 57955 12 +7526 16489 7 +51673 47581 2 +13010 29224 9 +11195 41162 15 +51528 27786 9 +6738 20348 1 +38507 40375 6 +17509 33741 4 +27151 57209 11 +21289 45739 14 +45108 1309 9 +48721 698 8 +11778 37130 15 +15388 3392 9 +22092 49870 12 +40996 48985 8 +57805 40662 8 +48924 46624 13 +15164 49176 9 +19071 22519 8 +15667 39987 9 +17697 38368 5 +1565 3812 7 +16009 56663 9 +44038 30308 10 +9546 23174 6 +17201 52443 12 +21597 20208 6 +9653 40979 7 +30872 38140 4 +45836 41936 5 +31120 8178 14 +40893 28348 16 +16294 37319 10 +38035 23907 6 +18088 45565 8 +16136 51046 4 +56488 43088 3 +21448 20611 5 +37656 55364 9 +39724 22749 9 +2022 52937 9 +16892 36073 7 +38513 9658 7 +58855 13757 12 +52679 29760 8 +51234 42869 11 +10875 42984 14 +52791 38361 10 +17206 22860 7 +59727 54826 8 +49213 23766 15 +3882 59340 8 +8949 51704 14 +35512 8105 13 +35520 2793 12 +47100 29107 13 +55741 56182 14 +58711 46361 10 +36003 29423 14 +15445 25468 5 +55018 23628 12 +43052 28465 12 +31496 14130 7 +16692 55404 7 +18366 49324 8 +2345 31130 15 +58810 41217 3 +29272 6630 6 +23592 15422 11 +5246 36488 3 +57887 15085 5 +36970 57996 11 +31299 34293 12 +43409 21519 10 +51863 11117 15 +52969 9299 8 +25680 4392 12 +51457 56864 8 +54937 1348 10 +36010 5265 12 +47166 24672 3 +21208 19198 7 +56567 36866 13 +39393 20183 5 +9585 6050 11 +56983 13218 10 +50782 14537 9 +37062 17538 10 +58295 30560 13 +19713 23539 12 +22366 25992 9 +5408 55539 12 +27860 24413 11 +49541 43374 16 +10756 33386 11 +45607 51824 5 +44957 10311 11 +48188 20523 12 +19108 32540 6 +24469 27193 8 +59402 5526 13 +38261 1968 11 +30237 55353 12 +59382 59343 9 +27365 57499 8 +57428 27756 16 +45673 24579 13 +15883 42359 5 +35873 7009 6 +58619 9604 4 +12243 32062 2 +16114 18703 8 +10860 28511 17 +10044 20212 7 +59743 10907 5 +2870 40908 2 +46600 36493 8 +47811 40466 16 +16721 31644 12 +42492 24844 11 +59456 23605 10 +22146 29290 4 +14329 54238 2 +36340 32973 9 +45412 35880 3 +56787 4096 9 +33976 18554 18 +13207 9276 13 +35298 23585 3 +28507 37312 14 +28370 32160 2 +49427 46129 7 +46540 35083 10 +56564 755 13 +12023 36714 12 +52456 1513 9 +3933 12778 15 +3902 230 9 +30336 5835 6 +25922 54062 6 +31387 9010 13 +50941 47829 13 +7131 4631 7 +21100 46086 11 +48732 44827 6 +10026 33241 7 +15840 36050 13 +42170 10241 4 +48275 18697 16 +44210 12746 2 +10888 12128 5 +31011 16797 11 +17789 33954 12 +30937 55883 12 +53813 11201 10 +17093 53551 10 +13391 22783 12 +29351 49252 8 +49354 33144 9 +30413 37219 8 +49601 30784 6 +39152 10071 1 +28441 52325 4 +36847 56997 13 +8244 33093 16 +59773 26698 2 +51837 9400 6 +46824 1477 9 +44541 14647 11 +30912 40133 12 +347 36025 6 +29768 16102 12 +23042 59949 17 +11638 37514 7 +40918 14212 2 +23677 10222 10 +38097 13729 1 +20625 22544 4 +36998 12663 10 +87 45864 18 +28005 24822 13 +28014 39796 8 +16516 41995 6 +19766 57818 14 +37048 24117 14 +9366 15450 11 +15411 25781 9 +11992 11046 13 +43622 13308 13 +50851 50227 16 +27464 28930 9 +56609 15485 7 +1753 12662 2 +32431 38150 10 +32819 28733 13 +25155 8696 11 +16046 37280 5 +41670 49968 8 +5644 8170 8 +54702 37294 13 +16853 43898 16 +17781 57785 4 +51268 14359 4 +31190 2276 9 +12790 57331 6 +31390 48356 10 +26935 22483 16 +9288 53182 4 +52614 7756 10 +5541 26617 1 +43747 1587 8 +5583 19247 3 +35984 17212 5 +42834 27576 6 +56617 36109 17 +4117 7495 10 +49692 9311 13 +32224 30967 9 +33265 12345 7 +58473 39062 13 +9673 53739 13 +46542 27428 7 +22830 16428 6 +1820 59309 17 +58043 55742 12 +27216 35252 6 +30052 36602 9 +8769 50846 8 +24887 51400 11 +50810 32231 7 +15858 59833 11 +42283 13838 13 +10989 57703 14 +22434 24585 10 +38456 52566 13 +42780 51095 16 +37582 41793 11 +51801 27999 9 +7197 20934 7 +16963 49939 6 +42540 45092 9 +19186 58540 10 +3366 4849 6 +55463 4375 13 +43056 16433 11 +25651 8668 6 +21014 46591 4 +39296 59325 11 +35996 48896 10 +51642 58986 6 +55573 37003 15 +58775 5207 5 +27619 46747 2 +34000 11817 9 +33341 43057 15 +46225 22410 2 +20607 59956 13 +25340 13543 9 +18501 29342 12 +42698 11722 13 +51256 34696 10 +7976 29072 12 +819 22153 8 +43070 39965 8 +58608 13276 14 +5329 49518 6 +54966 41151 14 +5298 43791 1 +55710 722 6 +46852 23700 7 +52263 38836 13 +58363 43499 18 +9458 17416 6 +20433 54072 13 +30271 2970 8 +19937 45305 6 +34298 30714 12 +2786 24684 12 +33194 4294 4 +41265 24505 7 +16232 57287 9 +16377 25165 14 +34027 403 9 +56322 56394 10 +15130 29765 3 +49332 15507 9 +19259 5621 10 +7019 43570 5 +32187 23414 9 +58933 43755 16 +45463 43296 9 +5257 26038 8 +58170 53204 9 +22507 36145 9 +53719 21994 11 +17146 23480 12 +44144 39123 12 +13999 5401 10 +53486 10700 8 +6729 39306 4 +3946 38710 8 +6915 22109 14 +56896 59554 16 +59797 5658 11 +2940 3145 9 +26061 44364 10 +55303 34202 6 +8764 54606 15 +45465 32462 10 +23620 6814 13 +41784 36941 5 +12553 33365 5 +21750 3818 11 +56973 53383 3 +35915 17443 12 +30954 55238 11 +38917 33377 10 +51876 43344 2 +18374 54711 9 +32430 56489 2 +35695 27770 11 +52636 33745 6 +50793 50637 6 +10335 36263 9 +57067 2929 7 +30993 32723 2 +6361 7703 11 +20837 53678 5 +41238 29335 9 +17265 43451 7 +16072 32935 13 +59806 54096 8 +58848 58799 12 +20027 2163 7 +43841 30429 9 +32271 19832 18 +45915 8895 5 +18048 57518 13 +44867 33591 2 +38788 57838 9 +31934 5538 18 +7242 22553 6 +20682 7759 12 +892 34120 9 +47288 57140 12 +46691 32126 8 +985 2305 7 +36175 15437 9 +4664 19928 10 +34173 18091 10 +26982 20835 11 +18094 49901 14 +1660 49418 8 +17057 39528 10 +22931 58085 6 +48699 41185 7 +29680 23305 8 +3260 49173 10 +31972 1152 8 +10621 21736 9 +14784 26879 10 +54549 30870 8 +30151 43015 8 +42808 51198 16 +11902 32203 3 +2665 37734 6 +3179 27325 11 +32443 16155 13 +25800 41239 10 +31741 46202 13 +46505 21430 12 +21374 31848 5 +37070 27379 12 +41932 8011 13 +17496 4601 8 +13434 27244 8 +6343 51873 10 +21019 41484 3 +15831 10632 14 +28137 38455 4 +25248 45442 9 +635 48891 13 +22057 7246 12 +52948 58879 9 +4460 37131 11 +24072 2848 7 +36495 41119 6 +48828 39589 14 +54375 52027 5 +52609 25696 1 +5215 47413 12 +28455 38781 12 +25713 33097 14 +36022 3545 1 +21792 17469 8 +18435 48446 4 +22954 5082 0 +23889 19342 4 +19772 693 2 +10532 36314 10 +4112 28848 13 +29868 23437 17 +33136 3288 6 +42776 29494 8 +45293 13930 10 +25848 3268 10 +11229 1171 5 +49256 34568 11 +58223 48286 4 +36173 5348 13 +29612 47981 6 +44912 27129 11 +25045 16396 1 +51752 24787 10 +44370 537 6 +39961 57812 8 +45023 23054 13 +23600 2723 12 +10083 36303 5 +16056 15666 15 +58213 29652 16 +45315 29165 9 +54783 41412 0 +1580 34859 10 +50829 3660 3 +39394 5591 11 +13453 38990 4 +12151 56757 4 +42522 10304 8 +5731 51028 5 +23065 51736 10 +46303 41420 2 +45921 57882 1 +29640 48339 10 +40184 9490 11 +26913 31702 10 +43237 59750 10 +15090 25674 11 +38559 41516 2 +42598 24561 9 +468 19875 11 +36839 28081 9 +32675 1379 5 +16305 26033 3 +23674 39847 15 +56914 48307 5 +59367 47043 6 +10011 9417 9 +10915 6381 8 +40775 54127 9 +50509 51486 9 +39497 3879 9 +24894 20014 12 +4795 9344 8 +14591 22268 10 +25676 57139 2 +47909 48437 15 +54970 716 7 +7112 49593 11 +42445 463 8 +30584 54722 11 +1156 50085 9 +5311 17175 9 +15653 15349 18 +14354 24528 4 +23254 27523 7 +56247 28917 8 +7390 11371 6 +12834 48327 10 +36229 40440 11 +28868 4328 5 +45933 5377 7 +54871 42006 15 +5270 9444 10 +30987 3322 18 +8131 38928 2 +26094 49051 13 +37367 39631 2 +40930 1192 6 +44165 17100 8 +26259 33529 12 +50524 35970 10 +7248 16448 8 +33412 39913 10 +23566 35170 9 +52615 28695 2 +48656 28078 8 +22274 28478 13 +42398 8838 3 +12424 40279 4 +33525 16612 11 +44898 80 11 +20728 33338 11 +56961 49388 11 +58963 22862 8 +20170 21455 14 +20398 25153 8 +16320 46769 9 +18615 46128 7 +2631 3309 6 +8872 37593 13 +15522 35656 12 +14960 53524 6 +9893 42144 18 +46655 1666 17 +22452 38527 3 +38923 45638 9 +57878 12658 10 +58828 14217 18 +37618 5903 2 +26219 7454 1 +39218 48676 4 +29938 36132 12 +47375 48909 9 +21618 56267 5 +11003 44287 6 +5975 15862 13 +59289 52995 1 +42375 21522 5 +41921 34910 12 +12315 32272 13 +12414 28280 5 +19021 21211 14 +13441 39101 8 +44082 53120 11 +1336 35761 4 +50669 5788 5 +8155 55729 14 +43790 49602 4 +12011 41605 9 +21047 44945 17 +13197 26855 4 +34461 31613 9 +19423 2777 8 +2824 56821 4 +41107 1109 8 +5623 19303 4 +48385 26767 7 +3480 22056 10 +6094 30700 0 +16668 808 9 +5158 53273 6 +45704 25368 9 +32425 13789 11 +1693 49948 7 +43740 31501 8 +49395 30866 2 +41125 43705 5 +59064 56168 7 +37985 55600 11 +44947 18619 17 +13075 31763 5 +47652 42839 4 +56049 13107 3 +52930 44714 8 +7193 18070 6 +33434 24412 13 +37285 34225 9 +317 42179 3 +44186 24821 11 +25018 12771 9 +41931 33951 10 +7200 46546 11 +39313 8306 14 +45011 35075 9 +42185 24715 0 +2077 45230 8 +59694 44450 10 +24873 4420 8 +16556 25195 11 +56844 22618 13 +54670 35101 12 +16796 44849 13 +41248 25465 8 +55698 7841 6 +37542 55920 7 +55493 21338 14 +29931 3307 9 +36914 34059 8 +38010 41155 11 +58768 2054 7 +23777 58921 11 +36958 14060 3 +18421 55861 7 +2602 41574 10 +24614 57911 7 +17060 56329 15 +4729 19267 8 +32282 33680 9 +38537 44406 12 +405 48409 9 +51667 49674 11 +36533 51706 3 +34786 39388 15 +54438 38127 3 +13985 50612 7 +34932 20409 6 +18347 9630 14 +11924 35378 11 +23055 59838 12 +7241 52048 9 +52265 6370 11 +36061 18100 13 +42031 27764 12 +45609 34908 11 +4202 19861 1 +53817 54792 15 +53777 11430 13 +20063 32128 13 +14946 32742 13 +22391 49331 2 +37642 29734 2 +52319 24915 3 +3426 58328 10 +10091 45728 3 +43480 16943 2 +36457 13180 8 +27478 31196 10 +6000 43940 12 +7211 53022 13 +9551 11815 11 +9218 26246 6 +8640 49973 11 +35452 48850 11 +56587 41229 8 +24638 45653 14 +18843 538 8 +27937 42134 10 +6385 42464 9 +20127 9147 12 +59350 51369 4 +13558 12095 1 +3148 48183 9 +37927 18273 3 +6434 18579 14 +38948 5634 14 +27695 29692 8 +2085 55117 9 +38248 27571 11 +1689 51160 11 +37689 53754 9 +22551 41402 13 +7302 28577 7 +18548 52876 10 +4886 32396 15 +34389 14657 8 +9069 17105 4 +12200 56335 11 +47554 29194 13 +34814 57622 14 +48591 8592 15 +44922 5029 12 +43140 35309 11 +35220 50820 2 +47611 32966 6 +37380 3997 8 +30174 56025 5 +39103 26430 14 +21456 1611 10 +12540 56570 5 +5358 7870 5 +44184 11170 7 +6026 9788 12 +3687 22130 8 +23965 23783 12 +45652 53094 14 +20424 12421 2 +256 42958 8 +10199 21037 13 +15761 14184 3 +24069 41268 6 +43807 34589 12 +31957 28469 12 +33474 51445 10 +38618 44254 9 +44862 47952 5 +24931 40255 8 +59292 24519 11 +56492 46386 5 +17843 53685 6 +32371 21753 9 +27159 45987 6 +24456 34045 4 +15527 58557 7 +24806 79 15 +20666 26143 13 +36637 2537 11 +28114 53732 14 +5293 18301 4 +21795 19830 4 +13039 4340 2 +51691 20730 11 +4562 26902 14 +32150 25700 9 +25788 2909 14 +36860 26624 11 +25360 616 12 +43340 12105 11 +16400 38939 5 +14887 38144 12 +9182 26084 6 +8122 10900 6 +21249 30640 10 +16780 15736 7 +31380 26881 8 +47062 51107 12 +5794 7834 10 +7310 54474 9 +31353 20423 12 +55626 15613 10 +25410 15129 12 +17820 49650 6 +22328 20830 10 +29093 38206 10 +39418 22619 12 +12314 32295 2 +44770 16672 8 +14441 54435 15 +46804 33402 15 +56559 25766 9 +41725 13808 7 +16653 9930 7 +41517 56350 10 +51981 26836 9 +45538 50376 11 +41625 58262 9 +12677 31139 11 +32420 6874 9 +34010 34020 9 +35446 23391 15 +29120 31304 7 +56321 58264 12 +2760 21279 3 +49651 36940 10 +26891 42811 12 +12869 37843 1 +10536 26854 12 +7886 24472 13 +28009 7650 8 +44294 42174 10 +1496 14910 7 +8918 10967 12 +8239 22793 6 +53160 59439 1 +3081 17050 7 +7254 4983 9 +6977 39466 7 +4867 29709 13 +9729 31532 18 +19639 33768 10 +56196 19066 13 +3594 4009 9 +29863 25090 14 +16714 11382 9 +1057 40565 7 +10394 41356 10 +46008 24990 16 +19755 48744 8 +21741 20643 2 +52323 46528 8 +19048 47257 15 +3883 7228 16 +32038 33606 10 +20942 56873 10 +35529 49951 4 +48153 46022 10 +26123 26479 4 +26239 56435 6 +11408 32107 9 +23370 39294 9 +7815 21079 11 +30617 11883 7 +30793 46262 5 +46906 9647 12 +20532 35281 8 +4722 4626 6 +54600 11480 3 +4298 36351 7 +8245 32260 14 +14495 28462 12 +5202 10288 5 +53830 916 11 +45066 2362 1 +49184 20475 8 +28644 25146 14 +30864 34401 5 +4394 54020 10 +22881 46276 3 +3699 57341 6 +12486 38171 3 +52015 31671 11 +17560 3347 4 +11903 4932 18 +27748 30173 17 +32209 29535 11 +29599 48752 8 +39343 35647 11 +32168 28007 14 +18280 25038 5 +36452 50479 7 +44447 6429 8 +44171 57229 12 +14002 43188 3 +44096 41767 3 +45295 8311 11 +33184 52160 5 +56118 55429 11 +31269 32862 7 +46250 43798 9 +21950 22682 11 +19040 25789 9 +6501 34882 10 +6727 48631 6 +4834 58372 17 +58874 6840 8 +29550 42636 7 +18514 45363 3 +57617 41975 14 +29740 43584 8 +58784 37526 2 +16475 14940 7 +11660 37581 13 +13669 53033 10 +14871 58470 16 +51045 22215 6 +19731 3667 1 +9469 20082 13 +58429 58968 11 +52525 6016 10 +17572 1575 10 +50123 7174 7 +10399 6053 8 +33558 40232 8 +32800 49052 8 +19706 24327 9 +38078 52594 9 +34592 4426 15 +26880 2771 6 +38573 11335 0 +4023 56613 8 +26693 7945 11 +19391 40287 3 +9707 16654 2 +29789 40034 8 +16616 47022 7 +10014 36512 14 +11720 20446 10 +45185 3959 15 +40237 23033 4 +53554 31277 3 +8086 51622 8 +1116 6665 10 +23491 24044 9 +3447 26456 13 +52592 38479 14 +8308 57303 10 +3105 31211 7 +16270 27116 18 +48867 38019 11 +49353 12044 9 +48291 31365 4 +27208 4131 13 +28807 3058 12 +13390 49036 7 +36739 16135 13 +30073 15290 2 +38409 43837 7 +59352 11731 9 +5498 47153 1 +32106 5156 5 +5231 1481 11 +16240 30750 16 +39081 1873 11 +51500 27667 14 +11202 9627 11 +44540 29282 10 +30665 46371 0 +27783 40982 4 +14860 36760 8 +18622 48165 9 +44108 28892 9 +38606 41012 8 +2144 39097 2 +17115 837 7 +49914 8190 7 +11432 39406 7 +8075 26086 7 +6785 43383 9 +21277 45386 9 +59922 46595 7 +15235 53927 14 +11613 24304 14 +37698 21590 9 +52526 35608 5 +22330 47921 5 +35013 15654 18 +49394 20133 8 +5638 47681 3 +39767 30037 6 +23881 23487 10 +22755 21369 17 +30884 21086 9 +53181 53308 9 +5964 50570 14 +11454 45449 12 +24986 6482 10 +2885 30593 9 +15800 34243 13 +45885 37555 5 +54543 25084 6 +51204 31918 16 +27012 6160 5 +7673 55063 11 +38889 10383 6 +28243 32402 10 +26974 10339 10 +18948 29028 9 +38357 49482 2 +7629 56426 7 +29609 26570 6 +26647 22475 2 +27449 26188 7 +19673 55643 7 +37691 56308 4 +35417 43084 12 +33946 52967 9 +32176 33245 8 +58121 52427 9 +11193 30221 11 +18810 12231 13 +38517 35103 15 +102 37502 7 +35432 58636 8 +12760 55733 12 +20687 53402 6 +38189 57497 10 +43699 35889 10 +59429 11108 16 +14082 42112 17 +46924 56687 7 +54068 5516 7 +24458 48319 12 +53452 3007 9 +27165 11464 11 +5459 48988 16 +21643 47091 12 +24281 43605 9 +15720 16531 9 +14730 36338 11 +7787 12072 3 +52114 57129 9 +38988 22892 7 +32703 16272 10 +6710 46508 11 +41769 51497 13 +38963 14079 1 +13024 42968 8 +37182 35985 2 +50269 54313 7 +29642 40231 10 +21191 40196 1 +27105 55450 13 +8230 20302 4 +50635 28052 8 +32667 8850 14 +55174 18593 13 +19717 46424 8 +25509 55724 10 +20512 29852 11 +36007 41942 18 +30367 18305 12 +29832 26348 8 +21996 32987 5 +27512 59710 7 +22693 31151 11 +14142 53834 13 +37198 19162 13 +8261 50689 6 +55646 6271 4 +41578 8235 12 +28968 56421 6 +27524 55005 12 +46447 33683 11 +48335 16054 7 +6398 49208 3 +6213 51250 4 +29054 52423 11 +34929 25988 16 +19461 55105 1 +40414 20099 3 +40454 19572 7 +13444 32359 13 +56503 42171 8 +17501 53920 14 +54281 59999 13 +14945 33595 7 +36399 34621 10 +23790 29189 14 +31121 25684 10 +10306 59702 15 +44177 11681 17 +12179 52346 13 +18112 56187 4 +3670 58396 3 +40790 53580 9 +18286 32073 9 +15748 5510 5 +40740 5689 6 +59897 19793 5 +4441 21990 2 +48404 690 12 +49769 14997 13 +448 39475 8 +23458 54045 13 +59596 974 10 +12459 52735 8 +6332 48454 1 +51103 30390 12 +49920 34321 13 +13503 28277 7 +24685 47556 15 +47925 14316 8 +54459 33513 6 +46840 35014 3 +7438 20153 13 +16708 30867 11 +50931 29226 7 +16287 20004 8 +29780 2757 14 +28303 56832 10 +49722 48050 11 +13234 11886 9 +30898 23667 7 +39500 28461 11 +35094 50504 3 +59644 10204 9 +17866 18005 12 +38611 23730 12 +12484 26816 9 +33004 42393 4 +12921 24250 12 +38547 21152 6 +32840 10290 6 +37949 54550 16 +5416 25320 15 +15496 4206 12 +22650 55144 8 +17609 45093 11 +22957 56728 4 +11627 44905 8 +56710 25064 9 +4098 14126 10 +39653 44452 14 +51944 28973 4 +19958 49375 10 +44156 48667 10 +40973 18460 3 +46868 52558 0 +20452 7270 12 +54532 39230 12 +11628 39131 17 +34472 48314 11 +10683 53060 11 +53924 23575 9 +33323 7735 13 +2837 34685 9 +29930 47721 3 +43890 35610 10 +19257 49801 16 +24664 51495 13 +16947 49915 10 +26153 29076 13 +15949 48819 9 +57001 24160 16 +28578 47134 13 +53651 5543 9 +13840 57245 3 +15005 1026 10 +37875 29314 9 +27589 17454 3 +44555 40341 12 +5180 42823 7 +24300 50503 15 +7603 1358 7 +13400 1712 5 +28098 16024 3 +29573 17141 6 +33320 42936 6 +28860 43519 12 +3104 56580 18 +9454 57204 14 +20190 46222 13 +28460 16545 10 +40732 56069 2 +868 50295 9 +37424 24041 16 +24725 7176 6 +3220 3677 3 +52321 37014 8 +13748 36414 1 +44991 49109 11 +31849 3455 11 +9537 9579 9 +11268 57367 8 +9951 8303 13 +39808 57493 13 +21963 30498 14 +30352 55479 0 +22390 40417 17 +3614 43750 12 +3457 1154 11 +26453 23911 7 +7847 54162 14 +13069 3452 7 +49046 41834 11 +22319 23570 16 +16318 36077 2 +13228 52516 9 +59447 31505 8 +1353 34023 10 +54296 41243 8 +40696 35388 6 +8124 17220 3 +6968 5748 3 +29193 58013 13 +13756 30324 15 +36443 34129 9 +53116 47944 5 +44308 46375 8 +36740 48582 7 +47587 16188 9 +48480 22589 3 +48184 9973 2 +31308 17536 12 +12086 59215 11 +6923 8089 0 +21372 18583 1 +51239 14557 9 +49170 29700 16 +12168 9041 9 +14147 10128 6 +45927 32426 9 +10538 43578 6 +53981 31178 9 +41978 49567 5 +17973 22178 8 +40526 52730 13 +7463 3270 10 +21740 30337 2 +37888 35177 13 +27190 54354 6 +29433 57988 13 +35426 47461 9 +24545 20114 7 +15390 944 11 +7074 24043 6 +15086 59811 13 +479 53666 10 +49357 9515 6 +51983 48052 8 +39310 50592 12 +32673 13592 12 +56215 33238 2 +7162 45306 5 +27038 56154 7 +40579 12377 11 +34966 58648 15 +30222 10644 3 +18821 15023 10 +23595 28766 12 +53677 20143 7 +8433 21539 5 +24383 33049 9 +14256 1647 7 +14317 23931 10 +8506 7777 9 +10848 5551 14 +52637 31662 4 +54028 11499 12 +22739 20228 16 +21367 23038 9 +47492 24910 11 +35954 13896 3 +5278 3819 9 +8466 32388 4 +27877 782 14 +54712 16082 17 +41972 30517 16 +47017 15873 14 +31799 43312 9 +8472 22823 7 +37170 33646 9 +37425 32017 3 +54989 38278 16 +55128 21028 8 +849 12472 4 +36142 3773 11 +25282 26035 7 +59677 46251 3 +16199 59355 9 +39363 53318 6 +14022 41149 13 +58535 22861 7 +27221 51195 5 +30524 8014 4 +10136 9649 7 +12774 24705 14 +25609 36400 6 +37669 843 5 +43134 57875 4 +44003 6678 6 +40320 29257 7 +39671 38982 8 +56231 46148 15 +39556 47090 8 +11725 32037 9 +41586 15352 10 +21169 20573 16 +15061 56259 6 +28776 49191 8 +59484 10696 10 +18426 58912 10 +45212 31798 14 +35507 21604 6 +50395 53495 9 +34218 49151 2 +344 44583 9 +34711 39145 7 +53647 41152 10 +954 51553 12 +49786 30786 8 +59485 14917 13 +1595 23502 11 +19773 5542 10 +24353 56903 6 +59513 51985 16 +1397 38986 5 +24702 18749 12 +18784 13855 8 +42103 33366 15 +52052 36965 9 +6175 24240 11 +25546 19239 17 +29755 13573 9 +16596 35763 16 +2770 23485 6 +30201 10344 13 +58258 57264 10 +34040 24147 16 +30873 5962 12 +59252 46072 5 +10505 57503 2 +41969 32931 6 +17740 3609 7 +38979 1498 12 +6958 46382 6 +32172 4678 13 +18866 52725 10 +39976 28291 5 +50158 29857 10 +42990 33073 14 +10869 32007 4 +11258 36939 13 +23234 29705 14 +7948 25974 2 +21684 47666 13 +42603 20716 7 +13429 13092 14 +43628 47779 11 +33484 48966 11 +29419 9786 12 +58061 51604 11 +42194 39737 2 +37866 31939 18 +16883 27474 9 +21017 26371 4 +3784 9519 16 +54215 3349 13 +11231 49930 6 +46220 20601 9 +22290 49709 15 +49875 28909 3 +4318 18448 13 +19620 45311 11 +34350 24026 14 +44911 15404 13 +8491 59416 4 +14247 26534 12 +16613 21267 16 +33597 9901 16 +58139 12998 4 +57977 35353 8 +9661 27830 15 +16299 40851 9 +23323 26092 8 +25200 17658 15 +17700 43192 11 +37023 5881 1 +39926 30198 12 +55586 12135 10 +59732 9164 10 +2025 8254 9 +59222 50684 2 +15231 21978 2 +4749 48126 1 +4647 30810 9 +22504 26200 12 +58917 16485 6 +10693 20009 7 +36348 44596 12 +38265 13399 10 +10658 12006 18 +2588 45477 9 +59361 20352 16 +56546 13463 11 +57928 37848 5 +53363 10176 4 +36691 54866 9 +12191 2375 7 +21931 51613 7 +11336 16659 14 +23222 59505 10 +6442 10467 6 +25120 59258 17 +55940 32103 16 +2796 8717 10 +37783 6020 9 +50131 39667 7 +52334 29943 8 +35327 48090 6 +43725 8231 16 +43654 49903 3 +30952 5181 14 +45636 9002 12 +22093 40230 7 +32766 49540 15 +13089 44676 16 +46410 21974 3 +39537 40435 16 +4383 36702 14 +10262 55073 8 +58088 28423 9 +40532 25531 4 +16821 51989 11 +58143 52661 10 +59504 46735 13 +46695 12715 12 +49822 5209 17 +25445 29901 9 +18775 13921 8 +52355 55315 9 +13248 4124 14 +48907 49683 10 +9719 4523 15 +35277 40718 9 +22685 7755 17 +47758 41237 2 +52858 11599 11 +21224 7152 13 +47530 7981 2 +43250 55163 16 +33961 22003 12 +43369 54192 9 +30681 11063 11 +36082 22509 8 +16553 14516 6 +32910 20005 15 +51496 35034 13 +13725 39060 8 +27882 7776 6 +14819 12134 2 +55267 50400 2 +37406 8958 11 +8661 48092 16 +35153 12756 7 +16836 21288 8 +8095 30422 5 +51895 28690 1 +51760 52591 9 +5374 15834 11 +45686 14705 6 +48954 8711 6 +46858 21141 7 +33863 21269 6 +26728 11530 6 +42086 766 17 +31683 55338 1 +25237 28877 10 +44903 40637 5 +6406 25777 6 +36731 34465 5 +19188 52496 9 +19557 14261 11 +28289 16758 2 +50128 43442 14 +20631 54468 4 +44503 4163 7 +53670 43515 8 +49559 59154 1 +8065 57916 3 +31903 28773 9 +57789 39002 7 +23713 45720 11 +7657 59791 10 +19771 54636 15 +51143 7702 17 +59713 6251 4 +53695 12037 13 +32762 7172 9 +57256 39628 5 +21243 3624 11 +46589 5096 9 +41029 37252 1 +58776 56248 8 +1446 26274 11 +6932 33223 10 +11605 48332 2 +27419 51548 4 +40560 8794 9 +1001 35646 7 +16395 36110 8 +24400 13528 3 +45687 42064 10 +16230 27497 15 +13661 14470 13 +18550 27863 14 +40823 58866 1 +11568 40506 15 +35450 48141 13 +41452 43964 8 +49188 7878 6 +59512 15477 6 +12664 18504 5 +28591 19029 7 +56981 15462 11 +21762 24761 10 +59031 51415 7 +34273 35561 8 +17161 58145 11 +6875 52810 3 +23861 24223 13 +17380 1906 13 +57743 45070 10 +54005 13761 3 +10787 15503 12 +49972 58388 3 +26112 38627 8 +59143 48457 9 +42360 2818 12 +17420 54034 15 +36862 51330 11 +30443 26733 4 +10106 2521 11 +25837 17727 10 +9108 52358 6 +45225 16830 12 +33082 3326 12 +2706 52374 13 +59853 22612 8 +53195 46207 6 +13196 15967 1 +2086 19003 8 +23819 30085 12 +16216 59918 4 +17314 29039 0 +10317 11446 4 +39739 2108 9 +45312 40690 3 +42928 31803 16 +46551 58322 3 +22354 9384 10 +20841 30645 12 +15863 35033 6 +25034 12042 4 +25893 3040 14 +19171 59495 8 +18187 50398 14 +32573 44547 17 +46380 9621 11 +19110 8975 8 +42244 49367 10 +30219 7915 11 +25392 1855 16 +2080 37354 6 +1235 25581 16 +53279 38394 15 +15705 35433 13 +41225 34777 4 +35419 52895 11 +49705 41922 5 +20270 16962 16 +21253 20466 8 +56102 17991 13 +4428 3538 9 +7567 42282 9 +13053 13420 15 +8297 52470 7 +32095 31310 7 +1524 52020 6 +5076 19683 9 +4246 57372 10 +45899 42395 11 +58829 24194 9 +37373 37733 10 +35105 35473 9 +26672 6838 17 +26271 55474 10 +6711 39461 6 +38535 44582 7 +2145 9479 9 +56136 20635 8 +59586 54552 2 +7494 22821 12 +50260 6594 6 +2648 47839 4 +53975 35286 4 +31569 50344 8 +24762 8676 6 +24763 33685 5 +37822 1265 5 +36768 35510 4 +19783 9070 17 +5558 50877 16 +32194 22089 8 +5475 9578 5 +30815 28543 4 +5507 12261 2 +18970 2382 8 +30655 21862 8 +13114 6451 8 +27869 24473 7 +20530 50252 10 +13660 59189 4 +18801 54504 10 +16406 40816 7 +49239 25150 9 +32446 6445 3 +56315 59531 14 +51208 7851 16 +48456 12748 1 +20141 4333 1 +51905 9266 3 +28296 20580 8 +26899 53697 8 +31878 25439 16 +12537 3751 14 +4320 40051 12 +42072 39805 7 +4116 14113 9 +3942 56181 7 +19474 2884 9 +17457 1951 7 +13227 48087 5 +38813 17511 13 +19636 20887 8 +28916 17020 7 +20334 19295 9 +56338 33555 6 +43110 27215 10 +28685 13432 4 +39546 44363 1 +52745 10999 3 +53296 33156 14 +20435 9451 7 +32611 16781 4 +7827 10411 4 +52705 30889 9 +18717 13846 10 +47068 21906 8 +25600 41226 2 +26048 1257 9 +51191 57353 12 +25709 3231 8 +46690 12180 2 +5729 29936 3 +49229 15114 6 +1103 31968 17 +29168 52557 15 +39149 59035 14 +44497 42257 6 +6100 33321 14 +45219 53312 12 +25363 43026 11 +27098 29011 11 +12438 31870 5 +49604 23256 11 +28480 54267 15 +14644 59473 5 +55902 45824 14 +7227 31092 6 +28984 4521 17 +19381 33942 2 +46062 36248 6 +6563 31726 16 +35064 39769 7 +7180 24879 10 +41573 13844 5 +41369 57454 10 +7801 33285 6 +889 33737 7 +40419 16144 7 +42030 34749 8 +38258 14205 9 +36367 56934 11 +28727 415 14 +31901 43425 10 +1373 56875 7 +42854 34629 7 +58148 47201 9 +37211 1955 14 +10834 605 5 +18492 55659 8 +56901 31370 3 +22720 20758 13 +37667 54895 10 +7478 58773 10 +26366 24177 2 +4134 26199 14 +12417 45668 15 +5684 29689 13 +52681 19480 4 +29830 52746 13 +39340 54488 14 +19938 19470 17 +18096 8697 12 +26835 28503 9 +42075 14676 12 +32249 21301 10 +24542 52063 8 +6880 59870 4 +9750 19217 4 +17303 29331 13 +39763 9078 7 +49373 39047 11 +17552 23958 10 +25721 15837 12 +11772 49004 8 +5105 49050 8 +4311 57289 3 +16370 47394 4 +8785 26470 14 +2731 7113 8 +49236 7408 7 +47277 26747 7 +13920 10041 16 +20799 53222 9 +19427 29104 7 +15989 27001 5 +51699 48392 5 +44124 27674 10 +9646 54489 6 +50153 33776 2 +27463 56384 8 +43716 277 7 +34082 55542 7 +12870 30588 8 +6552 42250 4 +9900 51081 12 +18227 29799 10 +20620 9908 11 +20720 50435 7 +26386 19467 10 +12146 45858 6 +28529 19786 7 +28015 49338 2 +43349 2641 3 +39411 11248 3 +41943 51515 7 +3778 55204 5 +47077 7185 6 +48343 58036 7 +11670 44075 13 +41446 50468 8 +53459 51909 7 +3895 32351 16 +33819 47061 9 +13774 26352 6 +15556 25010 10 +14613 18727 3 +21009 50715 10 +42353 30772 4 +55393 31316 8 +21033 22299 11 +5600 41597 13 +7784 53136 6 +21564 17801 10 +57961 14034 1 +21833 49824 13 +17196 59795 8 +12599 21016 12 +2584 54588 4 +26176 16382 8 +21121 44256 6 +16034 59256 9 +57929 2383 9 +22538 25706 10 +2127 16975 3 +45391 10612 8 +55714 4520 7 +43441 51083 3 +10746 39344 17 +34033 5314 3 +57033 57609 16 +19790 48532 8 +8951 48320 10 +10757 44871 3 +46771 26403 10 +48363 47862 3 +22985 24038 5 +9589 56506 6 +58833 12625 7 +50478 9269 2 +9563 20390 6 +55663 32646 5 +42646 24654 15 +40781 32449 10 +53342 34338 18 +36375 17786 4 +41322 27308 9 +29957 18364 6 +828 8284 4 +36634 47532 8 +12873 44623 11 +13575 47706 8 +20351 40115 0 +36453 50899 7 +33501 47840 6 +1650 50422 10 +30626 29277 12 +36559 21214 11 +41985 50672 5 +32340 22874 4 +45031 3956 4 +13159 35025 10 +29520 55132 8 +25636 7742 15 +18239 15469 15 +33121 23718 7 +3710 32981 4 +55068 7902 14 +7975 50510 13 +23231 2116 11 +5893 54249 12 +23947 51413 14 +29329 39619 8 +21780 27536 6 +59967 50472 9 +13620 13596 17 +15386 38585 7 +33797 50093 13 +36071 20551 1 +53502 15181 3 +38297 16060 2 +46084 30826 9 +34827 19296 8 +57797 54645 9 +49458 40243 14 +15542 41693 5 +44980 43495 10 +13461 57993 17 +1327 52387 12 +57551 1622 18 +14669 23327 17 +8219 3350 5 +33322 31996 11 +17624 39598 12 +37450 13496 7 +32397 29874 14 +7341 29243 6 +23333 10676 4 +27253 2449 9 +55924 48303 5 +33453 16452 14 +4291 25175 5 +7410 4751 2 +43469 19567 10 +59701 5003 11 +44987 49506 14 +36190 48508 8 +19434 29347 8 +58670 21977 8 +37965 8477 13 +47688 23652 4 +32159 54915 10 +23394 6792 14 +27909 1424 15 +48304 54289 14 +5495 1748 15 +42260 18700 10 +47456 28496 15 +10246 52984 7 +8022 25006 9 +20110 12034 8 +33698 38423 10 +37559 56949 0 +19406 38891 8 +48569 45670 13 +45635 49374 6 +51608 4970 3 +45536 39978 10 +39596 48383 10 +17414 21606 8 +57486 1082 7 +16719 32078 12 +17397 40433 12 +21045 44384 17 +45285 617 14 +26122 20807 16 +22564 42417 12 +48690 59822 9 +28030 24675 7 +50664 53403 7 +36582 1215 12 +59508 37571 6 +50365 1222 12 +32894 1983 13 +6196 27690 11 +44241 37830 13 +25863 15708 13 +26195 32968 13 +5673 6143 6 +10248 33769 2 +1392 20140 12 +17124 58510 0 +25218 47414 9 +55758 4585 4 +9848 46121 11 +46357 21013 9 +37758 8227 3 +21727 13841 6 +20304 37766 9 +51658 13874 2 +52162 18716 13 +21666 45194 15 +23073 36176 12 +59815 19590 8 +25679 26604 6 +51478 53943 3 +35480 21842 18 +54510 39329 8 +55657 33804 4 +40141 19638 11 +21110 25831 12 +2105 183 11 +18211 51805 15 +18136 11990 13 +26736 41286 5 +38759 2523 11 +21046 11250 10 +9165 25921 8 +35882 18679 5 +5015 49507 7 +20733 54824 7 +23357 35590 7 +7146 1094 6 +46470 36635 9 +8481 27816 7 +2980 36210 15 +46914 27634 6 +57524 26042 13 +21366 53468 3 +40773 48835 9 +2405 9314 13 +4224 40298 7 +10840 28977 2 +22795 31899 16 +20987 12522 9 +12496 13990 17 +22376 26183 2 +5512 27334 9 +51036 50405 13 +11756 10465 8 +41334 11988 13 +50742 25415 15 +22521 54185 17 +1312 28889 12 +1902 28356 11 +726 35320 8 +51306 33901 13 +56157 30771 17 +42122 27021 11 +5839 31113 6 +57330 4999 7 +7662 9484 8 +24349 41479 15 +1160 58638 6 +8476 53248 15 +42959 46991 13 +16940 146 11 +49967 27181 6 +52126 48247 7 +32855 6159 7 +44209 32346 9 +49614 53091 5 +54923 37762 13 +24502 22876 14 +27 32181 11 +2070 51424 16 +36825 53095 14 +8740 41360 3 +55672 32259 13 +40311 16157 12 +33115 56081 5 +45232 20572 15 +31908 48558 5 +31400 57453 10 +57566 10736 9 +23184 17399 12 +18487 20708 7 +52077 18325 10 +54700 28895 3 +41218 8579 12 +21881 35051 4 +51121 46060 5 +53668 30715 8 +4306 38830 14 +33351 26950 12 +59294 7502 12 +51578 10080 3 +9839 24424 9 +39365 19337 7 +1429 32327 10 +9429 32503 15 +37204 36490 4 +33812 9545 13 +10859 56690 9 +34330 43021 14 +28862 13631 2 +30207 17147 9 +24348 47225 3 +6340 41275 10 +21000 21077 11 +24737 32262 5 +54872 21217 8 +1473 36811 5 +47079 58189 1 +29888 10545 8 +13493 178 8 +9189 5266 4 +571 4942 9 +43715 58457 1 +35053 48089 16 +48784 19333 5 +22550 3389 15 +701 49134 7 +45175 20111 14 +58581 39202 2 +52565 5511 6 +6571 441 15 +4025 35640 12 +19290 47210 10 +37302 31032 7 +57629 54358 4 +58347 7542 5 +51879 29485 11 +41652 17245 8 +44373 26333 5 +17631 48274 5 +17876 4914 8 +58547 2880 11 +18432 9747 13 +43797 48495 12 +8471 8119 6 +59227 45577 4 +2308 40446 14 +40145 21691 11 +1934 33873 11 +669 15771 5 +16794 44008 11 +56539 26212 13 +21916 50615 9 +16159 2531 4 +12457 53890 2 +25463 2674 14 +32362 14920 12 +57833 21411 9 +56363 4655 5 +56459 51502 7 +40673 28344 11 +30917 59022 12 +19047 44410 15 +37416 17154 5 +43062 10249 17 +57060 15457 10 +44603 9744 0 +28477 57427 7 +1120 26047 15 +1994 34326 5 +38338 28583 8 +37330 41717 8 +4404 33216 11 +24489 25997 14 +23109 17645 13 +31682 15923 12 +11677 26097 8 +55786 56891 9 +8892 4768 12 +13365 41607 9 +2059 33925 10 +25178 7708 5 +11300 26175 10 +17532 23448 15 +7159 7015 11 +37457 15698 14 +14110 8632 2 +42058 34444 3 +34636 54425 8 +10727 28923 7 +59890 36506 9 +36328 48157 18 +39686 43660 12 +16430 26150 11 +27256 56516 5 +34958 29903 0 +14075 3740 11 +49064 16789 9 +34808 41179 6 +37278 25373 11 +50488 41914 8 +7779 22228 11 +39312 5232 5 +11803 8193 6 +18898 40403 12 +39112 6316 6 +41765 14462 4 +34419 49680 10 +20480 4200 13 +40139 49044 14 +894 27503 6 +21146 35594 15 +55360 15770 2 +49503 22781 12 +53247 57774 16 +38501 33008 2 +17992 32941 9 +15265 33480 2 +4976 27631 1 +35667 53193 5 +22458 52013 16 +44369 17628 9 +1431 18772 10 +4041 49877 14 +11181 22198 10 +32883 33467 4 +8269 5066 13 +15959 13538 6 +33141 44229 11 +23785 30788 12 +39684 5949 9 +24678 21074 7 +53516 53520 14 +23613 51806 7 +54829 16510 6 +28701 9841 4 +48796 35289 9 +35063 58706 5 +38553 10635 8 +23121 51453 8 +8232 57339 8 +4090 43479 8 +17752 49675 3 +28375 38785 7 +22076 14230 4 +39845 4386 11 +5987 27176 8 +50333 30023 7 +22816 46665 6 +30177 5698 7 +1843 696 5 +22837 36269 8 +22277 51955 11 +18742 688 5 +13027 44768 7 +51255 11095 6 +48171 49821 12 +2775 19879 13 +20031 57141 12 +5223 6218 11 +17065 14969 15 +34806 57766 8 +22388 7325 9 +9710 33297 2 +21118 10768 12 +7042 17953 14 +44720 23750 8 +30226 6511 4 +15134 59779 13 +59812 45694 16 +8731 50214 8 +33927 234 6 +736 1387 8 +45692 53376 11 +15719 1219 12 +43377 923 13 +7231 37723 8 +35568 36586 3 +20683 39155 9 +20711 2361 8 +56006 24709 10 +24957 2957 7 +36887 51637 7 +29420 28659 6 +49412 40676 13 +58496 53335 12 +52441 27158 13 +55979 17249 14 +3733 47597 9 +13909 21500 7 +47172 33416 1 +10436 48737 10 +40754 17054 13 +58629 52269 9 +18367 32138 8 +22592 10903 11 +38492 16298 3 +9804 8351 2 +35171 8493 7 +25040 23955 6 +55011 26019 8 +11833 18162 5 +7223 9724 9 +13490 17935 9 +18087 56514 9 +55471 12393 14 +39566 29552 12 +36663 31787 6 +36217 57913 14 +1331 46327 8 +36802 23567 8 +54842 13867 15 +14117 16028 14 +13827 929 12 +23111 8897 9 +24057 8542 10 +14342 4799 9 +21823 39052 8 +16929 44934 6 +57754 55538 10 +36183 50322 5 +56437 39195 12 +41305 58977 12 +2752 16999 14 +42863 31652 14 +2261 29139 10 +56034 42358 7 +31516 12991 10 +9617 55934 10 +49081 15570 10 +58826 57969 8 +29407 16194 8 +32680 55108 8 +565 7316 14 +35391 36723 10 +28039 31552 10 +34743 40331 15 +41409 22578 4 +5531 28198 11 +21681 35454 2 +35400 51728 6 +44550 37910 17 +32632 30241 4 +31587 20516 11 +6382 30296 18 +58419 38252 10 +37892 45085 8 +6179 35616 6 +50818 38378 11 +51480 49147 9 +47092 30863 8 +46416 18527 4 +25347 26 8 +33436 26428 15 +59986 29672 15 +55469 24142 7 +58517 11374 15 +43174 31311 12 +42708 24858 2 +40510 51702 9 +52618 941 10 +24921 36676 9 +4189 26957 14 +11715 29092 9 +16522 36321 11 +39929 33282 6 +24376 27584 5 +10518 33060 5 +17346 25913 14 +50307 3886 10 +44834 45763 10 +45596 42743 9 +10786 54458 7 +35021 4037 13 +38662 15036 6 +24382 34940 16 +51861 53070 17 +6809 7499 3 +24744 4893 11 +46560 42925 0 +54007 1710 5 +53591 20924 7 +32439 42738 10 +51783 26571 18 +17800 57841 11 +53773 52267 6 +14729 6448 4 +16266 14481 8 +50441 11273 10 +59664 10400 12 +29694 11664 14 +59673 40940 9 +43860 10485 14 +10085 28604 10 +603 37012 1 +58384 28117 15 +51224 18951 15 +54694 27911 3 +7964 50216 9 +25105 50169 11 +7386 51412 15 +59039 53931 13 +25751 50001 9 +37819 40195 15 +3416 16711 12 +38495 38148 5 +32748 24869 9 +31398 19749 8 +158 23201 11 +3840 9553 10 +46070 58762 11 +9498 1231 14 +53769 5040 11 +55581 7609 9 +3109 57513 14 +57455 50225 3 +39459 44408 10 +37879 748 14 +36365 35319 11 +57576 9918 15 +16629 16439 5 +13306 7599 7 +8135 39464 12 +27654 32531 9 +25762 46376 12 +23400 44963 14 +17215 14159 10 +55668 19456 12 +1791 44298 5 +51334 53832 16 +12288 21817 12 +28426 466 5 +25457 52466 12 +44292 26088 12 +59027 29716 11 +8750 56984 9 +13914 19862 7 +46194 30057 7 +57992 53425 3 +35299 44508 6 +19943 23946 11 +46649 27578 9 +32220 26956 9 +40574 48915 5 +12083 12901 8 +1149 45137 8 +17324 9947 8 +49944 6841 11 +42396 29808 15 +49596 7911 0 +12528 49941 7 +57946 58689 5 +28501 24548 11 +17149 38414 5 +46607 43472 7 +59564 35497 15 +23796 28046 8 +47932 27287 9 +47249 39189 5 +45057 9950 17 +55916 24871 13 +40315 26645 1 +5685 38023 10 +27141 24646 12 +25813 6221 12 +36343 32491 15 +11148 59876 10 +17274 3158 11 +16524 58732 13 +3709 57780 1 +35028 55169 1 +24296 248 5 +32815 31545 10 +23009 31160 7 +12430 8634 7 +25074 41507 11 +59003 43564 13 +58798 22094 13 +12924 10459 7 +39116 35684 9 +38496 39292 10 +39557 50676 9 +4653 24491 2 +23421 2294 10 +8621 45370 15 +46419 30879 14 +12641 45229 8 +16300 1025 9 +18537 33988 14 +2464 16068 6 +12527 34883 11 +9933 8363 18 +26046 28168 11 +2892 54527 4 +8451 47356 4 +33077 47669 8 +23626 34062 8 +15071 3096 11 +8534 4868 7 +29633 42242 8 +5256 22099 9 +21038 53717 9 +59299 17704 4 +33603 48276 11 +37173 23335 8 +22894 42458 12 +5858 23746 8 +37648 37404 7 +39319 57423 18 +7908 35285 9 +59920 25154 6 +22916 28506 9 +43254 11527 11 +41135 1823 11 +49804 54033 10 +22580 43883 7 +30140 35361 8 +51388 52150 9 +12319 34572 10 +14330 56132 13 +20374 7806 6 +19769 12994 7 +41168 23655 12 +31122 8050 1 +34639 37484 2 +11668 32789 9 +9671 27907 8 +6519 48012 4 +8577 59695 15 +8994 44396 15 +29858 51450 16 +58285 45595 8 +13984 15391 4 +1472 11135 2 +30795 21056 7 +29912 4513 9 +18183 26196 12 +54301 57436 4 +16878 51332 12 +31193 48434 6 +3427 53995 4 +27447 18982 14 +54047 13070 13 +10202 32758 12 +37068 48944 14 +52014 5098 9 +32476 47352 6 +57786 1790 17 +51466 24191 6 +57345 43122 7 +22162 27330 4 +56847 32063 3 +43020 23134 1 +15253 19803 12 +16533 3418 6 +50490 7781 16 +25289 20827 2 +58415 44594 10 +21869 59778 12 +59404 7539 17 +56792 51416 14 +28743 9840 1 +3001 3868 7 +12954 29357 5 +23221 28139 7 +1288 22476 6 +15107 29353 5 +32938 55388 11 +880 10406 9 +45043 54278 10 +21177 6344 10 +16166 43161 8 +18030 23165 6 +25541 1728 15 +54898 8992 4 +20362 8550 4 +52778 21550 10 +20137 42784 13 +20000 15216 8 +10883 20966 17 +9753 5186 3 +32691 9732 11 +10139 2102 8 +56368 18479 12 +17811 4816 14 +57790 44590 9 +2832 12827 8 +49225 24999 17 +31050 10868 5 +47676 51766 10 +52087 13854 4 +20678 3242 3 +39013 2431 14 +7512 22482 12 +56583 23286 6 +27505 22627 7 +28870 22492 9 +33771 19762 14 +25782 13616 16 +10954 44341 10 +40236 15336 7 +57861 30210 5 +11275 44277 2 +23203 1899 8 +55783 59385 18 +53482 1548 4 +47292 31002 11 +55147 14831 15 +12617 45810 12 +11551 33306 10 +46155 21415 10 +33208 47742 6 +39648 22107 15 +19751 49805 3 +20812 1008 12 +42814 24139 3 +6285 58712 13 +22461 33814 10 +27884 34264 10 +23341 56577 14 +5340 48345 5 +2750 56404 7 +36785 752 11 +18047 24825 17 +56425 56330 13 +52432 5732 4 +18118 27003 10 +5364 54274 13 +49879 10839 15 +23801 16930 5 +33930 40573 6 +5333 9862 10 +12491 41536 8 +49309 58747 7 +54876 8841 3 +5714 25003 2 +9602 29891 9 +30150 14484 8 +37768 6237 13 +13617 2313 6 +6779 41727 12 +32754 8559 13 +13663 54948 11 +39270 33315 8 +21796 16368 3 +13185 49463 5 +55980 23283 12 +24826 3286 8 +17708 24116 11 +15423 1932 5 +8399 34406 8 +50477 8744 6 +51647 8242 17 +7059 4074 14 +49195 12012 7 +54681 13818 9 +58598 27646 14 +1023 1070 13 +972 26697 14 +19362 41965 10 +2330 55230 11 +12178 54514 1 +41065 39803 14 +9875 28684 3 +55064 2136 10 +31459 57072 16 +45286 8912 8 +28594 59908 13 +811 18832 3 +21457 32170 10 +29395 3095 9 +14790 50271 11 +39166 46308 18 +9169 55532 6 +15076 48902 4 +19687 37255 17 +40813 47087 0 +27140 52216 10 +7592 41112 7 +22420 19502 9 +37964 50270 8 +19925 39278 11 +27691 56467 5 +19688 13221 5 +56251 20043 13 +7672 40866 6 +11378 40554 10 +38432 16251 5 +9176 38074 17 +37865 32532 14 +56840 5596 13 +29519 34773 2 +3264 27832 7 +3526 7852 15 +18575 16788 12 +5054 53724 4 +41752 48808 5 +4480 31592 6 +13058 50325 12 +35625 41428 10 +50788 30179 2 +56741 37583 10 +13280 48657 8 +13223 16837 6 +58462 46372 1 +29968 14047 5 +37647 8617 2 +16016 45691 14 +20889 51123 17 +54647 47644 10 +37588 10030 7 +16774 59463 9 +47161 51042 16 +22971 43919 14 +56666 7256 14 +54429 57010 11 +32075 16770 11 +16164 40739 8 +14936 22248 2 +21933 12329 3 +40180 42071 1 +13743 38900 5 +10556 17573 5 +15985 36864 7 +31276 32835 9 +10971 31096 3 +50594 9611 14 +19498 22603 4 +55877 58960 12 +7751 16397 8 +28135 49570 5 +18188 54756 3 +45034 50121 16 +40021 20510 5 +49019 19948 14 +56130 41948 15 +33569 30216 13 +23450 49440 7 +42414 57744 14 +42759 409 9 +31261 34499 15 +21846 24890 9 +7142 17652 14 +48606 19061 5 +55041 34438 12 +32489 3171 3 +35663 50083 1 +17829 43066 11 +48510 30495 8 +18957 33609 9 +30273 4444 3 +49061 7094 14 +24261 59868 8 +25779 26680 6 +28404 24102 3 +58741 40135 5 +31162 59417 14 +58306 17035 1 +3143 33996 7 +42642 48005 15 +17604 40663 11 +32441 49618 15 +13371 36569 0 +51880 42132 9 +46693 11006 7 +9243 26483 13 +30732 38329 15 +9117 265 16 +36809 49522 8 +40911 55810 11 +32979 18617 5 +44401 49414 10 +18216 27481 9 +8784 9574 8 +2800 1018 4 +25458 8222 8 +59234 34753 14 +45615 55255 13 +11692 36693 6 +25987 49285 2 +28399 17989 16 +15558 48927 14 +10423 6078 12 +55046 25180 5 +45051 55350 7 +21777 6794 8 +17620 11939 1 +21510 48519 16 +35642 10365 2 +19540 2902 12 +14660 34851 8 +18538 59681 0 +57157 44031 7 +9219 56881 16 +9037 12563 12 +51220 18870 7 +22385 40340 6 +36879 31145 2 +45049 4075 9 +43779 50221 3 +2895 45206 5 +24320 49007 9 +570 31282 3 +7992 33252 16 +28750 59119 12 +44122 15312 17 +48590 25757 6 +57364 33615 6 +2811 9587 6 +42211 14304 15 +31433 55461 7 +58116 17110 2 +57409 27042 11 +46019 6110 6 +27599 28062 17 +31941 39241 8 +18375 46839 12 +2901 41426 12 +27664 29162 10 +46864 16412 8 +12294 22799 18 +12917 52851 7 +20938 46159 9 +40969 42137 3 +26983 10502 8 +15825 39585 8 +58284 38797 12 +11856 1808 6 +47206 17231 11 +15754 50630 7 +27741 20820 12 +3685 53803 8 +48373 12337 4 +46343 53299 2 +46857 40830 6 +48288 35293 8 +46437 51883 1 +24051 9159 13 +6292 30791 2 +44272 9632 12 +3323 52879 10 +27994 16422 17 +54077 43487 7 +48821 15628 6 +17616 26074 10 +882 6806 12 +14769 36423 6 +26559 21860 7 +46050 54038 11 +30261 44735 7 +31471 57869 12 +57747 56656 5 +17077 17376 3 +10835 21670 9 +7736 25168 11 +39220 37496 4 +15852 8007 6 +51299 25731 8 +57735 47421 11 +19484 15145 10 +27661 6244 10 +29216 50679 9 +7734 6449 8 +22722 39776 5 +30188 35240 9 +14474 8614 10 +29549 18937 8 +19953 43464 17 +33568 47121 7 +4084 29160 14 +44700 22421 8 +5528 26436 2 +14121 50090 6 +12765 55260 5 +10981 33391 12 +59376 44481 9 +7347 53378 6 +50511 13318 6 +58077 59212 8 +32828 55895 12 +36515 59408 6 +48031 55624 3 +36216 44365 9 +27237 56581 12 +44807 10882 10 +33662 49919 3 +30807 43400 7 +48428 53916 8 +1963 4896 4 +36609 31826 12 +49629 22401 11 +28457 52055 9 +24593 37716 10 +22444 43980 1 +20420 16461 2 +50564 32664 7 +53231 45333 10 +34834 22814 15 +53766 56861 16 +25240 39860 7 +11942 46325 12 +4720 1566 12 +2667 54766 10 +10115 20359 13 +14908 5260 14 +38439 15688 7 +34452 29037 13 +20195 6718 2 +2698 47009 14 +39309 45658 3 +19955 19994 6 +3431 26787 6 +26299 6694 15 +379 49612 5 +30691 50156 4 +7279 58686 11 +4406 6003 5 +57699 39042 11 +27480 57108 11 +20058 42281 8 +28938 30458 8 +54138 13470 10 +9358 7537 10 +56522 22526 8 +55295 30692 12 +6455 20393 8 +13054 21877 3 +37164 25489 11 +27043 7822 14 +4432 14553 15 +31850 17260 8 +3984 42835 4 +46621 14216 12 +49103 38739 8 +10595 9528 16 +31832 33851 15 +651 41672 9 +14006 44007 13 +8327 41633 10 +223 23004 15 +3077 6701 3 +12193 49247 2 +52188 36865 10 +4838 45118 9 +39491 23155 11 +40999 55772 9 +56751 27928 10 +56962 20346 9 +23093 31265 1 +26770 8151 8 +50888 13644 13 +13183 4066 5 +47263 8454 6 +49712 5536 12 +39903 30959 12 +41864 41679 12 +31286 40442 8 +24235 56535 8 +16338 46652 7 +15490 45772 15 +42524 52788 15 +6064 20853 12 +294 27315 8 +9148 58003 8 +28367 53484 2 +38682 37224 1 +5900 38969 11 +46835 50108 8 +47425 32058 14 +51397 3811 2 +27787 13580 7 +4514 41131 9 +13085 46402 10 +50929 38113 5 +24780 12911 5 +33211 12590 10 +2668 23756 9 +26765 47179 12 +1463 39658 9 +44615 27275 14 +15650 46520 3 +53471 51319 7 +26599 11943 5 +59470 33086 7 +49386 46810 9 +16203 35024 10 +37773 55765 9 +8986 10946 12 +18135 6627 4 +22422 7086 12 +13474 39478 9 +33783 654 16 +23110 50149 10 +41081 30360 18 +33174 1461 13 +12608 27548 4 +37575 22596 9 +33316 5084 9 +27833 10694 14 +35728 24222 8 +10201 21599 14 +25938 39927 13 +13245 35895 10 +37712 44741 13 +26651 3375 4 +38359 51750 3 +27262 6529 6 +40189 39364 6 +15480 29865 9 +7472 46559 7 +4591 36191 11 +21954 4092 9 +44693 1443 1 +51808 48137 5 +39285 27568 11 +42062 13876 15 +36283 36842 7 +36460 16561 11 +30483 15898 4 +29463 34901 15 +43804 50566 6 +37021 26096 4 +45851 54067 12 +58391 40736 10 +12298 27989 10 +41076 48384 10 +22217 13643 3 +13057 14475 16 +28544 26280 4 +27944 5104 9 +9567 43293 10 +5585 36297 1 +20258 6299 7 +51440 13035 14 +1813 8478 11 +23768 18328 14 +25350 25677 9 +50297 20892 9 +28495 29425 10 +19933 27458 11 +21808 31272 6 +55611 55568 17 +49542 14375 5 +18765 55546 7 +4978 15146 10 +59276 7969 7 +30933 38518 8 +47607 36786 5 +12895 13457 8 +1785 7337 7 +4315 32687 16 +54622 52425 11 +4814 29442 12 +26305 37696 5 +41566 28204 8 +45478 11293 15 +2315 31707 12 +16862 15846 7 +2599 13091 8 +4199 4905 9 +7106 35830 7 +40146 46143 10 +29247 30011 5 +18996 12704 10 +55200 22691 9 +45047 37314 5 +21830 27468 11 +50586 43376 2 +47429 49266 9 +55062 48072 12 +19512 11322 17 +29163 54932 11 +6426 31154 5 +12163 56783 9 +31574 57163 6 +5116 54239 9 +36797 34715 9 +42097 1205 10 +32746 14950 9 +11934 691 10 +28491 4767 13 +23821 29551 1 +3257 37937 9 +20312 13602 12 +21378 40199 9 +27130 51004 16 +58050 12700 9 +14620 43522 12 +4940 39844 9 +57268 28472 4 +45267 29654 8 +7420 28534 12 +29955 14924 6 +27359 21747 8 +20251 44932 10 +7226 15013 9 +3938 32811 6 +38720 4876 9 +53161 52450 9 +28682 13454 8 +22747 30765 8 +59697 17005 2 +3693 16280 9 +20138 14957 13 +39564 53956 7 +1412 30076 9 +34464 38558 5 +28905 26425 12 +25262 10358 16 +29531 494 14 +19037 55072 15 +7648 42010 0 +18915 25631 6 +13370 20296 9 +31570 24195 5 +45437 50714 9 +35690 30348 9 +21728 7644 13 +25462 22054 14 +24794 2785 8 +3572 29748 11 +37391 47131 11 +10189 20165 10 +4885 13710 12 +40871 48251 11 +39699 36844 3 +53573 18567 2 +32920 26056 14 +18316 5171 3 +21317 11504 10 +6170 13682 11 +53509 57472 3 +15295 43757 6 +39494 58511 1 +6822 5445 5 +1935 52403 5 +4255 16843 14 +777 29498 8 +40726 19672 12 +20363 19340 12 +35456 36098 3 +38095 7571 7 +17871 47037 6 +11496 30747 7 +7840 331 11 +3056 33124 5 +54085 55229 14 +29263 43342 8 +21282 15003 11 +12367 20769 6 +55593 49929 2 +42787 48233 7 +28448 24729 10 +38672 43382 14 +53880 51370 6 +20245 45156 14 +6073 36673 15 +553 33532 4 +30465 37719 8 +51200 36164 10 +51991 26608 3 +15630 46780 4 +54717 2272 11 +9120 8759 14 +39879 47805 6 +58757 17887 12 +59599 34898 7 +15106 14643 14 +24059 35365 11 +46271 25640 18 +39983 58896 6 +37371 31087 14 +56621 32022 5 +39606 58161 1 +48000 44502 6 +23386 40019 13 +24980 17838 4 +44752 24615 5 +26001 41546 13 +47448 7447 11 +2332 36761 5 +35061 52057 13 +5983 57826 9 +54384 45894 6 +33283 18051 12 +46098 28409 6 +21892 5437 16 +38393 12292 13 +20652 50678 5 +21432 23725 13 +4773 54223 1 +51467 12695 13 +31804 11087 15 +30473 55583 1 +1874 2066 2 +15096 27082 8 +35641 55734 14 +961 14458 3 +38958 30156 6 +44073 6318 7 +28067 27608 10 +18714 47004 9 +59926 15249 8 +44299 13542 9 +59538 29959 14 +23624 45303 9 +23022 55711 12 +9013 46761 5 +36233 12436 9 +3244 30370 2 +32086 46137 9 +49308 50852 1 +19316 33887 12 +43447 12510 9 +11157 48042 10 +56634 55301 12 +8804 45993 9 +37462 23197 6 +36115 39604 15 +5995 12511 14 +28742 29787 13 +18480 14222 11 +44706 40536 7 +34087 40835 6 +17746 28178 9 +2962 16656 5 +17449 14612 1 +36289 26751 7 +46672 8694 15 +31441 34241 9 +33885 23427 14 +38915 15361 7 +53845 16105 9 +39970 11486 7 +38568 12365 5 +18854 2948 9 +45549 35702 11 +40317 36038 15 +55477 54779 14 +26141 58016 13 +14494 29048 14 +27266 36801 7 +59298 5068 9 +53771 3912 16 +19618 17803 16 +52741 40057 6 +374 22728 12 +16316 59448 13 +11647 50634 17 +37940 56798 10 +48707 39608 14 +58899 14634 13 +31866 19060 9 +25131 38887 17 +13492 11123 9 +3265 18180 13 +4374 6926 1 +23748 27990 9 +10713 35828 7 +1390 22197 6 +19435 24432 12 +34729 42001 11 +15992 58132 9 +4237 13021 6 +33506 48878 11 +54584 16536 10 +44258 56112 13 +4832 11665 10 +55139 52166 10 +39111 22706 9 +19152 17672 10 +42821 44309 8 +3894 52003 12 +31300 22989 10 +37223 58669 9 +46389 23731 9 +15306 8923 8 +15546 45004 15 +33153 23782 14 +40627 31072 16 +45192 11620 7 +40706 33132 11 +26020 6149 14 +35596 58133 7 +18384 21644 5 +59630 36092 7 +4366 53258 14 +47333 3936 7 +8789 45561 7 +29910 705 12 +47139 27431 5 +14747 30962 9 +6462 40752 7 +56303 59832 10 +52832 22367 4 +10332 9964 8 +29058 38960 9 +22576 59646 16 +37540 31695 3 +21779 23547 8 +8718 38834 11 +39874 57589 4 +2227 54254 2 +50801 28711 10 +15927 27630 5 +22510 50528 9 +45116 23668 13 +37926 15794 6 +15292 27703 9 +38941 19192 5 +44833 36013 17 +32542 37253 15 +2398 59271 12 +17007 42749 15 +42068 38331 10 +55825 54774 6 +24096 33908 5 +2836 34132 13 +25902 54657 6 +48749 21578 14 +39890 10723 2 +16710 31924 13 +40689 47963 14 +53374 57443 7 +28981 47920 6 +42764 53552 5 +40525 43137 5 +52276 30699 0 +14089 9472 2 +35958 56805 14 +3857 38552 13 +3644 39863 8 +44106 17343 12 +40400 57226 6 +33149 41371 0 +198 57931 4 +22681 58543 14 +1423 4741 6 +13291 1194 13 +16120 12217 5 +44405 11682 13 +14650 15261 7 +35120 1821 15 +32818 18444 3 +5895 10641 6 +14868 39695 3 +29515 38927 3 +7527 12202 16 +17026 49402 8 +48768 9434 14 +3029 6314 12 +33632 41342 9 +41407 19722 9 +19479 12019 11 +52542 23077 14 +3340 49906 11 +8168 26664 15 +35968 35043 9 +8104 32049 9 +58680 22649 15 +41583 42818 9 +33577 7823 3 +15189 42332 1 +47929 46344 7 +36662 46338 10 +18269 59975 11 +12742 59924 13 +56696 19415 3 +37194 9427 9 +57564 7075 8 +43745 43242 16 +45401 17248 7 +25701 48174 3 +27144 28065 15 +37034 49898 16 +8127 21915 16 +37833 3031 12 +46443 17839 1 +13011 39625 11 +3111 58022 2 +552 36502 5 +2157 41174 11 +36556 1065 6 +18918 53320 7 +50446 1408 6 +35259 21476 5 +58292 33835 7 +29812 36627 9 +11768 14233 12 +50445 56274 16 +9399 57899 3 +18182 7807 1 +41547 7666 9 +5852 25601 6 +7849 4750 8 +51774 1199 2 +7853 10918 13 +8530 6566 8 +52391 22625 10 +25569 20779 3 +52295 49762 7 +5280 17894 15 +42649 16091 3 +17547 40443 9 +53377 59125 8 +12766 57035 6 +6081 32234 6 +48912 2004 4 +26993 55793 5 +30676 13025 6 +45258 13731 13 +41907 50654 0 +54166 55410 9 +30224 7012 7 +47687 20125 11 +7222 53492 5 +23057 1124 11 +35887 33422 11 +34424 47590 6 +3147 12449 7 +33267 49264 7 +11555 34380 14 +17960 35785 3 +41148 4963 13 +40649 38214 6 +10357 4892 7 +24810 43982 13 +36333 35774 2 +10791 22811 7 +36732 30971 9 +44204 28852 13 +12720 23021 10 +20482 27827 7 +32336 19822 0 +49423 25431 7 +28196 7745 4 +25627 15319 9 +30627 23226 14 +42327 17181 11 +35922 43265 3 +53385 1863 6 +28849 41278 12 +52687 31056 9 +328 10942 9 +51867 36088 8 +59303 42753 13 +47360 4133 5 +21363 52034 13 +4348 29785 11 +16996 14810 7 +55021 47378 9 +29681 28434 7 +8509 30587 1 +33485 16967 9 +54155 398 8 +47934 35257 6 +20538 16177 6 +25991 25453 5 +4423 6436 5 +12410 28253 10 +27239 16358 7 +17594 31634 10 +58371 53802 13 +17870 4992 12 +27551 34885 11 +438 45114 11 +35451 24697 10 +8126 45019 13 +27642 17358 14 +14143 4369 8 +24594 55079 16 +42873 7528 10 +34012 12733 11 +51090 17603 2 +56392 17478 11 +40804 8954 16 +43364 39621 13 +47391 10714 3 +59511 24930 9 +22435 14496 10 +44554 59029 11 +2955 43401 7 +55106 3261 7 +8103 13384 7 +3555 33503 9 +25867 51141 10 +57908 23031 6 +53784 31936 12 +22918 3847 14 +55320 33099 7 +30365 47459 11 +410 57360 8 +33852 57383 9 +5106 5386 18 +7344 59383 5 +35986 12944 11 +19492 9470 7 +38310 28037 3 +42362 44529 1 +54196 25834 12 +48382 17897 15 +20616 55199 5 +54462 58271 12 +46712 13599 11 +5996 18208 6 +3599 21721 7 +42024 12763 7 +8899 27273 18 +13294 51506 5 +26988 58056 7 +42292 33687 7 +10649 11941 14 +31158 5363 10 +47405 10775 3 +42167 51588 12 +47211 6629 4 +50105 48423 4 +44240 4442 5 +7283 18309 10 +52578 29923 15 +29406 24272 13 +8063 52083 3 +51595 19896 15 +39602 6062 9 +54785 44775 12 +48357 16271 11 +1407 6954 8 +17627 17185 8 +1665 12564 5 +18695 213 3 +47876 25285 3 +3060 47108 8 +23525 44041 10 +54650 16220 7 +48790 59130 2 +31435 40254 8 +21861 35588 9 +57336 59580 7 +31944 59575 7 +15024 23444 6 +39599 57620 10 +15729 7470 11 +3532 31566 10 +46274 24141 14 +2371 8537 12 +46929 11493 11 +27984 54362 15 +54884 34706 8 +29159 5586 13 +24316 33828 7 +11192 44662 14 +3793 57807 9 +15176 25538 6 +58725 1997 12 +12502 38595 6 +53878 56369 10 +21272 27455 2 +49763 33837 6 +19578 50895 10 +57553 25258 14 +12902 31478 7 +3188 26942 8 +30968 29218 10 +36121 54547 14 +45570 42085 12 +20785 47877 14 +7933 39838 3 +15186 19809 10 +7520 27666 5 +42788 16731 15 +21212 43769 10 +41947 41003 11 +8891 50483 8 +11101 47118 9 +31513 32906 9 +15660 11701 2 +22552 236 14 +44659 3463 15 +44197 356 9 +16156 3196 11 +23213 31524 10 +28586 3679 3 +9231 58017 3 +51101 46803 6 +22668 41844 10 +34167 40556 4 +9937 56611 15 +1450 26401 2 +14626 42700 14 +41674 16476 14 +51262 22138 6 +28798 40852 14 +17995 13203 2 +28762 25497 5 +5926 41810 3 +8226 45843 5 +34139 59301 10 +56499 16550 11 +3011 24448 11 +29117 13536 13 +38609 2128 5 +14114 57514 13 +27332 41526 1 +6674 51556 5 +9851 2433 10 +20776 20003 7 +39673 25158 7 +37544 15385 7 +26789 26157 3 +43589 59327 8 +16172 40073 17 +55999 40174 12 +23068 16212 14 +10661 7928 13 +59978 47535 4 +6969 29055 9 +10221 50616 4 +36035 31240 11 +59183 6461 8 +26273 34053 9 +10061 58903 9 +46 16716 10 +59138 49521 8 +16894 42961 14 +50545 2318 4 +46192 37979 8 +49779 20956 10 +16141 25725 8 +6466 47740 2 +42060 7257 10 +12775 57560 10 +51980 611 7 +963 25271 14 +1233 2107 6 +57757 6138 2 +28086 49856 6 +32414 5917 11 +37950 7356 8 +25490 21381 7 +6437 48201 15 +12739 44875 17 +10404 39077 2 +24997 59658 3 +14038 34990 2 +18811 27508 16 +36026 27919 9 +47726 48333 15 +743 39460 5 +816 15120 6 +38488 59526 7 +46301 21958 10 +14667 21116 12 +3018 54640 9 +39747 10492 11 +33857 29751 10 +51104 40742 15 +57005 34867 1 +11699 32054 7 +26834 56362 9 +41300 20501 14 +30346 58179 11 +5422 9586 11 +8695 34586 4 +25107 50200 3 +39326 56765 5 +44873 46673 10 +39624 12169 4 +30205 41802 11 +57260 29018 15 +3406 1386 1 +5208 7569 12 +17941 19375 4 +8727 6707 9 +33648 6810 5 +26023 25435 14 +51422 21680 1 +36259 19504 6 +16988 40990 5 +25795 8106 7 +47600 41913 11 +54252 59998 12 +58781 20263 7 +24711 19630 2 +14802 37208 13 +20428 18611 11 +15060 40976 1 +36268 15034 6 +34149 45251 9 +1800 37545 4 +57077 54477 15 +6588 54341 17 +13133 21137 6 +26890 54522 11 +2500 57806 3 +30621 36316 15 +51610 56803 5 +6435 155 13 +16924 49326 4 +20549 49073 15 +46061 27807 9 +33361 55342 7 +55155 15233 4 +21663 47685 10 +43880 2881 15 +3036 9278 9 +28593 26592 5 +56721 40002 3 +32717 13514 9 +17703 22952 8 +47318 10866 12 +50263 26276 14 +16035 1553 15 +52899 42751 14 +42065 596 2 +51237 52180 12 +3266 49190 10 +5494 4862 9 +40901 15703 7 +50245 16030 11 +42590 13261 11 +13565 42685 12 +55406 30644 12 +57234 19892 10 +58040 55228 7 +19292 32377 9 +52333 36189 12 +10004 41408 15 +57700 19024 9 +23924 46812 6 +51650 56599 16 +19743 15459 15 +3953 15656 3 +29655 44790 15 +2438 9927 16 +13240 44235 5 +2580 36085 4 +37925 5773 2 +47860 19946 6 +28286 53150 8 +46545 55088 16 +36388 48220 3 +46046 38287 6 +43669 46328 7 +43465 16769 6 +10934 41111 15 +53869 53619 9 +29514 15820 9 +43362 9688 7 +10129 3367 1 +23166 30356 11 +56884 55681 9 +52698 32437 2 +49127 54269 5 +14073 34610 12 +12478 28815 8 +37554 15491 0 +21676 28547 15 +30035 43074 1 +53545 20068 11 +47846 316 10 +25322 10777 6 +8890 8465 12 +18893 28927 9 +46651 38985 10 +52142 9924 10 +1848 51094 15 +44324 57563 10 +31105 57968 7 +17362 50299 12 +48759 5847 12 +109 11436 4 +27344 51168 18 +36763 54499 10 +30227 36851 3 +34171 29897 4 +14054 433 8 +44134 23656 4 +52726 2568 5 +17482 30942 11 +37241 18935 11 +59901 5905 0 +32558 13965 7 +44011 33912 13 +17088 12689 8 +22255 47718 10 +48471 42229 14 +22768 56148 10 +16760 41900 4 +57825 20610 10 +43713 9540 5 +31793 59645 5 +40822 57282 14 +4870 1249 2 +54535 16419 10 +56944 46702 12 +55654 25408 4 +39679 47710 8 +49961 34560 8 +40211 40797 7 +50942 42773 16 +13635 43900 14 +7903 8873 5 +33072 5438 10 +59307 40677 7 +9791 21758 16 +43350 1471 1 +17643 15772 11 +45962 20441 3 +54219 36126 12 +32866 17022 7 +9883 1573 6 +50407 16342 9 +50621 31603 3 +7628 813 4 +31187 21092 10 +1395 3557 4 +36749 9362 4 +27931 40965 0 +59467 22829 3 +23397 59712 2 +49815 40359 1 +11052 11361 0 +46492 22992 13 +32534 22536 4 +41130 17441 15 +9333 7266 4 +17002 8100 15 +38103 53493 3 +39377 24958 10 +33630 831 12 +30563 8125 14 +9745 16589 10 +53030 30622 5 +41911 45110 10 +47965 39788 2 +28673 14213 6 +43994 28201 9 +51040 49825 10 +11558 40720 4 +35373 31338 4 +829 51887 11 +48678 8010 12 +28855 49341 4 +30881 37533 4 +29151 43115 6 +33974 41748 10 +13641 13790 1 +50199 29294 9 +39016 56041 18 +17693 24307 3 +31302 13072 14 +1913 42599 5 +7770 32627 9 +891 37651 10 +9853 22426 7 +55687 51149 5 +28899 21081 11 +12520 32648 15 +48722 33372 12 +39690 15351 10 +46368 49637 16 +29377 47896 9 +34784 51866 4 +27142 36398 8 +4910 17734 9 +41445 38149 7 +54708 45182 2 +23100 54405 16 +23548 25670 13 +1614 15266 3 +12140 51451 1 +33428 44332 5 +30830 32448 8 +19978 41653 10 +31461 10483 11 +57905 23845 7 +57038 22664 17 +17675 11390 14 +32780 21483 9 +2320 10180 7 +27573 47852 10 +52479 24782 4 +41192 36642 9 +9928 360 8 +22474 46666 6 +33728 46384 10 +16765 54525 9 +34324 29169 12 +20624 24081 2 +20065 40160 4 +34576 21364 8 +7460 56695 11 +5138 25160 11 +40888 31611 8 +20434 1421 18 +57982 51907 13 +22744 44673 5 +11150 5067 12 +35683 53812 11 +9373 35444 16 +43690 33022 11 +41224 21186 11 +31336 6961 18 +26468 33627 9 +29675 362 12 +26648 27722 12 +48894 34069 16 +46681 43475 6 +15441 6324 11 +24908 26126 11 +50196 52070 10 +49267 33345 2 +6035 35858 9 +40136 21353 9 +27152 17920 8 +53223 9889 12 +43870 50074 17 +48132 6547 7 +22304 43410 11 +10355 33312 10 +55107 49890 13 +37803 99 7 +45930 22736 12 +21982 16500 10 +34226 24290 4 +1605 43616 1 +5218 31783 15 +55137 15452 9 +9104 58602 5 +40762 13698 18 +853 15220 13 +35719 48068 5 +53751 8589 6 +43585 45355 14 +26901 14282 5 +28997 14517 8 +39545 20871 11 +8948 4851 5 +59300 4257 7 +472 11737 4 +9618 16513 6 +52498 24820 6 +24526 50555 14 +53538 2123 6 +32512 18938 9 +30869 20629 3 +47372 25822 1 +17504 39025 14 +1234 4452 10 +31238 31597 2 +11850 13982 14 +15274 13377 4 +58588 56354 8 +10740 33490 13 +51582 57320 14 +48847 58278 8 +58084 19715 12 +17202 57873 6 +31846 14050 3 +25152 13511 6 +51254 26422 5 +57505 37660 3 +49610 11023 5 +16634 53873 4 +29100 3124 8 +225 6709 9 +57090 39876 16 +2520 11317 5 +30075 52370 4 +24779 53184 14 +25272 38111 6 +19352 3131 7 +19568 28885 9 +32698 19320 14 +20151 31736 11 +40043 6433 6 +47936 41940 14 +6172 39800 2 +10567 26318 8 +57375 23290 5 +47435 23809 1 +33362 14760 9 +21983 33757 11 +32965 2916 13 +54507 51773 13 +32682 20067 7 +29236 58188 16 +44057 32911 10 +5554 37043 6 +30697 51627 5 +52549 50824 5 +10795 5448 14 +57270 30620 11 +6804 14203 12 +17451 48761 7 +42373 30238 3 +4991 4028 9 +14680 16700 16 +50693 59001 6 +37787 33645 0 +34877 2859 9 +27319 39433 10 +31110 14336 15 +24361 5339 14 +4719 31317 7 +53728 57439 10 +53046 25947 13 +24218 37009 12 +45106 49687 13 +49959 9035 6 +46498 55658 8 +48968 57277 17 +6012 23419 5 +56951 18808 9 +29245 19878 4 +48550 2430 5 +16526 9643 5 +3967 1540 15 +17661 2247 10 +31790 58449 7 +40563 26304 7 +46120 6416 9 +9295 47420 14 +28055 19799 15 +44859 3275 5 +49183 1724 5 +49157 42886 6 +19868 35290 7 +41208 59081 8 +37347 7995 10 +2492 4627 16 +13172 32046 14 +48709 3690 15 +1489 42918 2 +39352 20088 1 +30408 49799 8 +39090 40884 3 +40319 33028 4 +22108 58785 12 +28621 47363 11 +47545 8388 2 +7630 23872 2 +5607 40727 17 +22848 43739 4 +21060 37055 8 +28188 30722 15 +59115 25142 13 +23689 17354 8 +31388 55840 2 +50692 49936 10 +50171 4608 8 +30541 39207 7 +52536 41919 15 +4188 9129 15 +36976 21584 7 +21514 29533 16 +14008 50008 16 +51869 238 8 +18034 51534 9 +22083 20395 4 +53850 30844 12 +25366 21893 3 +7705 56582 10 +31677 30859 3 +702 5419 6 +55926 2695 4 +22676 12376 9 +45095 22689 5 +11526 51527 9 +49855 14861 3 +7605 34093 11 +926 30398 7 +15952 5323 10 +10845 4739 9 +42954 12091 5 +58079 21805 11 +4173 39711 8 +41202 8497 11 +44389 10870 5 +58255 45546 6 +20232 27235 8 +17508 8448 9 +40354 675 7 +54523 45476 6 +27985 20185 5 +13481 36514 11 +14745 25934 9 +7808 55420 13 +13374 31347 5 +11881 54615 11 +54117 33042 8 +42246 57481 17 +44451 45076 1 +51366 50162 12 +24888 2733 3 +36778 35068 10 +50032 45021 1 +8462 36581 12 +45294 3020 15 +10832 32285 12 +23632 45215 12 +26362 51687 14 +37665 26292 7 +56221 31800 9 +55695 52338 14 +40095 6284 10 +43347 53558 3 +5107 7809 8 +17192 20305 5 +3719 304 11 +14160 19370 7 +46875 33911 10 +38523 35787 5 +4515 27532 12 +33956 30234 12 +3259 21571 9 +44176 21596 11 +15163 27913 4 +42778 54271 10 +41061 7245 16 +29476 30435 11 +49888 35661 2 +14428 36047 5 +32352 23147 16 +19357 40280 4 +1619 8961 7 +48082 488 12 +52192 12115 5 +49399 19794 11 +21537 36090 2 +40334 47723 12 +37807 52764 13 +51970 25246 5 +16579 44132 14 +46715 58106 13 +8721 48711 17 +34670 21416 12 +6378 26790 11 +2148 32812 13 +55771 37605 11 +13900 29945 15 +35261 29467 4 +19354 50309 14 +36971 46576 14 +40042 19513 9 +33247 28266 7 +53041 22628 2 +31857 22924 8 +36621 36228 4 +42315 47379 13 +23986 29866 7 +55302 39815 6 +39784 55146 15 +9468 13136 13 +52954 11331 16 +57488 26301 9 +56037 46398 7 +1185 57304 11 +53075 15774 6 +40592 18667 8 +27758 58898 16 +32556 15901 4 +50125 45272 9 +45165 53560 3 +32473 8191 14 +35447 55426 10 +58423 6723 6 +51077 20470 7 +36031 41384 0 +20972 50850 5 +57778 966 4 +27660 20444 11 +33732 4841 6 +55647 53213 14 +40679 25526 9 +1931 58128 13 +43884 7182 8 +52353 5148 8 +54949 59641 7 +2052 43972 5 +44785 7511 10 +49204 58351 18 +19564 44185 11 +19237 23991 8 +25241 16263 14 +44971 41313 14 +5817 38848 14 +48844 51352 1 +28926 52583 15 +24200 57115 7 +7099 22152 7 +38202 29503 9 +3779 2744 11 +23522 5237 9 +44178 57568 10 +11467 48795 12 +34418 9748 3 +48211 24302 6 +6855 38794 10 +9494 59899 16 +1108 31464 9 +26554 36436 10 +36322 13801 13 +27345 57385 2 +2013 24531 11 +50972 47010 16 +33734 18002 12 +26384 52415 9 +13842 57753 16 +52874 54356 2 +25287 48413 10 +22961 51148 9 +57542 39321 0 +54649 26781 3 +45940 6364 11 +16680 8820 10 +35465 34068 8 +39481 45807 16 +37006 27535 7 +42129 58572 11 +40470 7295 10 +56060 13205 11 +6309 39785 4 +23998 57238 17 +7168 5132 9 +4367 25554 13 +56788 24262 13 +16257 38940 10 +44402 39925 16 +36638 2486 11 +36080 19289 10 +17845 18603 1 +31444 38138 8 +29126 54244 17 +4796 56505 12 +8287 38124 11 +40208 33982 2 +59305 45573 13 +19421 39361 5 +58100 20472 16 +58227 15580 3 +31679 59995 9 +20376 15032 15 +37690 39654 12 +44665 39616 12 +27404 55627 12 +39214 59749 14 +44830 41860 7 +26397 28485 11 +23161 57651 11 +4078 32152 15 +50767 57573 11 +46795 41000 9 +53868 41302 18 +22659 39169 2 +3122 40529 5 +50530 51898 10 +10955 28116 14 +29328 20460 13 +37104 10809 8 +56941 20146 1 +58456 41744 6 +29574 31536 8 +22844 30929 1 +20045 19100 2 +49613 53909 12 +8512 17766 4 +8684 22687 12 +37270 28214 7 +22168 11644 6 +37402 40729 14 +47682 41964 11 +5619 14 7 +47345 56547 14 +4122 1507 18 +22732 12316 9 +39069 33638 11 +45865 11717 15 +15131 40041 14 +42261 51411 9 +57676 30281 8 +37369 52217 16 +4555 16583 15 +30582 52675 7 +49715 8500 14 +40360 22463 7 +43336 27647 8 +47976 35099 6 +45730 53347 5 +13537 47495 8 +47612 59269 7 +34727 28265 8 +21835 19092 12 +59820 10073 3 +47618 20854 8 +31577 37516 10 +28649 36381 6 +44851 33324 14 +28391 2060 12 +57734 4798 7 +26167 52060 10 +47798 2989 15 +6308 818 4 +41301 48296 10 +48129 57933 4 +55587 23966 9 +37872 56106 10 +3495 26506 12 +37701 5009 16 +39269 28494 6 +30493 58628 3 +57570 59169 11 +21207 29706 2 +33395 48453 14 +16738 53605 12 +58245 22344 9 +57544 37133 11 +16557 47443 9 +43207 4686 9 +41989 58307 1 +2223 987 11 +53061 24726 14 +57239 19628 9 +57697 53714 9 +17119 19472 14 +44845 20300 13 +53089 30957 7 +14326 11984 12 +37318 22752 9 +24199 56324 12 +53122 22630 9 +29458 47961 7 +12003 3278 9 +52139 50190 10 +45957 49295 10 +41941 2534 9 +51059 41230 5 +50543 12302 4 +3909 41971 8 +39426 46191 15 +57994 15323 15 +33788 16106 16 +51363 29782 15 +311 38511 12 +26243 17857 5 +52472 8983 8 +24210 38780 15 +4327 38866 13 +23131 58477 5 +29878 43325 18 +46617 44415 2 +25688 11759 9 +2558 19271 9 +43800 38197 8 +22357 42880 10 +9573 38049 7 +23245 3951 8 +49640 48400 14 +19784 3292 11 +18618 15132 10 +5169 12649 8 +50464 13259 8 +46806 7515 4 +41641 37893 5 +20122 5499 11 +9227 39980 2 +42701 33352 6 +11518 35651 1 +2311 48777 11 +24070 24547 5 +8548 36826 11 +10402 13507 7 +20930 57464 3 +7758 51278 7 +6771 28202 2 +53716 38876 9 +56004 53270 8 +17438 32640 3 +4817 7731 13 +29125 14133 2 +5385 9112 14 +6813 58783 11 +21904 6554 8 +4897 36034 15 +19654 35256 6 +22305 58058 17 +942 39092 15 +27557 38291 12 +32413 33614 5 +1147 11311 17 +16187 5294 12 +31399 73 14 +19422 32897 8 +3910 46884 10 +11744 11062 10 +51385 57715 6 +52794 26657 9 +8612 8827 7 +47684 31175 9 +41843 49461 6 +23933 42285 13 +23543 19347 4 +29208 52395 9 +57126 59565 12 +31699 30344 9 +44342 31069 9 +49917 21140 6 +14358 48540 17 +28623 45100 12 +27207 33309 9 +51930 42555 14 +4166 4727 9 +31810 6493 16 +39068 59077 12 +21812 45439 5 +20880 30185 16 +46066 1406 14 +33425 48190 8 +25384 53894 5 +14295 4615 5 +53742 15062 9 +57078 40890 16 +22262 12703 4 +42583 32060 7 +12310 13124 4 +15488 47809 12 +25588 21689 8 +17856 11820 10 +59883 47328 11 +2460 53366 11 +22263 31651 14 +6518 6929 14 +18919 30425 3 +25883 23267 10 +7582 39380 8 +6905 26537 9 +26504 13591 7 +43606 16222 9 +8083 10525 5 +34972 16896 13 +38854 10283 6 +58164 22104 6 +33816 7885 9 +54512 55821 4 +21936 20979 4 +42615 28836 9 +17394 49869 9 +25579 10055 15 +15149 47340 9 +7446 17467 10 +7924 20664 11 +873 16281 4 +8267 33593 9 +45647 32576 14 +12238 43766 7 +24886 28550 10 +24018 53034 6 +27324 43428 11 +59867 22456 9 +14757 1492 9 +56092 22508 14 +18069 40540 6 +46269 899 8 +12821 44952 11 +40413 19369 2 +26069 18471 5 +53958 6692 9 +32764 50945 11 +36575 7585 3 +58491 14616 7 +11315 1557 6 +8395 40384 6 +3332 32404 11 +48002 15200 15 +28639 17767 10 +57445 21697 8 +40035 16762 11 +53344 54048 9 +592 55584 12 +16655 11991 14 +14855 48147 5 +21543 9679 8 +17662 24257 7 +17345 28394 9 +14752 16041 7 +9156 53085 6 +57415 13356 12 +26952 13151 16 +46104 19025 12 +56086 21951 11 +49396 53257 6 +58236 368 17 +5327 21220 1 +21693 16427 10 +22534 30662 10 +26062 29459 8 +4127 26135 17 +24275 37358 10 +37174 5214 10 +38641 59717 15 +28873 7067 12 +58122 15594 13 +37695 28518 12 +55590 50114 14 +4912 37786 12 +52416 5179 7 +29996 59218 11 +54736 16512 6 +11688 2791 10 +56366 15252 6 +48205 1363 9 +39866 4256 7 +26022 43959 17 +3445 18678 12 +31241 56719 4 +18601 58080 7 +41129 13321 11 +24474 26639 6 +15727 53479 11 +44169 30094 9 +21128 17722 11 +29233 10322 4 +23119 40542 5 +42159 293 2 +57828 42786 8 +3903 5910 10 +58468 47306 4 +36301 1130 6 +9279 19098 12 +56454 40483 3 +11183 31202 6 +28295 19842 7 +45903 3757 9 +53644 9260 3 +12124 17974 10 +43685 43434 2 +23900 58432 10 +4650 52009 11 +37861 58248 7 +9191 52058 10 +51277 39430 7 +5818 22229 9 +35211 41344 12 +49179 30110 11 +3338 13476 16 +29693 6283 13 +37184 20281 9 +12154 7364 1 +43452 20519 13 +33786 4289 3 +20439 59140 10 +17080 13483 2 +34955 14716 13 +43164 10062 11 +52196 35344 9 +8266 55436 12 +48429 39262 10 +46556 47222 3 +57332 29512 15 +20562 35947 11 +14362 22149 11 +25491 12008 7 +34756 19223 12 +59434 31795 9 +40089 19918 9 +59981 7664 6 +48926 15244 11 +50037 5063 16 +58163 8553 7 +37079 53139 2 +10923 11789 11 +49369 41806 2 +30559 8962 12 +21059 44188 8 +13353 25383 3 +5230 6750 9 +55354 41550 8 +23325 12980 8 +56445 22100 6 +25119 44808 15 +58559 47294 4 +44005 55947 10 +110 8930 13 +9963 58497 7 +55142 33537 6 +31221 13380 9 +3094 15260 15 +18090 6672 9 +55678 11695 7 +49413 49549 16 +42212 50020 12 +58254 1225 9 +32921 5647 3 +55652 31359 13 +24945 29747 10 +630 20955 6 +29949 36819 6 +22285 16713 5 +1547 903 7 +28121 12026 12 +6632 57669 5 +11475 50893 8 +46541 9226 10 +11742 23411 4 +43562 58702 4 +1361 51849 4 +10499 23299 12 +22020 23081 13 +9353 43396 10 +221 33565 10 +50183 34382 12 +21164 59994 3 +43288 3652 11 +44748 13940 13 +43945 37229 11 +2688 37351 7 +44728 24333 9 +44168 40412 6 +13581 56744 2 +54646 38898 10 +10038 51928 14 +11960 48799 5 +38815 46770 9 +23836 20440 7 +42479 45906 10 +34891 37933 10 +46647 15867 10 +37816 24739 7 +45794 54082 12 +51784 21616 10 +25865 59486 18 +43477 31952 5 +5349 32697 12 +23384 7942 7 +55913 54041 10 +21632 53653 13 +59601 9105 8 +40707 50173 11 +57302 30778 14 +45081 41034 15 +14371 41792 10 +52606 615 15 +46658 46821 11 +58537 710 6 +56742 43956 16 +13983 11714 6 +32342 27440 18 +31294 34210 10 +27681 17156 6 +35146 51084 13 +50056 49202 6 +58395 5720 13 +49021 35020 4 +59612 44413 8 +9021 13498 10 +38458 11998 15 +2828 54682 8 +48081 6784 11 +47955 47990 7 +41805 10174 7 +18889 55650 8 +22723 5880 7 +17095 14424 4 +58767 8240 5 +22622 45418 5 +34914 31422 9 +45883 28790 12 +21825 5123 9 +46923 54679 12 +42561 59946 9 +24673 15979 14 +56405 270 7 +5397 1751 11 +42864 29623 5 +34199 28532 8 +29492 49493 17 +37945 18861 13 +50961 16181 7 +18981 49608 13 +25305 44950 2 +57297 21453 8 +23098 12246 9 +40213 15678 16 +35729 30100 7 +24058 49385 13 +33660 34964 9 +35859 57672 10 +39198 36260 2 +41622 13433 9 +48764 3 5 +11387 23919 10 +45207 56624 11 +3562 30774 3 +44028 25328 12 +42879 57689 12 +42034 52670 15 +48244 59964 12 +29083 44065 2 +54019 43504 9 +3458 32283 9 +15161 3428 11 +56371 49384 6 +22406 33 18 +53187 16457 9 +20680 22000 7 +3070 38855 5 +13994 13524 5 +35971 40355 6 +40060 35049 10 +33657 26749 7 +59878 59522 4 +23685 41201 10 +36019 5564 7 +29580 34178 16 +25544 47635 16 +31995 36497 9 +36447 32619 3 +10531 45132 5 +481 43486 6 +20268 56180 10 +43863 22778 11 +34073 8174 4 +37722 5712 9 +14646 59792 9 +18534 34441 9 +3433 57843 5 +6938 23893 14 +18829 59933 9 +55718 15224 6 +9610 58989 8 +36439 57536 6 +25752 35409 2 +28443 28449 11 +34514 10185 5 +59514 24186 16 +10095 27299 15 +25880 18553 8 +42297 54973 10 +16227 45072 5 +7422 23734 8 +53316 47517 14 +5616 26405 12 +40134 34108 16 +2331 20922 14 +14182 48551 12 +36230 52869 9 +59430 18085 11 +40367 20366 15 +45972 21198 7 +56402 22569 11 +15172 25221 9 +3683 4429 6 +7288 55001 5 +48218 41394 8 +6760 34121 14 +28043 34953 8 +51708 44135 14 +46653 59593 8 +50902 30013 16 +49097 15765 0 +53143 33509 5 +6676 11031 3 +43901 22945 10 +17289 546 11 +31557 15814 6 +45436 41378 6 +56584 7792 3 +21791 54046 5 +1382 43320 11 +23899 48328 14 +20793 23008 13 +32602 18815 6 +22351 21273 7 +9423 55536 7 +28239 58461 9 +54321 11094 14 +17794 48044 6 +21620 39477 7 +6689 5476 12 +47602 44716 6 +55890 22969 6 +44300 55557 14 +49111 27621 7 +51023 48951 9 +30086 56724 5 +54086 25303 15 +49167 20303 5 +42488 50091 5 +51398 27784 5 +49953 15077 3 +7023 23362 9 +7578 8071 10 +55984 54370 12 +31593 59390 11 +54282 12338 8 +11207 41032 12 +35839 13130 16 +45313 49082 11 +11739 939 11 +59262 6602 9 +8906 10937 8 +15370 41892 13 +29822 20965 12 +27689 44101 7 +15974 20677 10 +44157 44437 11 +40464 16465 9 +30963 18814 4 +4779 51310 10 +30571 46728 7 +6473 5306 8 +26377 22719 15 +2866 54583 8 +56380 51294 11 +30065 32944 10 +45111 1346 5 +57325 21359 10 +49278 3359 9 +7467 31412 6 +18848 46063 14 +47007 26477 5 +9691 58165 5 +55925 17585 4 +4368 2965 14 +17945 8557 5 +19768 34351 4 +46219 57263 10 +51629 26421 1 +12361 38882 5 +507 37251 5 +36624 46227 2 +7913 51537 14 +59843 59959 13 +16070 59650 15 +30683 21799 9 +5828 29907 5 +55602 42694 11 +12286 15272 4 +56257 2994 10 +2804 41888 10 +1520 23618 10 +6654 37917 1 +1260 1118 4 +45830 24701 9 +42937 48020 10 +14041 40922 1 +47196 12835 16 +30701 54140 12 +6789 15906 13 +21952 20849 5 +15092 11659 9 +14479 17555 11 +17159 6999 14 +18252 57639 8 +24144 12488 8 +15635 29441 13 +47183 27988 11 +30018 32995 15 +55784 56096 8 +31539 35238 10 +32912 14843 15 +33894 24364 11 +15460 40219 8 +39504 43162 1 +25228 58673 9 +57684 47796 2 +21155 45907 6 +36012 34543 13 +42456 25916 2 +45326 6899 10 +24295 7051 12 +8681 45292 5 +30263 19103 17 +27185 22271 11 +12195 9953 3 +14015 50787 18 +15353 42432 4 +16153 20925 10 +53530 49494 12 +39795 35462 14 +7554 24962 2 +57701 3798 15 +51789 45961 1 +20674 6331 1 +57327 10892 6 +53922 40235 14 +36334 58463 11 +47775 47415 13 +13754 32015 11 +36029 26509 11 +31432 6321 9 +31029 18022 4 +47005 23329 9 +59469 34292 16 +43476 8252 8 +33811 23467 14 +29079 24506 10 +19270 13973 8 +32962 18453 11 +15099 22193 6 +18416 39636 9 +49996 45748 0 +42667 11125 13 +711 54798 4 +12305 24073 17 +37317 51674 12 +51076 14219 11 +43530 18890 12 +3636 49313 18 +47744 59587 6 +26831 22340 12 +8165 4577 10 +5742 26701 13 +12736 9352 10 +57907 20618 8 +5620 25847 6 +42771 22308 8 +58571 15372 10 +39823 46697 7 +37240 34368 11 +30579 54954 15 +42037 29885 6 +17182 41193 5 +45617 45371 0 +24019 33635 11 +54423 7611 12 +37782 23476 7 +46813 759 11 +25503 9229 13 +41198 11950 14 +3435 42565 6 +7165 59009 7 +22071 51597 13 +32244 25592 16 +50065 41113 9 +7726 33089 12 +2831 12280 10 +48386 15766 11 +53266 36632 6 +24231 49635 10 +49933 21200 8 +40204 23938 12 +44544 19336 6 +20569 47693 5 +920 9383 3 +16200 4000 9 +35572 45646 10 +48642 56709 3 +57668 43948 9 +58168 50855 9 +55556 6740 7 +15775 18779 8 +22582 48814 6 +24623 18778 7 +40621 17683 11 +55078 31669 11 +59105 4993 1 +18031 3461 4 +23870 2199 14 +5334 12976 10 +21497 54698 9 +18901 13826 8 +50869 13328 13 +31909 41820 9 +7430 924 14 +59248 9075 7 +40286 47562 6 +54300 59693 15 +34340 18546 6 +8775 45974 5 +45876 2972 12 +31673 45473 11 +33113 42161 4 +35637 5598 12 +14849 43067 4 +29558 30355 6 +5053 33970 3 +38427 12744 8 +45768 14215 3 +1144 23559 4 +9281 42468 13 +31415 40479 5 +18186 50633 10 +4114 4576 6 +50000 47217 6 +58808 19245 7 +21246 5943 13 +26402 45240 7 +3983 14671 14 +36562 19262 7 +58988 51535 13 +54235 41893 8 +40079 48436 7 +7249 18201 7 +38255 33899 16 +2346 35631 5 +47359 14062 16 +32215 12961 14 +42235 16584 6 +32519 51589 7 +11448 11097 9 +10354 2329 7 +59056 33212 16 +52631 17968 4 +48818 33019 17 +24332 51925 5 +58945 28628 2 +9760 30768 13 +18289 28624 14 +50354 21675 6 +55268 7836 16 +14376 42552 5 +26173 30342 18 +44200 25494 13 +47665 42180 4 +9969 53506 17 +6502 50182 4 +6409 260 0 +24530 25645 7 +18036 22245 2 +23466 59553 13 +13225 4558 8 +51514 16507 12 +57702 35919 14 +58249 42381 9 +40242 39249 9 +10097 388 9 +2567 56055 5 +1612 27785 10 +4758 59207 5 +3829 10282 3 +20139 29786 7 +50110 39568 10 +7619 50537 11 +14277 30248 15 +29810 29049 6 +40068 52719 11 +13038 3034 5 +38514 12370 8 +46254 32592 7 +8739 10151 8 +18293 18143 10 +7769 18114 15 +29688 50109 8 +16873 53780 4 +27564 9426 8 +15859 18065 9 +2947 34823 10 +34513 16245 7 +16393 1640 6 +55867 42910 11 +2166 24397 2 +5367 22036 9 +19360 29562 0 +40383 59333 8 +39672 12330 10 +50891 14465 9 +36589 58820 1 +458 52523 8 +58190 57329 13 +43921 19356 14 +55912 22431 7 +4850 49093 10 +49525 53170 16 +20543 22085 10 +53152 7710 5 +25578 14423 13 +59070 23020 5 +26347 43621 10 +40580 31470 2 +14991 11356 5 +23162 35137 4 +33623 3172 7 +29289 48750 12 +30534 28452 2 +25129 44465 8 +22595 10098 5 +5700 36696 10 +45731 54934 10 +32765 16644 8 +4640 53756 14 +39125 37260 6 +36409 42597 12 +47620 49513 7 +17530 28246 9 +8648 37443 4 +50864 33641 1 +19490 56252 1 +24560 36510 12 +35536 11732 9 +43556 57132 5 +26529 38540 7 +44961 34115 13 +18223 4611 13 +47158 54956 3 +56441 42859 7 +33111 27874 2 +14314 58086 4 +37150 5756 10 +24352 12346 4 +24246 37277 15 +18377 59019 16 +9370 5610 5 +25966 4981 5 +30575 32286 9 +57468 57963 3 +40874 9259 11 +19083 30476 10 +9836 34232 9 +3321 47906 8 +42222 1860 10 +33779 5129 8 +49704 59770 14 +11581 19272 8 +39777 36742 10 +41397 25047 8 +13570 27381 15 +2684 57637 6 +5853 55065 8 +5163 53692 10 +39886 28387 5 +38584 48614 13 +12709 37221 7 +15360 7177 11 +15649 16224 2 +58516 3064 8 +46834 13582 3 +29232 40363 7 +10731 38298 11 +4334 29394 7 +11832 45846 12 +8630 54963 10 +2834 14725 4 +23260 1039 12 +23622 36086 11 +44001 26735 15 +2257 55305 5 +7025 37792 3 +41569 16182 12 +11203 30938 3 +43661 15230 16 +34507 49273 9 +7640 8622 9 +55265 33357 12 +7600 39605 10 +21986 1883 6 +39696 54419 7 +30450 35069 9 +5254 51302 13 +382 15408 9 +19764 55605 6 +12490 285 12 +58739 36513 9 +52168 49721 10 +10125 13866 11 +56470 34539 7 +59440 26553 8 +43698 22281 16 +32661 1281 11 +48117 43864 15 +43040 173 7 +12397 27008 12 +6273 36095 3 +15798 37653 0 +5819 11423 17 +29844 40253 8 +37322 15779 2 +21515 7091 8 +27293 22996 11 +10450 745 3 +33396 23781 5 +4063 57280 15 +21671 27126 13 +43466 35362 7 +36938 19207 9 +2 46599 9 +55752 7292 2 +15387 18610 18 +44112 5938 3 +21822 24668 8 +20317 46827 14 +25399 49619 9 +3010 35035 9 +50789 58528 8 +29583 18121 10 +52932 21434 14 +27817 57206 18 +52503 21424 7 +50375 3935 16 +3276 3842 15 +16491 32500 9 +32566 13714 9 +1113 40063 15 +2565 20184 11 +35292 10804 14 +44416 54134 5 +55460 36695 10 +3646 43178 13 +58387 11141 6 +8037 8137 11 +52989 35372 16 +46627 33839 8 +2234 15067 4 +3507 41027 7 +2876 49691 8 +57693 58450 7 +49153 52833 12 +58448 370 15 +20418 14723 0 +8134 50832 9 +7682 50095 15 +31572 19194 14 +7729 34587 4 +35059 26565 9 +10013 19376 4 +38562 51714 10 +23061 13689 10 +56909 22469 15 +55213 37815 10 +15002 16957 7 +1198 54141 4 +37125 9696 11 +8651 15382 7 +44804 54025 7 +59768 2248 6 +59777 41685 4 +56680 41558 14 +50726 10608 6 +4245 50640 2 +239 11026 5 +6039 3312 6 +50912 47783 15 +4760 15685 10 +283 50059 6 +50904 40717 11 +7791 15481 14 +28675 53473 9 +50335 34014 2 +40293 46296 12 +33893 37053 10 +8044 17446 12 +2797 11411 15 +4929 8310 17 +39872 59872 9 +7868 20493 8 +11749 4943 14 +47278 12636 11 +29113 22407 11 +3381 42188 4 +14252 17003 1 +29900 12254 9 +47871 59583 10 +40921 37678 1 +24793 1639 8 +23923 40959 10 +30040 22585 11 +54383 13937 9 +56596 10933 5 +8316 15987 18 +50484 48055 13 +11245 45703 10 +834 45014 11 +9755 52001 3 +15250 3826 5 +24365 27854 4 +12341 34462 7 +1814 49107 9 +11189 12464 13 +25623 51839 12 +10424 37261 3 +47466 47080 8 +4958 706 5 +10662 39469 4 +31680 43711 6 +13856 16442 9 +39020 13309 8 +29561 10390 7 +58418 53265 11 +24265 27318 0 +35696 7988 12 +35159 29082 9 +24212 23710 9 +11811 3175 2 +15973 54997 8 +46205 36404 13 +41453 2063 11 +14321 1330 3 +11875 27111 7 +54417 53749 4 +44118 5292 13 +42383 9081 17 +36774 10123 6 +4267 22487 1 +33414 51831 2 +28063 16706 5 +32918 50808 2 +58447 36909 5 +36523 44002 14 +58031 39107 15 +26632 15112 11 +57197 33561 5 +54268 37305 9 +29961 55194 4 +7209 3062 8 +39962 45380 14 +39901 1586 12 +48410 42280 6 +49556 26721 10 +39253 56250 13 +36701 32740 13 +1804 57374 17 +28482 41506 1 +35584 33191 5 +1179 42760 5 +43045 14165 0 +29650 6940 11 +14312 15043 7 +25923 44894 6 +44506 28904 5 +6962 50378 11 +37706 11775 16 +10637 34811 12 +6942 17954 12 +54018 21442 10 +46685 57498 7 +26414 8031 16 +772 39579 16 +15953 48772 13 +19030 53483 16 +2864 22065 1 +21303 6494 4 +2987 33410 14 +44206 8469 8 +25738 7095 11 +52146 3432 15 +25599 37996 9 +27866 11154 8 +30877 19898 10 +59527 20765 8 +8343 33026 8 +29301 19281 13 +50088 43373 12 +39417 16696 8 +31201 53789 3 +49882 5953 10 +24467 16949 9 +58024 25094 11 +334 59018 17 +4027 26662 3 +35415 41514 4 +35087 49200 0 +25143 28510 8 +32858 46773 9 +784 15053 11 +12344 46141 4 +20582 9991 10 +10908 3403 10 +27281 59015 8 +26024 13981 9 +4468 30167 12 +17273 40338 13 +54769 39251 10 +32959 31421 10 +46165 58560 1 +35862 44679 2 +15904 19903 2 +49455 38309 13 +37738 41937 2 +55048 57269 5 +9251 5425 2 +20860 42852 6 +17481 29725 13 +24501 12589 7 +47116 51341 16 +34008 20706 2 +49859 24329 4 +45922 9374 11 +27881 18886 8 +50857 49464 10 +2342 40630 2 +20147 49907 5 +33731 38970 7 +53216 18795 17 +13763 46288 11 +20091 1664 5 +8110 7635 3 +17199 40162 6 +40733 49451 14 +56592 25968 14 +15574 12588 6 +7118 24766 13 +15254 8141 16 +54016 57641 9 +54334 450 10 +4097 34076 7 +3643 38395 17 +53051 49880 9 +35648 14373 9 +11304 11813 3 +50706 9655 16 +41388 29657 2 +49163 53628 11 +29359 43177 11 +6365 24171 8 +9642 44718 11 +13191 6636 14 +47387 45676 8 +2097 24168 9 +49969 12547 12 +31407 39955 4 +17497 43270 13 +37121 54095 8 +55858 18638 7 +5950 1834 6 +22037 40358 14 +29042 59281 5 +49230 50649 8 +51593 58939 9 +2611 17828 8 +40457 11867 1 +30758 43701 9 +52521 27905 7 +34986 44551 8 +51679 13685 5 +12603 46226 10 +40098 24636 1 +19189 44227 1 +23301 29121 3 +27490 52767 7 +24022 21910 7 +36023 3975 5 +7008 2516 12 +40924 55816 3 +47337 11204 14 +24989 12724 10 +52683 660 11 +24435 53640 11 +2730 18332 5 +5229 40606 13 +56501 9612 9 +30796 27538 10 +25204 53665 6 +28077 20603 8 +20240 51392 7 +40730 55773 11 +52408 14094 9 +6183 28041 1 +11946 1151 10 +55728 37461 6 +50272 21252 13 +721 23342 7 +35813 5577 13 +15278 38466 3 +19992 45756 18 +12263 39824 8 +33407 34124 9 +28706 26187 7 +33985 42172 7 +55245 59855 8 +24226 32659 7 +45220 58251 7 +44015 48023 7 +41150 40116 5 +33969 56057 4 +8317 55133 7 +4623 6914 8 +9096 6579 12 +4181 24484 2 +15271 48156 7 +10690 26361 13 +29230 29777 8 +57061 7349 1 +35079 14693 6 +6702 30169 5 +52170 48034 7 +26130 24031 9 +23896 58443 10 +26589 44093 8 +35316 34300 16 +37245 26101 13 +19446 6856 4 +26667 22938 1 +44562 27423 10 +50885 39272 7 +40271 56620 10 +59563 10195 5 +55227 53225 3 +51972 10349 8 +10296 32241 13 +18235 48433 7 +7269 29486 18 +4939 15050 12 +10802 38847 3 +46567 59472 6 +50794 34136 9 +47686 16304 9 +40651 6558 2 +25775 32254 12 +23420 29388 9 +43478 46390 15 +33564 5709 12 +27516 29496 9 +27768 40895 2 +53409 56237 10 +51835 34442 4 +8019 21460 1 +25864 7591 6 +11541 34028 4 +16455 54180 9 +21574 17282 8 +31320 52447 18 +49129 50045 11 +24440 1448 4 +28504 26737 13 +13628 9952 11 +56256 10911 7 +40514 51460 13 +2939 15926 2 +33770 24039 12 +32581 21029 3 +8313 56640 15 +48326 4223 4 +48318 51536 4 +57707 2456 3 +39736 43385 15 +12198 52454 9 +54084 25929 10 +3167 39957 8 +29437 2960 9 +52246 41421 6 +4475 18093 5 +15239 33489 11 +5545 39921 4 +28181 9549 15 +29002 18839 12 +33106 46793 12 +32873 613 10 +51178 43575 9 +39316 55701 16 +19447 27026 6 +19526 26353 10 +33227 8944 10 +2444 34106 14 +35694 3885 10 +550 55545 11 +12002 42916 8 +47930 35232 11 +16176 34657 4 +30426 13695 15 +43022 56700 8 +58440 58110 1 +57675 18547 8 +20585 21768 4 +20698 20289 11 +1193 58011 16 +49710 56808 13 +52019 34748 11 +3394 324 8 +7679 22717 10 +30288 44214 14 +36015 578 11 +36430 48723 15 +54349 51443 2 +34494 28170 8 +36054 21557 10 +48364 8719 8 +55997 10297 9 +22947 36223 9 +50069 1572 6 +47576 21786 10 +7136 33148 8 +21669 3465 18 +30379 10610 8 +21722 23968 9 +38954 62 13 +16895 41528 6 +3333 54311 11 +47615 26561 11 +4201 8673 4 +41336 1197 8 +9785 53454 6 +36551 36576 9 +8154 4559 9 +52584 44330 12 +3078 35722 3 +35397 10452 11 +28072 32596 8 +53487 9023 7 +13243 32714 16 +30321 20555 10 +4833 26712 4 +30067 20356 11 +31609 25291 5 +48718 44796 9 +55829 40291 12 +1816 49876 2 +1062 59980 8 +1380 9518 6 +15552 22095 6 +50092 40164 10 +31357 48015 5 +38521 45278 11 +42974 53362 5 +3537 17972 15 +10238 58691 4 +791 5584 15 +42278 22028 6 +3299 29685 13 +12754 8216 16 +36773 7637 8 +51079 5011 8 +10715 14187 11 +7357 50638 4 +41665 57642 9 +47376 23943 6 +43121 8408 17 +14995 46305 5 +30310 39514 11 +24385 44271 13 +21696 59405 13 +29702 42443 10 +38177 22838 4 +57335 43301 11 +12033 5050 12 +16340 46811 14 +14735 34349 13 +42848 5549 4 +37954 13980 3 +51206 8747 5 +9287 4455 0 +15374 12803 7 +49741 40430 11 +42084 11835 1 +14181 14541 2 +50970 52243 12 +44633 41288 1 +43560 28745 12 +37326 30467 17 +57522 34526 6 +44835 3182 11 +56735 23479 7 +4698 49105 12 +19249 50331 4 +45806 37339 11 +27731 26655 14 +44710 47648 10 +34090 10624 10 +23475 30694 15 +43761 50176 9 +13501 20529 5 +41497 46801 10 +51757 13716 14 +585 55081 12 +13161 6002 4 +124 31742 8 +21440 15553 14 +10865 54918 7 +39789 33950 7 +11977 17932 2 +20489 28095 8 +32475 2858 3 +21962 49812 15 +6564 13402 17 +12262 55060 11 +19576 38736 10 +32129 52760 8 +3085 16952 7 +57915 33083 8 +43492 9066 12 +58728 8692 8 +37785 11645 7 +42592 50716 9 +56548 17247 11 +48102 848 11 +39104 8960 10 +8793 38306 8 +48487 9477 6 +580 17243 10 +59052 12630 9 +51742 24708 6 +12842 11915 7 +45926 11469 9 +55939 21006 3 +4044 38339 2 +48535 14144 9 +20478 32306 1 +52530 54118 8 +32136 551 7 +8735 59131 16 +46689 46459 3 +8937 36744 2 +46979 48571 11 +24926 27796 14 +21143 52721 5 +29589 25177 13 +9919 33085 7 +32763 17770 2 +54347 5939 9 +42087 46184 6 +40081 44422 5 +45978 23086 15 +38146 56807 17 +49293 43432 11 +35894 36062 18 +10005 28231 5 +22999 22184 5 +36840 29480 8 +43213 2445 12 +3192 6484 7 +1491 50831 12 +47370 37594 4 +45362 37671 4 +15078 40277 4 +43838 4062 10 +7195 13200 8 +525 48226 10 +12620 25593 8 +46003 4123 6 +49299 30417 7 +38649 4338 9 +26771 18177 11 +28757 40076 5 +49024 49790 9 +10486 38475 12 +3511 10990 13 +57248 53912 9 +3363 59095 6 +22428 34155 7 +43283 54111 5 +13823 47836 13 +16198 30911 8 +17942 19644 10 +27826 31429 6 +10160 33233 9 +29828 51620 7 +23671 3360 9 +14990 5587 10 +43166 52033 14 +4050 43563 12 +11171 39534 10 +13134 33703 3 +42019 5410 6 +27311 1560 11 +25311 50500 6 +27420 5719 8 +48420 45459 4 +11099 7750 15 +6164 29859 16 +22806 55621 7 +46898 59936 14 +39413 50178 12 +45552 23521 6 +6113 37512 8 +29197 47002 9 +56668 30825 10 +5295 14435 9 +13214 12605 0 +18077 15998 9 +40545 32210 7 +3000 46285 17 +9432 43239 11 +37189 31437 6 +13598 52625 6 +56806 12422 9 +15258 41343 11 +12000 55625 9 +21631 1319 17 +55911 39263 0 +29739 28146 8 +30147 47927 12 +42311 57283 10 +15242 16385 1 +19532 36318 6 +59490 34697 9 +42705 8428 7 +39332 53757 9 +58130 27799 11 +19678 28380 9 +21781 20265 8 +28255 20247 8 +37543 1922 4 +29056 7332 3 +10284 52271 11 +15415 43356 12 +15907 20804 8 +54365 53523 14 +10359 53275 13 +2119 33276 12 +11874 51942 12 +18975 35448 4 +29201 2180 12 +28049 35460 12 +40648 47507 4 +33826 22756 6 +42633 41103 3 +56327 38640 11 +35125 36553 11 +9923 38534 9 +24961 15337 7 +39022 39323 4 +38607 8538 8 +4840 29933 15 +18389 43775 4 +53397 22652 13 +18693 49544 11 +38700 11120 9 +23052 13143 12 +14675 23891 0 +48549 44070 8 +56632 21368 12 +19849 29167 7 +7775 1812 12 +50410 30292 9 +27685 28614 10 +19804 42047 6 +34482 36079 15 +100 54707 8 +22394 2118 11 +42945 52901 8 +48932 37398 11 +24297 3805 15 +50470 27125 7 +14029 6257 11 +31858 36354 9 +42849 52238 1 +33548 22188 8 +20737 41821 5 +35744 42933 7 +13991 59319 5 +54945 29974 7 +25156 41732 3 +37384 20197 5 +49662 19279 12 +8015 140 16 +46698 26050 12 +27409 22159 1 +7898 8636 7 +4270 50622 6 +1502 59231 2 +32890 12317 9 +37363 22990 10 +10396 34212 12 +48435 29766 9 +59570 48605 12 +11105 24496 9 +13224 141 13 +5305 20627 2 +53145 33881 6 +32117 38450 13 +34143 8164 6 +17790 35931 7 +36091 57403 10 +52692 45917 12 +11521 52689 10 +7147 30218 4 +21667 36113 12 +51060 30836 10 +9557 2286 11 +34288 16555 11 +31771 42875 2 +49344 8924 8 +44645 51670 17 +55269 53531 9 +23399 18920 4 +21213 26673 13 +33329 30371 14 +35071 19334 9 +8005 22103 12 +56513 35926 6 +12301 14993 17 +27475 13670 5 +34836 49199 5 +9453 14005 3 +15322 26892 4 +51360 18621 11 +2919 48037 8 +2091 19923 9 +50434 12106 1 +50999 58375 12 +58353 5561 6 +20075 26699 16 +15018 39160 10 +26758 55223 9 +21085 30463 12 +6713 46079 10 +16925 54965 11 +53119 35583 5 +55423 8765 7 +58890 14012 7 +12251 46319 8 +46273 22980 4 +16990 58752 15 +4847 5674 5 +52138 10924 9 +30214 15031 4 +20710 44189 8 +17831 34135 15 +40972 14033 12 +37291 35124 4 +49100 34800 10 +25191 907 5 +9088 37348 15 +19796 17017 7 +23854 34183 4 +57356 44208 0 +59763 10348 8 +50031 20019 11 +7669 1824 5 +12239 26685 0 +35560 32355 15 +24712 16007 11 +39665 28476 9 +15182 25950 4 +54674 16328 4 +53967 23040 4 +43731 54133 14 +52451 56066 8 +44164 58568 7 +54332 36441 10 +44658 27529 14 +48870 46311 11 +8460 44828 4 +9273 26869 11 +59096 40245 13 +14630 22890 14 +41025 11425 13 +32899 36884 2 +11838 46700 12 +8835 7032 10 +58944 47252 5 +52012 12801 14 +17404 26852 11 +24485 27934 8 +10691 23751 1 +44683 59886 11 +27005 57740 8 +375 1987 4 +47493 38065 11 +36213 17760 8 +44358 24613 15 +49684 9872 11 +32166 30965 8 +50546 25414 3 +16124 48200 2 +27342 8123 16 +28025 23788 8 +19858 19566 13 +22082 19897 8 +7625 27948 7 +22386 48641 2 +54103 20650 5 +16492 34664 5 +37126 46353 18 +22212 26260 6 +41609 48826 5 +20878 14347 11 +41057 2993 13 +43274 50910 11 +28450 19601 10 +31043 44897 0 +4554 24695 3 +15427 10429 3 +11355 644 15 +6185 27466 3 +13297 13386 9 +59689 7790 2 +13111 48516 11 +57452 5391 18 +55808 12027 10 +32515 595 16 +2297 44097 16 +55494 28111 10 +17637 12378 9 +27050 28151 0 +3583 1688 15 +22762 30738 17 +25905 14848 15 +28074 50886 10 +49110 40025 4 +49873 9609 3 +53620 53206 6 +10107 10688 2 +45404 42405 10 +3986 20113 14 +36969 50197 7 +385 11588 2 +14418 22776 7 +41590 12785 3 +16588 55598 13 +16084 8998 14 +41403 44379 13 +50898 9149 6 +33129 19419 9 +55259 20360 9 +57216 11488 15 +31025 7057 5 +12362 15151 10 +29820 12018 11 +22524 41909 3 +58838 1695 18 +1676 41807 1 +6897 18268 11 +32584 3861 13 +44866 32074 13 +11257 59793 10 +42255 54290 3 +30989 50368 8 +50288 51212 5 +20694 6147 14 +44367 26799 7 +34661 9133 8 +17815 15142 1 +54905 42673 10 +17456 9847 16 +52723 9420 5 +8611 57015 8 +58221 39483 7 +4109 42424 2 +30945 46753 14 +38861 57363 8 +49245 53746 2 +10078 4560 8 +30444 6098 9 +29244 17862 6 +17240 23266 10 +6419 49726 6 +35156 11376 12 +52884 13933 15 +45447 21577 8 +14155 36651 14 +10504 57281 13 +38681 49071 13 +56450 4498 4 +12352 5287 6 +4264 11577 4 +2255 48058 6 +59389 27161 13 +54237 32324 11 +27106 40978 17 +55164 28050 7 +29632 24193 16 +38787 45891 9 +57987 46313 13 +14442 48787 16 +12242 35829 9 +2200 27876 12 +40016 31555 10 +40142 37329 9 +48638 25517 2 +44691 18851 7 +8077 36315 4 +25423 26603 11 +10216 23536 4 +1339 31534 9 +44532 20367 14 +31653 27028 10 +19550 16625 4 +15871 41530 11 +24460 49661 16 +17097 19445 8 +15924 31003 13 +57517 27838 4 +17114 1485 12 +3745 30906 10 +45662 49474 4 +43627 3731 14 +40769 44725 2 +3542 54578 12 +7805 42895 13 +35516 29240 2 +11069 42591 7 +21879 49845 14 +48928 13287 6 +17254 30657 8 +20502 53979 10 +36699 58341 7 +31967 29708 12 +40069 10647 4 +5263 33502 13 +52076 51222 9 +37501 43028 11 +40836 33065 5 +10110 25589 10 +23524 9917 14 +29409 33957 7 +16783 8175 11 +56298 49351 2 +37882 30472 5 +29847 27933 10 +18354 932 15 +10 37640 7 +5738 17239 8 +20980 14044 13 +15886 26800 7 +56143 5136 13 +15012 22324 13 +33090 31046 8 +59424 35044 15 +659 51147 3 +41280 21384 15 +14004 14208 16 +22721 20831 5 +47258 16938 3 +22520 50249 7 +51116 43482 6 +18485 54530 6 +38820 523 11 +43422 13417 3 +50937 6524 9 +42324 57210 8 +53986 38315 7 +18382 6745 10 +57846 52312 6 +52203 28907 13 +35245 1503 15 +27962 29436 13 +11993 7199 8 +8988 3915 12 +23237 33673 5 +37858 46056 6 +21657 36 15 +7842 29128 5 +55441 5913 15 +8776 48754 13 +40610 49074 3 +21650 32773 9 +9485 13048 7 +52650 29146 8 +32708 47174 0 +25157 22831 3 +6043 9499 8 +17835 9401 9 +1641 14608 14 +13898 46913 10 +20877 49288 3 +51746 50107 11 +16864 29939 9 +53881 43107 9 +17074 51761 13 +8541 50338 4 +14696 2454 11 +914 20982 13 +32541 58817 8 +25416 36178 8 +16472 24071 11 +37915 35332 10 +15759 10554 2 +41020 26656 4 +7877 37328 5 +11471 88 4 +29052 2967 6 +19264 56551 7 +47173 9388 7 +31853 41656 8 +27227 20324 10 +28054 40406 15 +37203 40879 8 +11547 44677 5 +5520 22466 2 +56486 5479 10 +33488 48071 8 +3810 39580 10 +24816 45154 13 +31418 8218 6 +40876 40288 13 +7551 56238 13 +14781 58576 5 +25722 45604 9 +32683 20477 11 +50562 44377 11 +10674 33138 8 +24972 38350 9 +5502 55684 4 +47433 20499 8 +19332 39688 8 +46163 53676 11 +43637 16441 4 +22593 30442 14 +48017 53601 9 +37643 13138 5 +50708 38012 11 +4100 19437 17 +59709 49283 7 +16148 6765 10 +26968 25583 9 +48938 50047 16 +58981 19503 13 +20373 55610 9 +26457 57951 5 +22353 19426 8 +6101 10972 6 +20901 50913 11 +1317 18465 9 +51037 58089 5 +11156 12602 4 +59942 48639 16 +52421 25174 9 +38185 42553 10 +13737 18415 15 +52957 58467 14 +30739 52320 7 +43238 4865 8 +5192 46661 0 +42325 49140 11 +51572 57416 7 +22942 9277 6 +980 26229 14 +42643 6591 9 +42109 36694 11 +9153 46366 8 +45729 35540 10 +27714 43302 11 +30821 18873 14 +19656 53797 11 +36781 25337 8 +55375 46280 12 +50778 37047 5 +24735 18318 5 +45213 28536 7 +1801 48381 15 +45769 48408 5 +6377 25267 11 +28580 1602 5 +13397 57192 1 +23176 6146 1 +1803 13767 14 +2501 4254 16 +32027 750 11 +20120 30585 5 +24087 12363 11 +26254 7955 13 +18357 35016 9 +4060 18076 15 +4685 10090 3 +4287 478 15 +29952 20670 4 +3708 43308 9 +36984 52918 11 +20442 28613 5 +23587 45327 7 +5869 2239 15 +33599 41529 17 +18097 24983 11 +17465 21365 6 +16321 48651 15 +8429 9124 12 +54280 1741 14 +8217 31462 9 +26596 14999 10 +52586 7121 9 +42146 11961 9 +53557 38312 15 +11127 21702 5 +8458 50454 6 +57382 45559 5 +42308 47335 2 +51128 19285 11 +50157 44442 8 +50925 2638 14 +15215 5866 12 +23141 32498 11 +53418 36162 16 +31916 54395 3 +33187 40920 14 +39100 21558 8 +27421 33522 5 +56412 42980 8 +7918 43310 17 +56406 54721 11 +58493 33234 8 +18045 41593 8 +12211 38168 7 +20412 39485 9 +25979 21348 13 +52158 21425 6 +30388 38342 6 +57214 34947 4 +561 6211 5 +31133 54652 8 +17400 52039 5 +38082 47632 17 +30164 15506 6 +33039 59042 12 +15177 21239 12 +54727 41684 12 +40297 11827 17 +404 38654 12 +22920 11434 9 +53358 38277 6 +13464 5404 9 +21992 24009 13 +7367 8723 3 +11440 38905 10 +7054 3491 10 +8995 37352 13 +52946 11424 10 +31608 4663 11 +19082 5279 6 +33586 19934 10 +46848 14360 10 +19330 14327 7 +4051 15143 7 +9599 42213 5 +44123 45124 13 +12759 10752 15 +54811 29958 8 +37776 2197 9 +21491 10229 10 +8172 30951 2 +57312 23348 4 +58705 22341 4 +41257 17270 14 +7949 18035 10 +49921 22120 8 +1715 7934 2 +54055 1060 10 +42265 41623 8 +20271 32776 12 +11438 16684 8 +26029 55616 7 +14753 18486 1 +3057 33878 5 +38034 54608 7 +41580 51727 9 +9303 154 12 +1541 39779 12 +39206 50920 13 +25960 37631 6 +53765 3263 17 +34488 17985 12 +57556 42117 10 +25906 42142 8 +38290 32233 11 +6934 33161 12 +53699 680 11 +33449 6421 2 +14652 51953 10 +25087 40949 3 +5085 9202 10 +20066 1029 1 +22669 13029 0 +24554 25140 13 +44973 47523 13 +55713 33458 7 +48518 27892 9 +28851 34433 14 +15975 17998 11 +11938 13226 15 +58218 22231 7 +58738 47314 5 +50506 15920 7 +8345 31053 12 +45351 22529 8 +18275 42523 12 +44486 49553 16 +9555 12570 18 +44355 58769 6 +41279 54511 11 +38003 24805 11 +31527 22087 8 +9090 30163 6 +43719 58822 10 +30949 6980 8 +482 59127 18 +59209 42293 16 +10018 24846 4 +20962 15304 9 +16223 42932 9 +36907 8843 6 +6677 19917 9 +31127 38323 4 +31119 1514 13 +1556 49764 14 +33585 50118 12 +48066 56245 9 +44660 10233 8 +36611 42793 11 +29449 563 4 +21088 36962 5 +33236 14294 18 +30847 18897 6 +14804 10270 12 +1125 48177 8 +57219 52531 10 +12845 59175 1 +18167 30236 13 +55222 40516 2 +43764 55366 1 +26503 38738 10 +4275 59075 17 +8305 51082 15 +56705 11632 6 +43817 10337 10 +36201 38637 11 +55617 3901 1 +38085 34640 9 +54106 52686 12 +15986 39359 6 +40039 44126 1 +7431 40599 7 +15463 21790 13 +49005 1014 11 +44652 30901 9 +18060 5645 18 +13649 2764 10 +27834 13001 3 +10555 527 1 +30479 5750 13 +49530 4402 6 +8319 48608 5 +42093 35916 9 +21036 31086 11 +31764 42409 7 +42380 47220 7 +41696 12402 6 +8052 34775 3 +48805 25908 7 +55757 56923 15 +4422 3075 10 +32871 35733 9 +45231 12278 11 +15955 34778 1 +58488 22005 13 +23000 5766 13 +51441 16307 8 +48719 57052 9 +39877 49123 8 +50397 35849 5 +11323 41094 9 +31935 2911 11 +55263 4675 16 +25014 36550 13 +42994 47464 10 +40767 18221 14 +51384 51263 11 +51093 28754 18 +4461 51741 4 +33150 7035 16 +50004 53066 13 +52103 23839 10 +55851 51118 5 +2790 53235 10 +5532 41014 18 +11497 50816 15 +6764 41058 11 +34009 30505 15 +12136 42746 3 +24236 57359 7 +36018 19067 11 +3732 58404 10 +43534 18063 14 +30865 36385 9 +20227 1145 10 +43459 9293 8 +15158 15063 17 +52950 42560 13 +35533 56082 12 +7413 10660 11 +29893 32712 10 +2225 27697 1 +30290 28254 12 +38373 55158 6 +54146 14167 15 +2242 28850 12 +50935 53236 11 +48490 9123 13 +7483 28473 12 +5945 43489 11 +6037 20552 11 +23449 12165 12 +46335 40807 8 +7082 5147 2 +51791 29217 10 +6413 58729 4 +34817 19855 8 +3488 27894 1 +9099 59530 9 +44910 49980 12 +26379 8498 11 +45171 14692 6 +18362 41355 8 +15790 39234 10 +37067 47691 8 +51575 58950 9 +58618 36052 8 +56320 20520 6 +44099 8147 5 +27749 2844 7 +44537 39300 10 +4341 33688 4 +16631 42136 8 +48206 29210 15 +23002 6913 8 +44698 25372 10 +34523 34508 0 +25403 52825 9 +9816 55191 10 +20760 56035 7 +40044 12258 8 +38021 41254 13 +11174 8870 12 +24056 54873 5 +24721 37570 3 +29157 21018 8 +1311 10501 14 +12800 54802 11 +32660 42135 3 +24279 54090 3 +25873 47709 9 +41180 48514 12 +34017 30036 7 +14241 58283 12 +27904 34348 13 +23468 43000 11 +35080 9608 17 +34515 16207 7 +35494 48160 10 +56507 33415 13 +26300 35972 0 +22213 4869 7 +59277 37408 8 +49632 9316 11 +28867 58226 12 +5047 32297 6 +15788 23711 13 +42450 25924 9 +19999 15327 9 +3968 8361 15 +52829 52897 8 +27586 19686 4 +56430 2045 9 +9503 57742 15 +58097 43328 8 +36830 5253 16 +8038 41269 8 +1589 56325 9 +32083 51966 5 +53973 7043 8 +14607 15701 12 +52043 24000 13 +11216 52859 9 +51559 12683 13 +34767 3301 11 +39960 21196 1 +55007 10003 18 +34072 39743 13 +4493 28924 9 +41836 49135 4 +6041 13626 16 +3174 52981 8 +13301 16865 12 +50043 55 11 +59782 49379 13 +53718 9387 10 +32980 2226 13 +25614 22758 12 +9944 10007 9 +55151 17261 10 +54589 33162 13 +21102 53532 12 +23948 33230 8 +39565 30101 3 +30855 46453 11 +6131 58709 14 +29970 8202 8 +57981 54777 6 +33929 5667 7 +2914 14253 8 +8023 13071 9 +27513 34637 17 +37724 12392 9 +37143 59442 8 +10478 43784 10 +7888 18246 9 +55054 23328 11 +38404 43983 4 +54678 34828 10 +52598 18631 8 +48685 18115 5 +36043 11305 14 +41276 19298 8 +35217 51017 0 +39575 7330 9 +26896 21283 7 +47266 25024 7 +24166 59465 9 +34327 24562 11 +37193 57613 8 +38364 856 3 +518 753 14 +26388 20419 9 +11352 39449 8 +59126 57823 10 +24471 18587 7 +15807 7406 9 +35213 39685 6 +28955 15412 6 +32326 1530 1 +48334 27553 5 +54270 38544 11 +32699 41496 12 +20326 57892 7 +45641 23515 10 +5417 28470 13 +49776 40308 9 +26750 4726 14 +56843 35938 12 +38411 51367 5 +731 16185 12 +28237 20975 12 +25017 25669 7 +7311 20015 13 +18562 7134 12 +29983 7354 12 +52017 57572 16 +18770 38484 3 +36824 8447 3 +36037 56766 1 +5716 24143 8 +3662 28939 14 +13960 3373 9 +31717 43123 8 +18251 14280 16 +18896 3505 7 +34803 3295 1 +54493 50686 12 +55446 58532 7 +57411 6337 8 +46669 3450 10 +59371 20808 15 +29336 19867 5 +2093 46356 13 +24308 9380 3 +59619 30543 9 +57712 58333 7 +948 44275 11 +50892 53633 7 +24525 32653 3 +39027 4470 6 +10239 28125 1 +56077 56163 7 +15573 1536 8 +54927 52079 0 +30152 43292 8 +52088 37735 9 +52999 4951 5 +5144 43765 9 +19176 38519 10 +32330 49975 5 +25485 6310 10 +7762 11655 5 +11899 9535 9 +44018 48797 6 +53432 18943 2 +7817 44519 7 +8346 25565 13 +25253 24918 2 +5947 34554 13 +30363 56817 3 +3635 40523 9 +32743 23206 4 +48910 9536 11 +48942 94 12 +11054 40906 14 +5966 28076 1 +18804 34843 7 +56429 27234 9 +19366 24134 13 +12004 59366 3 +29781 39250 11 +20748 32686 7 +38025 53387 0 +35119 56185 11 +4721 12888 4 +14870 8643 4 +33523 27246 4 +45314 41702 9 +49775 42703 2 +31856 34427 9 +42262 2946 5 +22557 47891 14 +9517 15305 13 +8883 48881 4 +51033 17750 2 +13471 13947 5 +51259 34335 9 +59322 17685 4 +38128 1733 9 +21839 6982 7 +35100 26615 10 +34322 56413 17 +52264 48929 5 +18450 5517 14 +7476 31528 10 +8831 50879 6 +45109 807 10 +54412 22571 9 +31061 11110 3 +51328 22068 10 +2630 56956 9 +52235 45265 13 +58401 36709 12 +50507 16923 6 +21069 3076 9 +15183 39379 12 +30871 46822 0 +52574 40049 4 +29946 2009 4 +59498 5247 3 +15981 51779 5 +45039 51357 11 +31131 24221 16 +5552 48396 4 +20217 26936 16 +49376 42915 3 +12877 49095 11 +57972 12941 11 +23102 18240 12 +17521 45381 12 +44703 27902 9 +60 42480 9 +3918 7804 12 +2685 24741 17 +6327 16662 4 +44818 30653 7 +1486 37511 3 +15371 27819 13 +59418 36846 2 +27233 14886 11 +8604 30748 7 +53244 29534 10 +36932 49616 11 +22909 46779 13 +56838 3021 5 +31027 50413 17 +33549 7444 11 +35852 16986 11 +22074 41789 6 +9903 39476 3 +12311 47719 10 +58048 54066 10 +56229 12100 8 +16525 24389 1 +25900 27648 14 +56373 2282 11 +19556 33433 3 +54853 27737 1 +59310 38288 8 +7687 44865 7 +59121 50327 14 +2473 52728 0 +41646 10910 6 +53750 9988 8 +25226 51737 6 +55045 20685 3 +4258 40998 16 +48848 54165 2 +56971 52306 9 +13520 51793 9 +39305 29596 15 +59235 57810 8 +54310 50388 5 +24436 34948 5 +22563 14032 11 +46347 39086 7 +31058 6070 2 +39640 39911 18 +35204 41837 6 +28633 14714 8 +37037 44795 13 +54099 3233 10 +14457 51956 17 +12483 27435 6 +49496 14339 14 +27983 53289 13 +26607 58302 13 +259 2847 7 +15662 42500 9 +38385 45334 9 +4734 10247 16 +20872 44483 9 +9342 26911 4 +46421 29502 15 +12323 39597 3 +1525 4824 16 +39117 54823 10 +30141 18139 2 +56496 22794 5 +59381 52364 14 +485 12811 9 +31035 30991 13 +28634 24569 13 +10269 42168 6 +49257 17237 15 +50268 27285 14 +26204 21711 13 +33481 47234 9 +58354 57737 14 +30220 9543 8 +13872 20293 4 +10087 28343 5 +3206 20648 15 +6252 12039 7 +19651 56340 3 +30502 17902 16 +21373 34102 10 +56487 13466 14 +10899 37627 7 +37702 45152 9 +57792 42721 16 +34400 33104 12 +44669 46256 8 +2951 411 15 +49580 33966 5 +47 43680 10 +42870 50521 11 +43071 41998 8 +36561 37670 7 +28397 16226 13 +39390 38478 13 +21089 52311 17 +24599 57768 12 +33328 40310 7 +3728 14258 7 +25695 57594 6 +59785 15400 10 +22729 34506 12 +57183 9067 2 +31672 36232 12 +534 56966 9 +34682 56059 8 +36841 45820 7 +14156 19282 5 +13394 43224 8 +37845 56339 11 +58552 43304 14 +8863 52616 1 +15568 54379 8 +56481 22561 12 +893 5290 8 +14170 2519 14 +13971 13313 11 +16880 34862 11 +16117 22586 10 +845 10117 7 +6681 38833 6 +8285 29821 17 +45254 13368 6 +48178 38696 2 +39172 16965 17 +59984 15225 9 +11232 27209 3 +45113 49860 7 +6945 45866 11 +58392 55632 14 +3848 54467 11 +35090 45740 6 +21276 7187 7 +13396 26822 17 +2291 12644 10 +24414 7435 4 +46746 30359 7 +31442 40377 6 +4541 19148 10 +31463 9660 7 +47538 24511 12 +6831 27482 13 +57150 27053 12 +22266 2448 9 +54880 56978 13 +41984 59502 7 +49405 49356 13 +46001 36281 9 +47454 13094 11 +15128 49952 9 +55661 25346 8 +35788 51011 15 +52688 25028 9 +54201 51852 8 +30294 26407 12 +15243 43158 16 +29704 24789 3 +43832 39869 13 +21961 55770 12 +2654 42871 1 +49484 23297 6 +44366 34681 2 +45863 19409 13 +20112 1538 15 +54815 29548 8 +10949 11982 3 +34769 998 12 +30400 16330 12 +29154 39607 13 +53037 6649 5 +18192 17576 6 +34420 12525 13 +39541 108 1 +49587 5200 12 +5080 11805 10 +5380 36788 2 +32788 30158 6 +30572 10928 9 +25935 5978 5 +57016 45938 11 +36738 55019 7 +39136 37847 6 +11129 52508 0 +50361 14410 6 +18782 39108 8 +32967 9272 12 +24606 44376 10 +59344 33650 13 +21351 55887 16 +14112 400 4 +27966 17466 9 +7796 47411 5 +40587 46738 13 +34693 40564 15 +29818 36129 10 +58112 40085 15 +52993 18544 2 +7186 23114 6 +48458 45631 3 +13794 35237 15 +31047 9777 4 +15365 46580 11 +22554 51875 7 +17814 32219 12 +21659 38671 12 +17331 56288 4 +52891 6851 4 +21 52605 9 +1552 14596 11 +38753 47365 6 +26103 20347 3 +8813 47606 4 +18880 38729 1 +1729 23125 6 +15764 22594 5 +1544 54450 9 +10680 25881 11 +52167 27112 10 +57529 37697 6 +38731 13813 7 +22525 9845 9 +39438 19036 9 +6861 48372 10 +29928 31373 8 +54748 19657 8 +1817 9870 14 +11173 58076 15 +1847 24343 5 +40509 43840 8 +3992 27624 13 +35438 15915 13 +52731 19560 10 +5487 59038 3 +58809 20274 1 +7481 41525 6 +1770 45985 15 +35271 38159 4 +43403 24434 15 +35997 22219 11 +32047 32328 13 +20999 38950 13 +20898 20457 10 +10237 52983 8 +34211 14638 17 +4032 17081 18 +45944 47946 12 +54594 18710 16 +13315 24630 6 +11303 56603 5 +10015 48107 7 +51201 10912 7 +850 14097 4 +55184 22497 12 +2034 24023 10 +26108 3087 16 +27552 19977 7 +49262 38374 16 +56797 46394 6 +867 50958 3 +3852 42023 16 +303 26253 6 +40112 34142 11 +1663 29601 12 +20915 34597 12 +25063 31892 15 +45382 7987 10 +18858 25553 3 +3205 49502 13 +33583 20355 6 +19908 6667 3 +3625 46797 4 +59893 59116 5 +50897 9418 5 +39945 29856 14 +7818 14374 8 +10378 59520 8 +22009 10334 6 +52011 13718 9 +34454 56064 14 +12418 55642 13 +7566 42485 1 +42877 32339 3 +52251 57018 11 +14186 17190 15 +20633 5654 10 +50115 26611 15 +51253 1953 16 +37757 56653 16 +51165 36640 13 +53785 16086 1 +6348 15583 2 +10778 39215 7 +29131 31129 9 +37972 21913 9 +10998 38718 4 +40894 47422 6 +38028 38743 13 +6010 36313 16 +18274 38712 6 +33195 43326 6 +54648 20767 12 +44316 45943 4 +20488 24227 3 +24632 13462 11 +20958 54885 6 +47471 40109 8 +48346 50874 13 +53744 3409 2 +51910 51751 11 +55922 1802 6 +4732 33426 10 +47897 24728 11 +26164 3790 9 +33036 36548 12 +51322 7856 15 +27147 47101 10 +50018 53287 3 +19510 53315 9 +16062 19548 4 +20458 9335 6 +20656 9575 8 +38090 11617 14 +43284 1545 11 +49152 18888 9 +25572 45889 8 +364 52657 10 +56952 5288 8 +58021 3873 9 +20029 22276 6 +36070 36706 10 +55427 32057 6 +36486 38445 12 +47020 59059 5 +40581 23606 2 +55329 22741 10 +19496 55445 14 +19573 52471 11 +28390 6397 15 +54817 7214 3 +58259 3813 8 +29532 2769 3 +19253 36557 12 +4988 41737 8 +30531 49022 13 +15799 31484 6 +43193 53272 14 +45475 4738 0 +46354 41544 11 +22840 27620 6 +29913 18336 9 +46071 29205 9 +13407 59079 17 +23997 12787 12 +37178 29316 13 +54143 17134 1 +49116 11400 14 +37064 21318 14 +6609 33098 10 +33413 36981 7 +23490 1381 10 +27477 12233 7 +34261 40017 8 +23691 49201 10 +5854 42369 4 +54241 32796 13 +31281 15321 10 +53974 52490 11 +27960 6565 4 +55949 43629 17 +18709 11379 9 +24206 40132 3 +23589 4222 9 +41417 32221 2 +57146 52056 5 +19010 55838 6 +48105 52601 10 +5243 42894 9 +23992 1966 12 +1184 35239 15 +44105 44657 6 +39632 30725 10 +49325 203 7 +21752 17591 15 +51298 26868 10 +31963 32641 9 +14408 47344 1 +31600 34057 8 +26006 55016 8 +25903 55842 11 +53367 43816 5 +11251 18313 7 +18591 48561 10 +15760 18845 5 +41776 53905 7 +7457 53456 14 +29269 42209 12 +46268 32730 4 +25012 53130 12 +59312 3251 12 +29215 2106 5 +56462 12053 9 +31606 46502 14 +30783 37188 12 +42289 51507 15 +46981 23262 12 +30740 39035 11 +20192 16664 5 +17863 40591 12 +18694 34468 13 +51935 42104 15 +10401 10343 5 +3815 46166 3 +35679 23484 6 +32090 41934 10 +9685 10435 9 +55025 16593 8 +54626 23623 12 +53598 58884 7 +46859 45890 11 +43275 18634 6 +34887 22377 7 +33758 30528 8 +40447 4703 7 +36188 27326 7 +16286 39183 7 +47574 8337 11 +56894 16417 16 +32012 15622 12 +33250 34841 7 +32308 40093 10 +41350 8432 8 +47298 14850 15 +6057 53709 5 +24935 888 14 +50184 41660 10 +58927 37478 12 +36832 43934 13 +33017 8029 15 +31179 20193 10 +54750 59315 8 +58954 29045 5 +19653 45528 11 +1699 49033 9 +19123 36402 13 +52782 11134 3 +49625 51694 6 +12755 25664 13 +23379 5751 8 +20945 11847 17 +47338 802 7 +26796 52202 6 +56047 42706 11 +42697 33659 6 +20789 38064 16 +44216 32705 4 +32877 2896 11 +28822 59537 3 +1248 56777 4 +50813 5482 4 +49366 2103 9 +28744 33237 2 +10953 27023 3 +57674 1115 10 +9109 1289 12 +45300 35054 15 +42022 10274 14 +35609 29838 11 +35539 32621 7 +11852 13215 4 +37993 43596 15 +21630 44671 5 +17179 18698 17 +28416 3015 9 +4900 14606 2 +44678 45200 16 +2906 11536 10 +17977 50203 4 +43526 43835 13 +16901 44037 12 +24917 1262 9 +45898 32429 17 +23719 58181 10 +54820 4022 9 +16732 49574 12 +9257 28858 3 +36872 20473 5 +44717 3003 15 +7531 38242 12 +23512 44520 11 +8486 49778 3 +8898 3834 15 +1464 6262 9 +58476 15762 5 +40158 49303 16 +42446 56159 0 +55314 35126 10 +40950 13436 12 +32832 12604 9 +37625 13691 10 +9938 25717 11 +4356 4506 3 +16509 43604 5 +18947 35618 10 +49649 58643 13 +55706 4586 14 +34646 3941 6 +36780 5281 8 +13062 59078 8 +43294 27465 3 +38043 12473 8 +17607 46261 10 +44588 8904 8 +29341 32536 1 +45243 51799 7 +48875 25521 13 +41460 56351 2 +49234 33383 4 +24010 59904 5 +24310 21223 9 +28124 37088 13 +36524 3855 3 +56892 15676 12 +23366 7384 16 +34287 3979 10 +22974 57919 7 +35029 15313 9 +22418 41764 12 +38122 56020 5 +22160 45135 9 +57333 45260 12 +35067 48065 3 +32043 42856 12 +22284 35082 7 +37084 28803 5 +27079 53097 12 +39226 26480 10 +43609 53596 12 +54298 25527 14 +11989 40088 13 +41164 47323 16 +16022 26104 9 +12762 8352 12 +26524 52299 10 +50936 17531 12 +54042 13544 4 +55156 41240 15 +38730 2100 6 +19226 57255 14 +15197 22792 7 +57028 12308 13 +27839 54181 18 +36218 18078 12 +48498 58679 11 +56750 39307 9 +31616 3862 6 +35055 32614 11 +17301 51533 3 +54882 44202 15 +29971 6634 15 +19403 56032 14 +50519 40541 15 +29176 15537 4 +16603 23878 9 +55396 34278 12 +7725 7425 12 +44 28890 9 +47584 53065 14 +21257 21569 14 +43850 40156 6 +58955 26679 10 +24584 19935 3 +28080 36244 10 +55113 30407 15 +17881 59652 3 +47754 32972 11 +16466 30481 11 +29000 49232 3 +19695 50066 10 +33708 32242 10 +49573 32240 8 +9885 28419 15 +29061 48687 8 +19236 27167 8 +51016 26291 5 +23106 36783 2 +50705 29686 8 +18119 8055 6 +7677 602 12 +3212 314 5 +20614 28064 15 +32079 51470 10 +24492 59211 14 +8459 55271 6 +14986 28256 4 +5351 33006 7 +48529 51487 8 +39787 35027 15 +19173 9017 6 +50306 943 8 +25314 15908 13 +16807 45555 13 +50244 14503 8 +1183 7608 9 +58683 19785 16 +32243 32734 8 +46413 10086 16 +51468 1209 1 +30797 47528 10 +18688 24868 12 +6264 36674 9 +45150 38073 8 +24311 48417 10 +51734 32457 15 +1758 32864 16 +40143 15549 12 +23888 11247 10 +28312 20802 5 +25242 3404 10 +3471 30249 9 +8763 57745 8 +4167 22455 6 +40171 47041 1 +45356 26065 14 +34492 16569 9 +59284 50279 11 +34118 7580 13 +36435 26832 4 +21421 29411 14 +49319 19981 17 +30623 30421 15 +37661 9107 4 +35549 38213 14 +29644 44062 13 +54213 5005 9 +40067 47432 13 +3671 29241 10 +58287 1915 3 +21377 51389 16 +46123 14641 15 +47389 51841 8 +10902 54257 12 +26518 28990 16 +34038 28556 5 +7656 28840 1 +31236 36501 9 +46014 16165 5 +38176 43085 7 +42050 47624 7 +26416 49010 18 +23337 42399 7 +34259 17294 8 +39524 18075 8 +9025 20641 13 +35280 41753 9 +43936 32092 5 +53343 17027 10 +25507 13044 2 +38683 25449 8 +59086 17982 14 +4093 39373 17 +18499 19317 10 +31864 48931 2 +25547 55347 9 +45276 46167 6 +46684 50756 8 +35737 43722 8 +46230 57137 11 +54964 2325 7 +47746 18463 6 +19739 703 10 +42334 30468 8 +45445 206 9 +7680 33571 16 +58550 40327 4 +25301 26975 11 +12007 31760 15 +4791 56120 6 +55349 56156 14 +32350 33785 11 +43743 21068 3 +58518 25663 10 +40082 15373 4 +3718 54663 14 +46713 1964 13 +37772 56219 9 +4597 53229 5 +10587 43267 5 +43264 14289 15 +57117 50655 7 +10855 21099 9 +59836 7525 8 +15484 57528 11 +20046 8752 3 +48864 40202 9 +9435 42363 7 +56811 31083 10 +34162 5321 1 +34930 55165 17 +19306 39235 8 +25995 36716 7 +5424 34954 14 +34875 15097 7 +26409 48063 3 +29312 8181 10 +40896 47513 1 +53002 434 17 +59541 46275 1 +54126 39861 3 +31716 8324 8 +14454 38658 8 +47951 31686 11 +38867 41431 11 +6245 57624 2 +13780 22389 14 +16888 53897 0 +53610 7282 5 +58704 35084 11 +17950 39009 15 +38031 28514 16 +3385 17608 10 +33120 24791 10 +19931 45033 12 +36580 11864 12 +32418 34532 4 +20097 2250 10 +9884 22237 7 +6520 51682 11 +46293 49918 9 +54148 9704 11 +18630 7208 12 +2129 5012 5 +36337 1815 4 +54243 44558 8 +44167 33096 6 +3628 33505 3 +34755 17994 14 +17090 52117 10 +38525 5150 5 +20851 31164 8 +37139 44767 3 +37341 35089 6 +4150 43047 8 +52444 28548 7 +498 27048 14 +50834 38487 4 +22098 58733 11 +34537 18769 9 +50467 36045 10 +13244 50211 9 +53646 21126 12 +58599 36474 9 +42457 51712 13 +30664 35278 8 +21661 6642 7 +27198 12920 9 +31231 7819 7 +55331 13264 13 +52433 48654 11 +2206 25052 5 +20792 58169 18 +38135 49391 15 +24515 40929 9 +11132 31755 7 +51010 35627 7 +20637 14027 10 +47971 30269 6 +918 36827 16 +24577 27292 13 +12479 29916 6 +41285 2468 3 +46846 7959 11 +39669 29895 2 +7387 34752 7 +31809 25791 13 +11696 30979 12 +35554 58716 9 +21492 45017 5 +16772 54443 11 +2513 32822 7 +28565 54508 9 +14521 22338 10 +57506 32895 6 +40153 44975 8 +42069 26536 18 +25540 29676 1 +960 21307 9 +38351 48431 11 +34819 44104 17 +41504 37790 8 +21333 14864 10 +11553 59944 7 +56391 23202 6 +12333 39888 4 +4499 34938 8 +43546 39974 11 +10207 26460 7 +16955 41338 8 +2629 10725 6 +29177 23452 5 +962 19581 7 +28902 2138 14 +44525 4497 8 +47368 22660 8 +3199 15914 8 +27746 32157 4 +35185 53549 7 +33748 53031 9 +38993 57714 1 +2582 47085 8 +1441 26181 15 +10346 6769 10 +5977 12843 13 +15717 16744 5 +56660 22711 8 +18434 19493 15 +26340 6368 11 +9141 20741 7 +37814 50842 1 +27920 14303 4 +31637 7694 12 +37650 31480 10 +48295 27335 4 +39998 28386 15 +47698 18102 7 +22445 17053 7 +49446 45083 3 +2943 42225 10 +23227 2193 8 +27693 4648 3 +41967 35275 1 +6660 27581 14 +39170 52212 8 +41387 31969 5 +48166 38621 14 +57419 7939 8 +27041 49193 7 +39912 8781 2 +39339 12672 6 +48502 53704 5 +20902 15625 9 +30480 20747 0 +23574 42896 9 +23034 44878 10 +7676 6028 13 +329 55832 5 +19151 16311 9 +45790 25037 7 +11244 5028 8 +28946 150 4 +48061 23353 2 +3329 44404 7 +11506 24101 16 +4434 25931 12 +34075 5724 10 +25506 33882 11 +4594 52511 17 +11736 43679 13 +46774 47974 14 +45422 55352 2 +4072 17223 6 +50228 59060 10 +32957 42238 15 +14051 42726 8 +47123 30817 8 +11625 55472 11 +999 44050 13 +2303 11445 9 +21503 427 4 +25348 37400 16 +32183 14515 8 +50675 5864 6 +28360 27932 14 +30787 10219 9 +43859 29677 7 +57352 28610 12 +15786 23190 9 +27153 17525 7 +55243 50419 13 +40561 6983 6 +47786 20216 9 +13413 37586 10 +14715 27236 12 +36777 38809 18 +37137 45 14 +7105 9697 2 +33193 47845 13 +27507 59068 14 +51845 5225 1 +45125 41070 11 +51333 37827 6 +57657 14952 15 +14767 55473 10 +13769 46379 3 +28935 11880 16 +12968 4186 4 +47704 1237 8 +42501 15275 7 +35933 19979 5 +51029 14785 4 +30233 29022 13 +20462 55751 12 +26552 44799 8 +21204 43990 12 +56138 9151 1 +13459 19328 5 +54496 8914 10 +4463 26695 11 +12047 27941 11 +30508 36577 5 +59182 40638 14 +30129 33675 2 +28533 3988 7 +53966 44263 6 +8894 9971 0 +8588 93 7 +25215 48531 11 +11295 47364 7 +25810 56453 16 +31364 50588 15 +40274 49421 10 +16389 41865 8 +25875 286 10 +57584 3712 5 +10721 45096 11 +45709 21233 9 +4787 58438 10 +47073 42234 6 +21330 56013 7 +23339 56673 10 +40148 47086 5 +23876 10914 9 +24620 30529 13 +54612 7575 16 +23423 22432 9 +9274 24557 9 +46808 24155 12 +38379 39560 7 +18080 24657 13 +769 15040 10 +47949 22773 13 +15884 3642 7 +52819 54319 5 +10046 32524 11 +20123 38673 9 +23631 45578 12 +13439 48785 13 +10879 43907 14 +38205 7404 10 +36708 58792 17 +16871 53416 13 +28490 36532 9 +15932 24776 7 +19157 38046 15 +1296 38716 12 +55293 8701 16 +30010 38841 17 +23062 58652 15 +175 28412 5 +28341 31289 14 +41071 13249 5 +33574 57156 9 +55439 42116 11 +13447 4378 17 +31275 21111 16 +6706 20387 11 +30596 47451 8 +54451 29848 9 +19745 21498 7 +1971 23488 6 +46487 41828 10 +6777 29012 16 +27199 22339 13 +27081 36680 8 +35976 52046 6 +43232 46756 9 +48242 25707 7 +19411 5328 4 +52603 17877 7 +5059 161 9 +6603 1948 8 +21583 17635 7 +5555 3976 15 +35817 17564 14 +33566 39757 4 +25342 46616 13 +58197 31234 7 +51171 34853 5 +2821 24938 16 +54247 24564 3 +8507 4880 10 +38987 53011 16 +55034 53945 11 +33123 24906 10 +14123 22170 2 +53427 19441 2 +33916 41709 14 +42692 322 11 +20109 38929 5 +4485 29070 8 +34643 24371 2 +37747 58551 8 +50527 35045 5 +24690 59973 11 +49276 31340 12 +21902 4466 14 +40796 10443 10 +52291 10739 3 +17257 38524 7 +18649 24647 2 +24046 40750 16 +16356 45361 12 +27900 20846 0 +25892 25159 0 +19185 3230 10 +35149 48936 5 +58209 53838 7 +17253 27559 9 +12580 55111 14 +14523 13903 3 +58745 49908 3 +55585 48681 6 +27539 8583 9 +55935 46900 5 +26967 9150 16 +11431 55205 15 +24731 53423 10 +45964 1182 8 +55941 45155 8 +27166 30135 16 +50666 36030 7 +59032 21436 8 +34590 20568 3 +9252 48647 11 +15524 46363 7 +58699 28163 15 +49131 25223 6 +51273 21053 4 +29 1842 11 +28083 42411 10 +7394 7164 2 +41829 22859 7 +25963 14617 8 +12518 54338 8 +36928 44983 5 +38059 40957 13 +51929 38740 13 +53488 57820 6 +5413 4409 10 +10703 56830 11 +19077 13329 12 +34207 18291 15 +49807 23360 8 +3183 42337 9 +29296 4609 15 +6895 5717 9 +45705 53887 3 +19373 41087 4 +39398 57113 4 +3387 4476 8 +49987 56508 2 +26338 3254 8 +34111 4695 12 +1453 6608 12 +41120 42042 8 +43588 2907 9 +2953 24650 14 +24483 15095 2 +46002 28276 5 +15065 42 7 +44398 57026 11 +16208 56225 6 +23036 32620 8 +11111 50953 5 +25111 53587 6 +33403 40613 0 +10729 44535 10 +53443 22097 12 +51209 48724 16 +55071 39953 12 +45204 59406 6 +48091 31713 10 +34826 45103 9 +1598 10323 8 +32989 19499 6 +36796 24462 9 +52137 47823 11 +27824 58478 8 +32516 11481 11 +58490 9247 16 +21898 19382 6 +2237 53370 17 +44914 26635 9 +10570 38838 9 +20117 40683 3 +43963 24925 8 +24543 42862 4 +23800 11996 8 +53203 32275 11 +14902 4937 11 +6254 17802 2 +35436 59246 8 +39439 54604 12 +32003 57954 12 +45496 25381 9 +34854 16848 7 +41267 24180 8 +29345 45991 5 +4358 1320 10 +56879 41869 14 +8886 58215 4 +34487 31493 9 +3293 44972 15 +38703 9233 12 +37401 25273 11 +54123 36543 3 +18429 7993 8 +25687 51356 3 +57398 59798 5 +36961 18181 8 +1383 58549 15 +9979 30148 3 +7716 24441 7 +56149 37612 8 +34170 21968 7 +1313 44974 13 +26138 50323 14 +20096 29376 10 +45297 13345 6 +57284 31915 10 +22400 51974 11 +40124 45041 8 +24085 1035 3 +5039 57730 6 +17529 57516 10 +46972 52239 18 +19828 32268 6 +22405 8503 9 +13362 41650 16 +53775 54667 15 +32924 54745 11 +40756 23071 13 +57802 32175 14 +12690 4556 0 +17666 21306 12 +35222 51350 10 +35923 2162 1 +56203 43736 11 +54373 11994 8 +49197 43986 8 +629 23594 9 +30262 10051 10 +22472 50256 4 +14368 56606 5 +54017 57623 13 +21150 40407 9 +42816 13662 11 +41804 5857 9 +1747 23775 8 +520 33689 15 +37420 7417 11 +10036 27064 7 +40449 34200 11 +36461 34684 4 +8517 16562 8 +36246 50138 9 +41643 4004 5 +8629 11612 7 +4354 25277 16 +37975 32760 12 +34410 56913 15 +51847 54524 4 +45331 27936 1 +16877 25773 9 +9636 50469 9 +9943 32672 15 +21248 6864 2 +6527 16423 5 +45346 41811 2 +7307 5418 16 +26127 54174 6 +38804 26694 6 +46339 44815 0 +46534 19414 7 +15876 20746 5 +58083 10208 10 +42967 30485 3 +15564 35264 8 +35504 872 3 +51289 3744 17 +41691 13954 12 +43086 34448 14 +52627 55799 8 +59232 9073 9 +23557 41040 9 +54214 40883 3 +25926 16277 15 +35474 37550 14 +52759 53892 12 +38803 16913 8 +10961 22846 10 +45809 5048 3 +6695 14565 13 +31266 38245 8 +4984 52809 7 +16398 18661 10 +50290 55982 4 +36715 36339 6 +3097 47649 8 +19361 45115 0 +20842 22505 12 +12071 15934 1 +17779 3627 2 +19814 11178 10 +30142 3083 15 +51544 55525 3 +20828 39141 9 +2754 27077 11 +40670 45913 17 +39228 43013 2 +57076 59941 8 +10475 56595 10 +44474 46959 8 +56314 34370 10 +55944 14672 9 +58574 53349 16 +43532 39279 8 +27352 48298 11 +35704 47882 16 +51448 8264 5 +41357 21858 4 +57019 41482 12 +26439 50040 14 +1455 12579 16 +12228 26391 6 +25013 15080 10 +40097 53828 7 +9580 55880 14 +45586 4015 8 +19057 15610 15 +51285 5097 4 +28698 21156 9 +41416 57482 12 +26316 8915 17 +47542 34686 7 +54739 4010 4 +25946 59020 13 +40432 43337 12 +46826 3262 0 +9412 46989 8 +27337 31350 5 +39828 53039 4 +43260 40154 11 +55290 59004 2 +5952 56899 8 +30070 52505 4 +14642 55631 7 +32483 52905 17 +8120 10712 4 +21529 9997 14 +20353 19693 8 +45777 45126 7 +15550 53970 9 +21082 25537 1 +29905 8372 11 +0 19671 6 +24624 8331 2 +44631 23905 14 +22259 53353 7 +25274 34626 6 +34431 59147 14 +25183 46168 15 +16814 38925 15 +46612 46359 11 +27841 21481 5 +46575 2312 10 +19232 17483 9 +2774 46854 6 +46481 36937 1 +28929 59741 11 +6142 22301 8 +12694 53226 9 +187 33693 8 +1110 25728 7 +56842 13574 9 +40697 34341 17 +53430 17495 7 +12960 2221 17 +1716 20413 4 +1468 47770 6 +23562 56575 9 +44248 42972 8 +12474 28827 3 +10925 27773 15 +42421 5276 13 +9349 21230 9 +25917 29727 13 +24339 31981 7 +13031 9554 10 +50940 56622 12 +6653 51251 9 +45664 53504 8 +48733 59692 6 +2414 34237 5 +1084 15447 7 +14987 46182 11 +48451 48004 11 +31097 45589 9 +91 36646 7 +45580 55209 8 +32216 8940 8 +24691 4213 16 +37644 39675 11 +17187 34866 3 +19163 22740 16 +41116 55503 9 +54776 3335 11 +57606 1993 7 +13853 17776 7 +10490 26411 13 +58811 16727 11 +15495 47502 3 +29309 34201 9 +1259 5142 10 +45735 5478 16 +32045 59902 5 +30702 15599 12 +55620 36977 12 +53436 30580 3 +46259 1329 10 +10526 36058 11 +21535 57824 6 +26372 17864 9 +35577 26752 13 +3884 10012 8 +18736 20211 11 +16652 48576 9 +46598 33922 5 +46173 44668 5 +107 36389 12 +22898 47996 13 +27460 52324 18 +51100 19902 2 +42528 31604 7 +52743 14775 9 +47438 51940 11 +57451 1542 8 +5579 5902 9 +28327 26557 12 +50515 2428 14 +50991 44343 10 +49150 968 7 +45519 57706 9 +56527 47156 9 +26433 55958 8 +32279 28829 14 +41761 8392 14 +3831 2462 2 +50096 47013 10 +36347 8672 11 +4309 42491 7 +12423 10326 6 +52205 46078 8 +44304 5101 6 +33101 4693 17 +33420 190 7 +34089 59594 12 +57092 50420 5 +20156 11462 6 +54503 41644 15 +14447 29817 8 +42659 40693 6 +1494 31165 6 +53652 58805 10 +58849 26124 14 +47453 39894 11 +38130 19834 6 +36579 50699 9 +52065 24137 13 +17174 31030 14 +51850 58500 8 +48172 59518 13 +38727 4842 16 +3574 9861 10 +45879 34732 16 +1933 24340 11 +46321 4744 7 +16567 42440 3 +16215 35247 10 +57944 42197 6 +54692 5248 9 +29318 54546 18 +32978 12838 9 +23105 29350 11 +5898 48478 13 +1220 45105 5 +30131 159 5 +2624 383 9 +12743 35778 1 +35635 25011 11 +13977 8189 7 +5956 51526 13 +25026 47985 8 +2506 11389 7 +14758 49813 4 +9307 15263 9 +40935 12887 14 +14416 55391 8 +6407 7581 11 +13924 27623 15 +2427 41582 12 +52863 19650 16 +39165 13068 10 +43508 47792 11 +26131 28654 10 +50512 16534 14 +58527 24248 6 +48185 32930 9 +618 27570 4 +29577 23663 7 +31036 58303 10 +2605 58403 10 +5755 33907 13 +11870 52883 15 +42052 24742 16 +12406 34690 16 +53371 19475 9 +12322 58325 8 +33626 27247 4 +58976 36949 7 +47923 1402 5 +39940 2862 11 +10301 29850 8 +26946 52081 7 +12627 10733 16 +24409 56283 8 +47553 13351 7 +13281 54087 9 +40053 13939 8 +13623 46077 1 +6488 41673 6 +20431 33258 12 +28985 31739 3 +40939 26134 13 +23647 22459 4 +37123 5125 18 +35342 2935 6 +39622 41038 1 +4596 31248 13 +6732 49606 7 +10431 13393 15 +18958 51675 4 +19952 15283 8 +48778 46217 8 +54416 46620 3 +53544 49224 9 +9377 7007 3 +34303 9172 8 +52383 53688 12 +23367 10481 6 +951 48306 2 +4352 14764 9 +49702 39308 10 +3990 26113 9 +24837 30799 12 +47999 26343 2 +45420 13624 4 +9920 15690 11 +19627 20406 13 +43212 16268 10 +51175 37870 12 +11328 6879 11 +30679 50526 3 +42408 37934 17 +16948 55411 3 +55464 22176 18 +34782 20528 6 +25442 50240 2 +59874 17717 8 +47355 25637 18 +57042 48873 11 +836 231 6 +36668 8430 5 +7845 57466 9 +28953 50318 12 +37636 42809 7 +38957 17901 7 +36834 16740 17 +37547 6049 5 +55091 27772 4 +35962 37569 9 +39058 30763 8 +1846 35944 10 +30304 54031 12 +48756 25948 9 +29637 44264 9 +24836 19150 3 +15615 4056 13 +20371 2514 9 +19309 18178 6 +9305 6891 14 +42957 51345 11 +57406 50865 15 +57107 57313 9 +39950 47546 9 +25767 37732 2 +28229 21544 9 +19579 5629 9 +30401 40744 9 +54851 14407 15 +17305 21626 6 +46385 4403 4 +21737 50274 9 +9363 40603 10 +18348 31833 17 +43461 39000 16 +58494 11326 3 +8762 39487 5 +58317 5595 11 +12161 30328 11 +2662 31930 7 +30976 39237 3 +36537 54775 1 +50312 49148 5 +28811 20686 3 +29439 43366 6 +26363 41839 6 +59642 39118 8 +39744 19545 5 +49527 29225 8 +29783 28073 14 +53192 18776 6 +37473 29200 10 +3715 16920 8 +45091 27438 11 +52051 44774 8 +24781 23350 12 +51932 934 8 +33592 43261 10 +14697 18998 10 +16087 51747 9 +59670 19914 14 +25388 20191 16 +25536 21034 8 +32450 37490 6 +16775 24968 11 +35081 14914 9 +58591 44242 2 +21295 28979 9 +50955 25478 12 +14194 4781 8 +11597 31837 2 +58274 52655 9 +33805 48047 8 +14389 38826 7 +59734 20401 8 +33210 22442 4 +51868 34104 13 +51807 29920 16 +45523 30161 12 +12469 54788 12 +23942 46395 10 +32854 19971 11 +45336 30284 6 +42080 24732 8 +44872 34029 8 +23078 43746 9 +40947 58356 3 +19097 31780 3 +32184 53748 2 +51483 53630 4 +10846 9059 5 +7560 56137 4 +35376 38752 6 +24156 34744 13 +8563 58533 11 +47304 5412 11 +36056 7265 4 +25886 8650 9 +28901 32140 7 +16899 48871 5 +49636 18672 11 +14884 40778 9 +12764 35998 7 +47332 25022 9 +59609 21459 12 +58603 26724 7 +33535 58441 3 +15611 26450 9 +47369 5356 6 +5765 26512 18 +41840 50286 12 +16572 27865 4 +3864 58405 13 +36072 19331 8 +28040 14526 11 +30612 50389 5 +2851 8304 2 +24442 22895 12 +27793 6990 5 +5807 23664 8 +51172 33078 8 +40902 42320 12 +15951 9497 12 +47504 9356 14 +58632 14754 13 +21538 55781 14 +3809 51402 6 +49397 49926 4 +12111 27782 12 +24006 57418 10 +58707 28774 12 +16274 47308 12 +48946 3351 6 +1146 3219 11 +59174 32887 10 +41429 41688 8 +49590 53093 11 +44907 54946 6 +2251 14533 5 +46317 55183 4 +4016 59275 5 +56954 35023 2 +29119 16405 5 +41694 10975 1 +34791 35392 10 +53238 40072 15 +50769 52307 3 +5326 14826 15 +34738 20266 13 +34279 39797 5 +19852 18781 6 +37740 859 5 +19756 26399 11 +17759 39120 6 +976 4789 12 +16137 4800 15 +24347 21044 4 +46043 6392 12 +38125 9644 8 +58471 10017 11 +56969 1127 7 +50648 7514 15 +23760 14508 9 +25036 54893 3 +12499 29948 8 +42566 49575 9 +3772 42333 10 +2553 38112 17 +15941 29303 16 +31348 45525 10 +39995 22767 11 +46188 50632 6 +14119 18244 4 +7563 51623 1 +53401 46491 6 +23032 21268 6 +11770 45147 5 +49742 48538 13 +17091 21836 9 +46326 18361 3 +54324 35008 5 +8213 12897 8 +10929 33749 14 +47051 56162 14 +37399 29622 13 +98 49740 12 +43389 10293 2 +27354 44336 7 +47014 58570 7 +33780 28650 9 +35026 38749 6 +14906 37412 3 +3250 35206 13 +24064 24201 13 +44940 47788 16 +22073 24834 6 +45500 49378 5 +24738 12710 9 +25002 26528 12 +42532 2089 6 +21829 6227 1 +25872 20280 16 +41871 25297 13 +16667 16003 9 +53322 17082 5 +11884 33061 18 +3764 57134 13 +4087 16725 14 +48308 22007 14 +30486 51202 5 +52962 48203 13 +33401 4894 15 +10978 23941 9 +17946 19355 8 +32370 23571 13 +38286 25877 11 +18704 838 7 +9268 37919 12 +30347 52282 14 +21157 57087 13 +55425 4538 11 +11249 44890 10 +2562 41745 11 +29990 12145 3 +47942 55149 10 +46510 36114 10 +42605 44597 11 +31838 51403 9 +6017 49470 13 +10895 25827 10 +47351 45724 11 +9404 47672 2 +27806 15826 6 +3846 4360 13 +19466 26295 3 +59082 2232 10 +19511 19666 11 +38332 28315 5 +26140 3626 6 +1596 35546 4 +19417 20782 15 +56996 59106 13 +55028 4606 12 +51442 12823 11 +8980 15293 8 +44033 7882 2 +16604 8882 9 +20560 14878 11 +48175 56534 8 +56560 25841 15 +14737 29067 5 +53661 36983 17 +7300 48323 10 +6082 28747 5 +41782 17724 13 +27951 47582 8 +24287 414 8 +56433 22359 14 +14319 5711 0 +44346 19206 12 +41567 3494 8 +37684 33440 8 +28022 3856 10 +3071 30982 11 +51321 46605 10 +30820 50544 15 +601 55208 14 +24604 8601 13 +57376 26839 16 +52262 52335 3 +43124 32879 3 +42665 41676 14 +45826 8982 5 +45301 46231 17 +37603 40163 8 +45405 54093 10 +50450 18483 8 +37918 40869 7 +19165 35877 11 +13305 37969 7 +32061 14000 9 +7052 20658 0 +45721 32963 9 +35399 16469 7 +25814 30781 8 +4171 426 12 +34198 37049 11 +55256 2566 13 +12713 8282 11 +17259 23661 8 +27781 45003 7 +733 14056 1 +21027 29374 5 +36161 51573 9 +4603 32322 7 +42199 20931 9 +35585 13359 13 +431 8419 11 +12208 4304 12 +13032 47694 15 +25824 49686 14 +25611 58474 10 +58487 54490 8 +5984 14933 10 +4857 51210 6 +50985 52960 12 +2015 49848 1 +10377 26971 15 +34721 37395 7 +18607 53260 8 +34429 15954 11 +48971 23834 9 +4861 47868 2 +11836 47089 12 +14580 3794 15 +54351 12958 8 +37075 55311 16 +35961 24995 12 +35698 13087 7 +44881 41274 11 +55340 25811 8 +56109 58967 16 +13409 52846 18 +27818 45000 8 +49352 16697 7 +33015 47778 12 +27873 58687 16 +18122 44653 9 +52747 12949 4 +46403 28971 3 +13858 52214 15 +38857 51856 14 +21496 48822 5 +44190 46662 13 +50202 24007 13 +30969 3656 4 +6051 58368 10 +13137 51858 8 +14478 23049 14 +8658 18106 15 +401 49481 11 +5489 16647 10 +54130 59220 12 +55392 30026 16 +9187 31 14 +20218 45941 8 +32645 1158 10 +55787 59595 1 +58541 12852 12 +1857 7752 1 +26007 18248 5 +35558 39397 4 +20876 10705 8 +48116 28627 6 +30540 22970 7 +15658 6387 15 +48470 1564 11 +33589 31124 11 +23145 52940 12 +28150 4938 8 +12463 11342 4 +68 4384 8 +31861 28402 7 +8043 35466 7 +52990 47759 3 +2240 22904 10 +35295 35832 10 +28635 12082 9 +48219 34860 10 +39853 50861 10 +44490 6154 6 +10008 39298 9 +470 1284 10 +26742 30887 16 +1632 28090 10 +11381 38903 4 +297 47861 12 +53285 18719 13 +12729 49659 9 +35129 45025 12 +51232 18082 15 +33478 40760 4 +54044 18368 13 +55819 8431 9 +55797 34909 13 +32666 30017 7 +1428 49178 9 +42729 43748 11 +16487 33917 8 +56565 12626 8 +51886 15088 11 +29348 6770 12 +58708 56105 13 +23726 34483 8 +40285 33334 6 +2321 57476 18 +46733 52807 6 +24309 22436 8 +9256 22973 6 +16917 52867 9 +737 19524 8 +49372 57914 10 +57923 33550 13 +43500 16726 7 +23117 58167 9 +41211 4946 7 +48367 32008 7 +24677 39810 10 +59314 55943 15 +42618 25251 9 +40168 15340 11 +17267 396 12 +18116 44763 10 +12275 5042 16 +45196 49578 11 +23133 26953 9 +4269 6298 5 +43332 1713 9 +38864 41494 8 +44739 52982 10 +26344 47320 11 +55234 45464 10 +20540 50101 3 +19265 16111 4 +7085 37867 16 +54637 52836 4 +18424 39036 6 +44523 7488 12 +40681 10456 7 +36209 57857 6 +28875 27124 1 +57539 25149 17 +51490 33658 1 +55664 18756 9 +18255 4858 9 +41842 58090 13 +37905 39881 11 +38909 17125 2 +6900 39576 10 +46054 57358 2 +8522 3971 9 +30609 19506 7 +5882 18916 10 +19680 6205 11 +21206 33068 7 +56865 40585 10 +48401 15965 2 +1398 10947 7 +16701 2344 9 +21943 29416 9 +40682 10114 5 +53759 5969 9 +47408 42354 9 +3525 18219 5 +19721 6526 8 +37587 23577 15 +52729 27029 12 +28601 15016 11 +38702 28016 8 +43414 7616 14 +3771 26753 1 +20712 39986 9 +58913 23023 1 +54236 18606 1 +13110 53123 6 +35600 58479 2 +4936 51364 14 +49433 43307 10 +19323 50106 9 +50696 32665 3 +35117 39194 5 +9166 30689 6 +53103 47970 4 +49713 37246 6 +13460 42723 11 +42038 18520 6 +39887 24770 9 +23733 25307 6 +56451 55241 15 +3154 17987 8 +13915 44642 12 +55197 53404 4 +28498 25104 14 +12925 59051 9 +23732 44161 9 +48580 15839 15 +32901 46971 5 +8358 15942 13 +15380 21066 1 +122 38281 5 +35523 47969 10 +6108 32447 2 +24045 52784 7 +4208 48359 8 +5030 55325 13 +32655 19701 13 +665 22523 15 +2825 29597 13 +50206 27302 10 +36703 37693 13 +19174 11398 13 +4207 32548 12 +16000 26643 17 +53162 31711 5 +24645 51439 1 +58856 12600 17 +22801 32829 9 +31722 28294 5 +775 22031 9 +4829 11088 11 +19577 32455 15 +5594 17222 16 +50348 6243 10 +58834 11142 13 +21872 52250 17 +59675 32230 8 +44816 19274 11 +14700 43607 11 +39245 5878 13 +42844 12933 10 +15474 14624 6 +41320 54109 4 +21314 56333 1 +45754 37834 8 +30992 24148 0 +39254 32798 11 +54368 41906 4 +59458 22533 10 +9639 48155 11 +38616 3741 6 +23772 14649 9 +32305 38981 7 +37628 12090 14 +46154 33959 16 +18733 26677 15 +338 17347 13 +51995 18560 10 +33457 51754 9 +32474 9408 9 +346 5124 13 +12574 22746 10 +46874 25953 7 +31485 48547 7 +7881 12597 10 +50097 16795 15 +57794 23163 17 +11550 56293 13 +31257 42889 12 +56311 58362 4 +52607 20840 3 +2469 21687 4 +4752 45375 7 +56555 34238 7 +13330 14275 5 +10181 36111 12 +57764 41642 12 +8728 37105 8 +18867 18160 11 +12160 25374 15 +42339 2228 4 +47203 10397 12 +15614 5659 4 +34790 55288 9 +33388 50639 16 +47781 7649 3 +45238 17957 12 +55690 4453 4 +22841 42827 14 +24082 43631 9 +10655 39610 14 +3581 47409 10 +21284 20249 6 +14104 8826 13 +58305 46094 10 +3190 9556 15 +55362 5078 14 +57457 34440 3 +27897 24245 2 +27887 614 11 +34839 24108 8 +9244 3423 13 +37984 14024 9 +31601 39572 6 +2702 53701 17 +2496 25312 6 +16754 46018 10 +43717 4797 11 +48176 32338 1 +58545 15320 10 +41867 59891 8 +9961 31287 11 +2055 28224 10 +56337 11843 11 +12905 16614 7 +47145 28632 10 +21705 5109 13 +1410 48390 10 +37083 44273 4 +23739 13480 9 +54000 21080 8 +30674 2457 9 +30474 50163 15 +54330 56191 5 +40179 45942 0 +428 41395 18 +15536 2498 7 +31106 20071 6 +2998 58630 5 +37479 35038 16 +6105 40411 8 +55837 30000 3 +53360 14102 0 +49447 53562 9 +5034 3345 9 +20690 26908 16 +6918 42731 10 +13015 39315 10 +44285 39389 17 +17570 33815 14 +9317 56001 10 +35605 45995 12 +25657 16904 9 +33045 8921 3 +31337 42686 11 +46750 21945 6 +57186 52669 7 +25496 12031 8 +3044 40994 18 +50768 28320 11 +15025 47804 8 +15103 1459 9 +3729 19736 4 +3533 48310 13 +25726 14768 11 +53079 16875 12 +34189 19183 13 +49115 917 4 +22155 10088 12 +33305 45372 7 +43565 4183 11 +41491 23996 8 +28567 47761 8 +18068 55745 6 +13064 35518 14 +7958 43574 9 +19141 15433 16 +50737 35678 6 +5833 30430 2 +42077 21876 16 +31854 43648 14 +1102 21209 3 +29649 4296 12 +9345 32781 8 +57893 27742 8 +48316 58199 12 +55960 49685 3 +56349 32885 4 +54318 16149 11 +9048 48908 1 +51447 43558 9 +53814 50950 3 +59010 58787 9 +36765 33620 3 +57500 28017 3 +23799 29752 12 +52971 43278 6 +55092 45618 16 +46678 27810 4 +44906 14898 10 +34792 57194 4 +56444 7382 5 +28160 15007 3 +24342 3904 15 +23579 29511 8 +52941 36214 5 +54746 6703 11 +56697 47038 11 +50113 15562 15 +26876 44692 5 +10291 52484 9 +35393 40404 5 +36249 38515 13 +13446 29141 8 +55153 37710 16 +54596 49783 13 +53111 40615 7 +37887 3405 16 +42756 15037 12 +11569 58830 6 +26734 46987 16 +16529 6544 11 +49480 10957 6 +42610 26063 7 +13104 20375 6 +25199 19839 5 +23650 35708 8 +56685 24276 8 +33746 18635 17 +25257 34857 8 +37795 46680 3 +4361 4634 12 +2961 51135 7 +42652 59153 6 +29940 43873 14 +13146 48866 10 +48496 6912 7 +11786 34097 7 +24040 25265 9 +51324 42268 9 +59435 27004 16 +9086 3807 15 +8453 58813 6 +14570 4632 9 +16806 7080 12 +24998 23635 18 +39977 30852 2 +24609 10750 6 +32826 45929 8 +21507 43050 2 +14411 14602 8 +12227 42264 7 +34717 28883 8 +6586 29356 4 +31917 33235 14 +39521 48207 9 +36994 38482 1 +31573 9895 10 +14398 21304 11 +44044 56950 9 +7638 44006 13 +55830 58888 8 +41886 44399 4 +51565 50253 11 +26853 37802 1 +53639 24815 9 +25096 25323 14 +10973 10678 11 +19871 38039 1 +45680 4511 10 +1427 49248 4 +37849 2815 10 +46091 14023 7 +51780 24601 8 +18006 377 14 +58690 48693 15 +15929 50254 8 +50617 36954 7 +12844 23698 9 +30704 48104 17 +35414 51211 4 +7723 52031 5 +59072 52384 12 +59492 21270 9 +49993 49243 2 +13084 14260 10 +54492 52630 10 +44635 47950 9 +26510 57546 11 +54761 27380 7 +50003 57508 15 +14882 1703 5 +26563 35592 7 +36310 14799 11 +44629 4882 6 +27180 23868 13 +50319 26804 12 +55696 41368 8 +35524 41031 10 +25866 13772 10 +21125 13894 11 +52404 45044 12 +8540 11314 4 +56644 45771 7 +29854 31560 11 +24748 59754 9 +13583 36595 3 +31533 53467 8 +41680 3215 7 +1859 37930 8 +3791 48466 6 +7764 10261 1 +47244 43772 14 +22080 15848 12 +41877 33980 6 +35142 16693 7 +17757 39392 4 +27309 15588 3 +9628 59481 5 +27844 26916 9 +27058 29608 10 +10987 31379 8 +38247 37165 16 +11487 41333 7 +40380 37334 7 +24747 34815 6 +44742 50049 10 +3024 37928 3 +54091 41635 5 +41538 11358 7 +1937 59805 9 +25278 46767 8 +5688 53477 9 +54733 1341 9 +26018 49453 13 +26017 9302 8 +8059 2763 18 +3325 44722 12 +54567 48771 13 +57997 49548 9 +4546 30685 12 +41541 36854 10 +8590 13504 7 +15693 43335 17 +51828 21815 3 +4645 6220 9 +29641 17873 5 +59917 40608 18 +53109 19582 12 +12077 13411 4 +46047 22264 7 +29310 40597 7 +19642 12780 10 +32603 39402 6 +16559 38966 12 +4305 51479 8 +51140 20079 17 +24680 34970 10 +50983 59957 8 +57114 40936 6 +25502 58889 10 +10364 3724 5 +42126 23293 2 +4048 50969 8 +27220 34562 7 +30918 33450 11 +3703 1054 8 +4326 52304 16 +51317 39738 6 +14293 51346 8 +35477 17655 8 +48241 23382 8 +17390 29196 8 +52331 47291 3 +52755 3799 4 +53686 8255 9 +17176 36358 12 +24757 9593 14 +53811 46562 8 +56929 40589 2 +10534 37623 10 +9413 5216 5 +57160 9852 9 +54428 56905 10 +574 44233 7 +3957 33944 13 +39105 40324 14 +27063 52893 5 +52336 55279 1 +38212 22828 1 +39907 12645 6 +34615 345 10 +18477 19758 6 +26185 32946 9 +19833 56920 9 +18396 44475 10 +11306 4934 8 +7215 9807 13 +48085 51896 4 +634 59818 10 +38897 50670 7 +25608 18758 7 +45804 4566 8 +29464 3140 10 +12020 44203 8 +4260 47250 14 +47894 28866 13 +34377 46988 17 +54071 3689 9 +27611 56964 15 +5763 30745 12 +45504 7595 5 +14818 22412 10 +59647 28505 4 +31219 39746 7 +52656 41868 7 +37589 50247 7 +6021 8896 1 +12828 22930 5 +12036 42960 10 +24184 55876 9 +39975 38276 1 +44509 41770 9 +42509 35771 9 +27922 30780 7 +25095 27358 15 +57485 40572 10 +1772 58239 15 +11152 59415 3 +57144 46304 5 +41024 6772 12 +47159 54940 11 +21662 35364 15 +53499 53197 12 +6516 14951 8 +9433 16953 16 +29260 26915 7 +8533 20272 6 +30196 1270 2 +23722 8852 7 +14895 2676 16 +49582 812 11 +48730 38362 9 +44348 47031 9 +2515 21920 11 +43248 59546 14 +18079 37074 7 +55403 2552 6 +29873 29840 1 +40928 47052 16 +51842 17128 17 +4284 587 1 +4278 27555 6 +43811 7693 1 +27355 14066 10 +49672 35634 5 +17167 45822 12 +7572 11288 9 +11706 30706 5 +30525 22704 13 +3273 27093 10 +49592 32149 10 +59588 57361 7 +37548 50805 12 +25801 44107 18 +54571 47082 18 +27068 53528 5 +49842 32417 14 +2019 41991 6 +30184 12669 18 +32790 42368 10 +29339 56408 13 +20076 29166 10 +56921 3970 12 +48127 17626 7 +29418 43620 10 +48781 46799 10 +22727 12380 10 +53218 27173 5 +16080 1499 8 +36308 31152 12 +52828 50631 4 +54137 24219 11 +51798 13978 10 +1841 2215 10 +11713 52734 8 +19608 17651 9 +28173 22935 9 +17287 24126 7 +2471 10983 10 +55359 56053 7 +16140 14177 5 +44421 14514 12 +22017 29667 8 +28856 29524 13 +16269 43367 2 +10190 55649 9 +35005 3298 15 +56028 51408 13 +52806 19520 10 +11767 39595 7 +40916 13521 7 +10418 23717 14 +34848 57650 2 +52922 38614 3 +32296 22380 6 +41822 40624 8 +39549 47601 10 +23053 25173 7 +47967 45714 10 +12411 35756 7 +17150 57649 8 +20 49843 8 +10191 51615 12 +36250 29231 11 +29606 42714 10 +37036 14043 3 +53448 19129 3 +13653 6959 7 +23601 25936 5 +59721 13607 5 +50284 4968 13 +28764 33518 13 +24839 54665 9 +15348 16312 10 +55959 58436 12 +51418 44853 9 +36666 20589 10 +42767 38522 7 +57119 11730 13 +32947 1807 3 +23160 57176 1 +26249 31788 7 +54701 40995 10 +17854 10974 1 +44771 31824 11 +17680 51710 6 +41883 29571 8 +40257 54899 7 +1999 40048 8 +52835 22111 2 +48341 20655 9 +40534 4290 10 +47544 15528 12 +27263 32424 7 +35502 10179 7 +1977 37331 10 +52124 11618 10 +4216 3237 1 +59028 18824 8 +15302 42361 15 +19307 44740 5 +4054 36944 14 +2390 58261 9 +28297 57048 15 +724 4697 9 +29123 39236 4 +14944 50728 3 +31247 10010 2 +10867 51142 12 +48991 39084 3 +25430 58521 6 +42962 10403 5 +59912 4388 9 +14963 48159 15 +44020 37534 12 +28019 4317 8 +7524 45509 4 +45210 1217 11 +39360 50036 0 +20735 44262 14 +34698 50167 8 +22190 22566 11 +57749 30955 1 +46145 1910 4 +38893 52640 6 +17436 51482 9 +37980 15135 1 +397 52779 8 +46088 27400 11 +36390 5840 7 +7887 36542 9 +57713 30431 7 +37407 55699 12 +20400 11260 8 +32547 37142 11 +23355 26364 5 +9285 40344 15 +1006 1413 9 +10200 56509 2 +20034 16418 6 +57864 19779 5 +48253 39542 2 +44042 37637 16 +58230 6475 8 +7374 24759 14 +34996 41917 8 +50177 1711 11 +43795 36886 15 +45547 36487 10 +47452 13266 1 +49745 8056 9 +44158 21549 9 +12282 28087 17 +21756 59700 13 +25796 15869 11 +56802 47343 10 +7003 744 9 +44726 37335 7 +40129 57645 2 +21133 56776 10 +30619 53174 7 +27320 38536 9 +41005 53901 11 +16134 28681 4 +424 50623 15 +56893 387 11 +42413 45986 8 +28497 52230 15 +43582 55791 9 +37851 57845 8 +20164 9466 12 +46949 53278 5 +8971 12289 14 +23224 29529 16 +30547 48823 6 +49826 14874 4 +9844 7926 14 +54837 36607 11 +50533 10515 13 +59353 820 7 +57391 5553 9 +24740 46830 10 +50448 4617 13 +33409 2739 10 +29005 37025 11 +13272 18215 3 +6624 45733 8 +19575 34616 12 +58617 56907 8 +41826 47841 4 +51998 55345 3 +45541 48287 1 +36155 55377 9 +43643 19989 13 +26489 37293 10 +19843 20333 11 +50311 38786 4 +10616 16235 4 +38044 18342 13 +56197 13719 3 +58962 4620 8 +54434 26482 7 +6423 54867 4 +13103 32888 15 +43931 30811 8 +30595 22023 5 +20465 26755 8 +10940 37986 11 +32977 12876 12 +49132 30925 7 +2996 14131 0 +12176 28667 7 +8786 4595 6 +41925 49974 4 +35269 35764 10 +38937 43577 3 +45134 21587 12 +53999 24574 13 +49499 22026 3 +6648 26850 2 +2262 39934 12 +52419 2340 5 +28999 33384 15 +36683 33038 13 +21895 50847 6 +57267 59669 14 +15795 47911 6 +39 36885 11 +45834 58042 11 +32484 42585 6 +6083 49501 5 +47525 37503 11 +43079 11084 4 +20891 32630 8 +677 32817 10 +26231 55175 9 +49689 29046 9 +11872 53214 15 +27641 5940 11 +45416 47789 14 +11700 18014 10 +15707 32357 14 +57610 28502 7 +17049 27967 1 +12867 14172 10 +45877 44955 8 +30601 17439 10 +28127 49831 14 +41046 37466 17 +44761 32237 4 +40119 17070 4 +9896 39421 11 +47674 27201 5 +37016 59816 7 +28831 55096 7 +37002 46025 17 +53259 36917 5 +55103 38145 6 +30299 15195 10 +33299 39053 10 +42902 9682 5 +16974 16239 11 +729 36649 5 +6580 11727 10 +31306 3854 15 +39755 1252 1 +13342 57318 16 +43017 13665 8 +40709 57094 5 +48807 19753 1 +59386 51904 12 +59970 46631 10 +16687 22467 14 +19888 20129 14 +3149 40583 10 +27521 50560 12 +41835 3985 8 +31875 57065 11 +28549 58503 9 +55476 58982 7 +42802 41611 17 +47657 5304 10 +13576 26306 6 +36933 8599 3 +30586 36304 0 +15113 17678 11 +43483 3395 9 +8768 48067 8 +5433 57782 12 +41247 41157 7 +34244 33263 3 +23763 18213 7 +51607 45750 12 +38236 38539 12 +2741 6680 11 +41659 52153 8 +29410 38079 9 +36758 52172 12 +32831 39734 7 +12676 59421 7 +189 45776 9 +10614 56376 13 +29063 31873 6 +20162 53648 15 +44758 41852 3 +10549 31393 4 +32178 31645 10 +17285 53917 0 +6281 9057 14 +40531 4953 10 +22522 13697 4 +44485 28182 15 +29268 2977 6 +19452 7538 10 +30427 38029 2 +58140 37127 16 +26159 29877 14 +43044 19750 3 +15791 10931 7 +14687 23969 16 +44386 59708 8 +29749 48674 7 +5120 48698 2 +57618 14369 10 +17025 27958 8 +13055 17799 17 +16002 4144 8 +38845 41082 3 +14686 38520 6 +29507 59427 7 +20214 16554 15 +37227 22368 6 +48282 48758 13 +34358 25170 4 +16511 13006 6 +8527 44354 6 +55891 43658 7 +13122 33553 13 +18944 13334 10 +16325 46342 2 +37206 5846 15 +51612 9513 12 +49291 33451 13 +37039 53833 9 +38087 17749 13 +41439 13792 14 +39335 39303 9 +54913 31099 10 +5954 33486 10 +58923 24608 7 +19997 2183 4 +47536 30244 8 +15964 48191 3 +19876 35905 2 +51267 28407 11 +35759 58513 7 +49760 55318 9 +25734 57796 10 +40864 921 6 +13578 26031 10 +576 21738 8 +12776 22322 3 +17554 55841 7 +44571 28519 6 +11531 54852 6 +32025 15468 10 +45629 3408 12 +1012 46853 4 +47527 34785 9 +3200 8753 10 +53134 47237 13 +5405 12977 12 +11437 49387 7 +33718 29021 9 +46951 24298 6 +16205 38538 8 +19380 39019 10 +37202 12946 17 +54026 28238 10 +50062 25590 4 +14797 17712 5 +13811 21178 6 +26539 31830 1 +41788 39240 10 +59615 27445 0 +25853 46714 10 +52086 24447 6 +3878 28508 7 +10300 37191 3 +17427 3958 17 +31327 59544 10 +12476 36835 8 +26309 9899 9 +101 28948 11 +18744 15227 12 +44302 36503 10 +36803 56740 7 +37652 27811 3 +9783 9568 10 +8901 47112 6 +25446 49708 14 +42206 5764 6 +55121 50727 1 +58146 7766 3 +27192 5656 4 +39008 44527 10 +38051 9902 10 +2047 49345 5 +19801 4385 7 +41422 29555 14 +22787 46035 7 +56189 45299 5 +16071 46708 5 +30171 21445 10 +51803 19625 15 +35341 29185 12 +34407 48634 12 +48430 46336 15 +49223 49320 16 +28546 22882 0 +31349 1778 6 +24430 13344 2 +58916 34372 8 +42575 39905 9 +28922 34923 2 +3234 25747 6 +5519 21171 8 +44933 38542 2 +40705 24159 9 +19220 59201 14 +52396 57130 2 +15933 26811 8 +22638 59931 9 +911 10544 14 +1822 34230 8 +17353 56461 7 +49664 45018 6 +19302 36615 10 +10272 53736 9 +50572 48670 1 +11483 50663 15 +41508 9442 10 +56723 57932 7 +12252 28403 7 +23264 18868 5 +33495 34379 9 +22742 50934 8 +16828 15584 17 +7873 2924 18 +45908 9670 3 +28714 32000 8 +50077 4435 9 +57000 40469 9 +16384 30024 11 +18156 24080 9 +31735 24927 12 +38108 20754 7 +34146 18806 5 +47943 32113 12 +23456 11542 13 +18757 40631 16 +40653 45797 7 +1570 33763 10 +13735 33584 8 +52148 39813 16 +33053 58430 11 +3046 17216 8 +56186 21772 12 +22379 25315 10 +1046 8396 7 +36403 29719 9 +8885 35184 6 +23431 45639 17 +28919 26228 9 +10588 45341 11 +45357 28563 11 +48976 18528 10 +43805 30876 9 +39299 6203 10 +14943 12318 10 +51151 19810 6 +2963 11501 4 +43381 31230 12 +18640 5759 7 +56995 1031 4 +16 16934 11 +27240 41618 11 +440 35322 6 +51563 16735 2 +54868 53664 6 +14197 34342 13 +9306 47838 12 +36319 22958 11 +37108 42312 14 +4957 37117 8 +26658 18089 10 +27708 53939 7 +55854 59730 7 +51611 24358 11 +15970 53352 14 +23816 49516 16 +23210 37093 7 +29340 53960 7 +9128 7155 18 +31440 11296 11 +49294 37226 8 +14736 35383 11 +15865 24952 1 +14699 23439 12 +38634 43875 9 +19161 28775 11 +2424 33939 7 +16822 22211 11 +29087 17106 1 +14480 17988 11 +49965 9230 9 +35482 50994 11 +42828 32173 14 +4397 17258 8 +16123 13164 2 +14045 49647 7 +28656 49489 10 +50440 9763 12 +59111 28454 11 +1291 59150 10 +39822 48098 16 +14042 3528 18 +12334 1051 4 +31262 8932 9 +27241 57964 10 +13165 53237 7 +9527 24465 3 +24264 45269 9 +29103 27501 8 +11962 27457 5 +18570 41246 10 +52974 9787 10 +2578 20838 5 +55389 29317 11 +4464 32198 9 +13388 32051 12 +43718 11456 14 +25247 41731 14 +14540 50567 11 +30650 9541 7 +20638 41908 10 +18662 19598 3 +48013 34612 5 +20850 37056 7 +10706 55201 18 +7097 8167 10 +40401 49154 4 +34594 11074 17 +9336 56139 8 +14566 10143 12 +6208 58217 6 +35179 34436 6 +56500 5151 9 +39956 47848 5 +32807 42853 10 +28842 13873 2 +53713 20407 4 +14237 57402 9 +15190 8463 13 +53980 45379 9 +10521 45752 8 +30730 51519 5 +44938 19961 9 +54715 54157 8 +24239 27376 9 +51150 2037 5 +50867 11045 9 +43634 58297 6 +47578 35766 7 +8186 52219 15 +18581 17102 4 +5808 47863 9 +36275 46074 14 +23435 34363 7 +45514 25375 15 +21083 40258 12 +43391 57298 2 +45717 33517 16 +17927 23950 11 +54926 11564 15 +21980 30374 11 +48980 59443 18 +56782 3978 9 +1178 7205 4 +58329 33538 6 +3998 43384 16 +7449 9 9 +5146 41406 7 +18199 34694 15 +44970 33071 10 +44751 17200 4 +38849 37909 16 +42139 5790 6 +1526 14890 5 +56612 19679 7 +43520 51630 9 +5400 22877 2 +2441 45455 4 +56545 17544 11 +32072 2812 10 +15139 133 12 +30902 339 9 +46299 9544 8 +34354 56549 10 +28646 22759 8 +23681 23701 11 +12403 2046 11 +55768 51603 2 +5661 21315 16 +10905 47873 13 +18322 6589 16 +55033 38714 5 +17599 22295 5 +2694 49034 4 +52274 15286 12 +23930 14851 11 +51822 20121 8 +10495 29793 15 +27084 27289 3 +1297 13673 9 +38604 34220 11 +12432 31192 16 +27739 37529 2 +46177 10220 7 +14140 14305 16 +20273 56686 7 +4192 7098 14 +10561 55704 10 +39818 46667 6 +4405 28169 3 +48101 21337 14 +20653 7891 12 +21325 38253 7 +40819 39989 8 +3180 1649 7 +29195 6941 13 +19665 14809 10 +37368 34191 10 +51419 47205 2 +39781 47276 13 +43305 39221 12 +7306 3668 11 +33533 19325 8 +50761 5091 10 +5483 39283 3 +50863 15512 15 +13112 17044 8 +29135 51860 8 +41088 25513 8 +4055 7681 12 +31188 23843 10 +15713 5122 5 +17113 14448 14 +29306 35792 9 +37774 41464 16 +32329 44403 17 +44456 38336 7 +11242 48858 8 +44318 6287 13 +44174 40880 4 +612 29274 1 +48426 50523 3 +32248 12400 13 +16825 14031 9 +31638 58364 6 +35506 17277 14 +15689 2429 8 +778 35742 5 +35270 51709 9 +1299 34044 11 +58937 43710 10 +15976 971 14 +52979 49719 11 +24118 1140 4 +59678 34141 4 +26595 47937 6 +31590 45745 7 +20873 4006 16 +22439 28047 17 +25769 2238 13 +30078 17435 5 +1942 46279 4 +55127 15747 4 +15632 49401 7 +23678 30434 7 +35412 35870 7 +19453 45708 16 +28759 41927 3 +49437 58837 9 +38412 42930 3 +41878 39095 3 +58093 35233 15 +10260 53171 10 +7961 50733 9 +32102 21712 14 +29375 24185 10 +20188 3238 18 +36678 31940 13 +30762 1168 2 +31091 38771 11 +40569 35837 9 +30124 30767 10 +20981 3612 2 +59629 6380 5 +52054 5440 11 +58253 54357 4 +41009 23906 12 +38413 28377 7 +39964 14904 9 +13123 54367 15 +38575 42718 16 +9291 52921 5 +35856 38732 4 +56957 16431 6 +16247 14278 7 +57527 44944 11 +3328 43675 4 +25365 19831 12 +24812 50232 6 +45374 43871 9 +41681 2229 16 +47814 17644 13 +8118 30604 15 +55844 44166 5 +33256 42913 9 +47242 3103 8 +7303 40205 4 +54472 13584 8 +9576 25132 14 +11708 19995 12 +26659 54598 12 +901 50674 8 +47045 5560 6 +3585 49834 2 +272 2153 10 +37381 30146 5 +1000 15784 4 +1726 17415 10 +40007 30245 8 +8188 10513 11 +31194 3654 6 +52816 6878 10 +10183 26156 12 +22300 25184 11 +7189 47675 11 +12822 16514 9 +17725 4431 6 +1739 30166 14 +19597 7448 14 +37019 6388 10 +11650 12098 11 +53293 58554 6 +28898 36356 7 +27872 46628 7 +34579 57906 6 +59562 19699 14 +24985 52898 2 +19211 37013 4 +43567 58452 3 +18052 38659 12 +25136 36391 2 +11333 38644 9 +8910 11255 11 +10984 33272 9 +1465 48935 12 +8957 24021 8 +1606 10245 0 +4715 57895 10 +14422 55720 8 +17375 12666 8 +42447 53882 14 +23645 55803 16 +42637 55577 4 +26081 878 6 +35697 47668 1 +24357 31355 9 +17633 21527 6 +1887 15402 10 +6690 38755 9 +12554 56265 3 +36103 32768 5 +34722 58718 8 +18774 1466 14 +425 38992 12 +40837 12156 9 +41849 34177 6 +39642 45427 17 +3297 59632 12 +21011 16936 11 +25812 24963 7 +22443 35343 3 +24641 40557 15 +46482 4763 7 +53029 10305 11 +2050 28218 6 +45250 45498 7 +18952 5447 15 +31000 10405 9 +56164 20335 10 +9713 57597 11 +47125 54054 17 +41882 32739 14 +56151 31510 15 +30846 10945 6 +11683 54069 9 +19798 25122 2 +37448 51978 5 +111 14340 5 +29624 28020 6 +58186 44581 12 +7795 20313 9 +24670 17083 13 +40474 55613 12 +6555 42917 10 +31411 47058 10 +22101 31723 8 +27791 20461 3 +33932 53267 13 +54625 29566 12 +12183 36757 12 +45349 40789 3 +1170 26118 9 +373 44899 5 +40843 3245 6 +8344 59903 6 +15922 15598 15 +43639 57273 6 +13155 8046 0 +15282 39931 9 +30054 51320 4 +45337 48014 12 +6468 13323 9 +17310 35377 15 +19611 10759 14 +46503 44917 4 +19326 24671 8 +58115 694 14 +53078 42722 11 +22789 55732 9 +13422 14225 16 +34176 24208 10 +57904 59923 7 +19964 6862 13 +50845 34726 11 +6077 1754 7 +13379 24356 4 +22748 37249 6 +35091 34766 10 +30322 42433 8 +19746 40144 10 +53938 36473 11 +33134 56116 12 +38947 25356 14 +41710 38708 7 +39527 53533 0 +43156 1095 5 +48039 16037 4 +13252 48592 5 +15782 23758 14 +42252 32546 11 +4184 47128 9 +42153 52659 2 +28612 50132 6 +18458 22032 12 +24119 31473 8 +18871 13664 11 +37655 48903 6 +38667 26027 2 +3658 38037 11 +51501 7954 8 +43433 41742 7 +14795 30677 14 +36478 6844 11 +58936 20985 5 +47259 44850 5 +39615 37935 9 +32202 14520 6 +58567 36657 2 +21327 53581 10 +45818 35566 8 +34733 59830 11 +17229 51393 15 +2403 37423 7 +5802 22900 8 +17541 17699 4 +33043 1141 8 +26121 2620 7 +33512 58107 8 +58993 37145 10 +15429 28569 5 +48762 25068 13 +1539 44009 7 +7816 32406 1 +5486 26928 9 +49009 30520 11 +14600 51126 14 +32846 7046 3 +58667 41697 10 +336 52975 6 +40600 5524 10 +47755 21101 10 +8258 46267 2 +42900 44780 17 +51723 57733 8 +41958 51002 7 +37960 48613 7 +24659 33838 16 +45007 27109 9 +21603 38828 8 +11060 55424 4 +44621 4391 13 +16623 18099 10 +53164 18230 9 +58162 51190 3 +55884 59141 15 +10836 8180 7 +45946 32898 12 +31297 34818 9 +32905 38901 2 +20570 250 6 +28177 23063 14 +36669 43630 3 +8798 30280 12 +7339 43011 3 +4954 3504 8 +50320 18245 9 +11626 6854 15 +45203 27952 7 +51335 28144 15 +14896 29924 13 +5060 47404 11 +19791 22982 13 +35328 34131 3 +8409 52189 11 +32994 52852 10 +27306 53496 15 +38533 58672 6 +5567 19589 8 +53626 34228 7 +51550 44605 8 +23887 43030 3 +11143 47290 3 +1616 55833 10 +33374 28767 5 +43960 37030 8 +30042 38143 7 +55038 44592 12 +25353 49029 6 +53871 47831 9 +8115 10976 9 +38420 10719 9 +33714 52943 13 +16122 19418 4 +38141 35660 1 +12711 13720 13 +50291 30958 1 +444 26619 2 +7765 41739 4 +8153 25882 9 +41524 28141 6 +48889 17773 13 +41485 26245 2 +19954 24368 7 +12659 26918 12 +54433 46823 1 +14983 9811 15 +11698 43544 18 +19044 1844 10 +6216 28964 4 +36618 2554 6 +53800 7980 3 +33253 23945 9 +45727 40318 5 +18391 227 16 +21197 5095 4 +37100 24799 12 +40575 20893 12 +50071 47053 13 +16206 52021 4 +40241 29057 1 +8079 645 10 +13819 15192 14 +6355 56619 12 +43318 9253 8 +11878 25234 10 +887 14662 18 +53325 44863 12 +2400 53649 12 +52907 23501 4 +37931 35495 11 +20246 57051 9 +9983 51592 9 +48078 49523 12 +34915 30034 6 +467 21151 9 +29570 53919 1 +2686 53311 10 +59657 57205 6 +37378 35736 7 +51240 35486 12 +56911 33619 14 +32685 14572 7 +5127 17833 10 +14573 16339 15 +25952 2983 4 +18169 34957 12 +32891 58998 15 +17490 3515 14 +30643 12121 11 +34269 50783 8 +6408 27430 10 +11073 13916 4 +41703 2886 9 +30 21302 12 +19056 38761 11 +39345 13398 8 +11560 5500 3 +18541 1811 10 +41973 10039 15 +14388 10508 4 +17418 28263 4 +7400 6260 12 +18101 36159 12 +56605 5885 3 +33470 28444 10 +30268 56475 11 +15042 40546 5 +59590 13127 8 +21312 25294 10 +10047 44810 14 +11190 47322 17 +57417 45497 9 +12863 11895 8 +12354 7480 11 +40713 10434 10 +33531 48445 12 +36373 6748 2 +36483 52277 8 +19055 3541 4 +15152 40477 10 +30286 34945 8 +15435 34268 4 +22975 53336 12 +43644 29020 9 +12521 35078 5 +21490 12001 8 +35411 48229 7 +38965 30058 1 +41328 58311 10 +29871 24583 12 +20910 25061 3 +30723 13572 16 +37207 4285 8 +4263 15874 8 +129 32715 15 +17733 8797 10 +27886 24840 1 +7862 50329 12 +26882 14543 12 +3282 58947 11 +55262 38099 1 +14014 350 17 +51624 11427 12 +32048 47795 10 +23396 4472 8 +44672 21320 5 +27853 40183 9 +32801 7289 10 +6782 30182 11 +33682 26208 12 +36096 31449 10 +48958 10572 14 +3415 36570 9 +40771 6036 15 +4950 16723 5 +26825 30104 4 +37138 4103 9 +29053 33664 16 +8374 16907 3 +18876 18388 13 +5602 58136 8 +1805 30835 7 +3134 35713 10 +11957 58943 0 +20104 21704 13 +10321 40958 2 +50005 5268 4 +40472 5455 11 +4089 11920 1 +58407 47695 13 +13825 41235 9 +29779 42688 12 +13355 511 14 +15033 41349 17 +10444 34425 10 +6531 29761 5 +45075 12633 9 +20946 23175 7 +28709 52868 4 +42028 48297 10 +32031 50369 6 +45576 43671 9 +23504 32444 3 +38610 6556 9 +19906 9712 10 +31588 3649 15 +55824 21995 12 +29530 46975 4 +47717 26587 8 +4604 54049 13 +54999 2435 0 +50921 36247 9 +27797 28372 7 +49449 50230 8 +3554 25542 8 +7044 40107 7 +44570 14694 9 +44644 61 10 +3133 7931 9 +24213 6075 3 +19895 11350 14 +2865 28755 7 +12103 18653 9 +9457 5061 6 +47354 59688 3 +34426 5985 3 +32273 41346 6 +7509 20764 13 +13349 21235 10 +25115 26527 16 +38056 567 8 +39591 12222 6 +16499 28432 13 +123 8535 15 +48599 6734 12 +12130 48043 10 +47044 20038 9 +2801 12107 4 +8061 42387 9 +56717 53891 2 +7167 456 15 +50491 23268 5 +56520 23025 8 +41216 7333 6 +33722 19072 3 +43523 33705 6 +38763 1879 8 +48627 21263 4 +48570 21886 10 +47095 31479 11 +56292 27976 16 +56799 9854 11 +11608 3748 3 +47756 58206 0 +54911 21528 6 +39650 7204 10 +45849 19967 11 +12638 46525 8 +28099 28796 4 +39996 30512 6 +50868 59953 14 +56439 4147 15 +28959 33344 11 +8963 25703 2 +14437 6570 7 +51958 27350 13 +32772 39655 6 +38438 59155 2 +16717 49468 16 +56276 13802 11 +45743 23263 9 +26082 13523 11 +13179 7899 9 +46740 34289 3 +47872 49495 12 +29178 28471 10 +2495 3606 4 +20172 27175 10 +46404 56319 11 +54813 41478 6 +12816 30357 12 +7786 28079 0 +33178 18605 12 +21949 13182 12 +4536 31460 13 +28155 43723 9 +22885 4244 13 +30769 50815 5 +32561 50418 8 +3048 34467 4 +33182 28392 12 +47236 58761 10 +18989 50485 0 +26683 56211 15 +57549 26464 12 +15284 39287 9 +43430 13239 5 +38502 39401 14 +58740 41277 10 +19172 40619 10 +41232 59983 9 +43806 28250 3 +47243 44813 2 +5251 25317 13 +24187 35897 9 +44150 22090 14 +25089 1825 9 +22270 40492 6 +28156 7782 4 +56588 39558 11 +43068 56532 8 +23692 42302 8 +35721 57925 9 +10828 46463 9 +38381 4043 10 +53379 38975 10 +9209 37325 8 +27131 42606 9 +45799 23281 5 +24098 7540 10 +16540 59704 0 +45562 14998 6 +27777 50702 13 +11908 34339 17 +9450 3563 10 +25562 50459 17 +10184 58276 10 +6657 16866 6 +51049 49802 10 +15621 31906 8 +34675 22896 3 +11015 42696 18 +13216 55225 5 +4164 51605 12 +24100 55480 12 +48534 53712 4 +8766 45786 8 +41846 15809 5 +40123 29851 6 +55332 54216 12 +42550 12296 16 +19210 14433 6 +549 57103 6 +15999 34221 5 +29728 9641 14 +4689 38871 13 +45521 10158 9 +39728 14935 6 +54742 38529 7 +37798 18011 6 +56457 16755 6 +17016 58046 13 +5743 38200 8 +43946 58724 11 +32544 7370 10 +5580 44826 4 +47480 51088 7 +46400 11587 7 +29088 48901 4 +38500 45800 13 +54614 22288 1 +21256 50189 4 +59552 4707 11 +14524 41587 11 +47722 29027 5 +12426 23752 9 +14615 3355 10 +18684 14846 3 +50662 32422 8 +35246 59230 9 +27395 1826 9 +6811 57180 7 +12747 46968 8 +1631 29937 3 +17793 37259 18 +51855 18558 14 +5740 24667 12 +46820 53529 12 +84 48213 10 +36299 11131 12 +34144 53368 11 +4621 5779 9 +24163 10793 7 +38002 38962 3 +49124 16066 8 +27036 48691 11 +16341 58668 9 +25066 35004 14 +49838 46117 9 +17426 48197 11 +47627 52480 11 +19444 25745 3 +40045 39317 4 +28200 28420 3 +31270 32608 13 +29935 13428 18 +26927 40250 11 +4045 13455 11 +1858 31816 15 +15518 33327 7 +5989 50175 7 +22991 37097 13 +33431 9060 14 +42514 43855 12 +11282 22227 7 +38406 54314 4 +55965 41062 12 +37777 22738 5 +25566 25488 9 +49277 27371 11 +28845 9365 6 +15576 44734 11 +5699 5514 8 +56301 35718 8 +32445 41786 10 +241 13505 11 +36074 5446 16 +17448 50948 6 +53526 26562 8 +48801 51022 16 +37565 4691 9 +15845 23833 10 +21717 27871 4 +52577 14183 14 +52107 23278 10 +31784 46763 1 +24374 22450 14 +8920 40371 17 +16982 51005 5 +8114 17959 16 +41953 18984 18 +47645 59506 18 +31647 29701 4 +16383 32552 16 +39969 35205 13 +11918 38567 13 +10072 24644 9 +57537 48269 9 +41884 29373 8 +59165 56361 2 +16648 39203 8 +18234 57011 9 +8393 4819 11 +18995 13274 10 +47958 47350 10 +5944 38058 7 +30802 3937 2 +3716 33128 11 +33170 21560 7 +55348 25130 7 +39562 15950 13 +18986 44848 6 +53842 46496 2 +16933 23519 15 +512 5960 9 +52904 21856 9 +18423 17109 9 +41013 25235 9 +27462 54442 5 +54294 21299 4 +16253 39571 18 +20583 39210 9 +21562 27422 14 +5217 31264 5 +48439 51957 2 +23608 34987 7 +4081 16817 7 +30460 27211 11 +20550 47381 7 +15291 4162 1 +33062 54206 13 +51012 3736 15 +23463 22145 15 +44281 35268 6 +23995 3780 9 +2579 46909 18 +37856 23956 4 +26345 39110 10 +17028 52289 1 +49088 35741 17 +6166 16076 10 +1558 35886 8 +7001 47594 10 +59331 8675 8 +16793 17352 6 +31055 34541 7 +7700 5198 10 +9651 56313 16 +16479 22333 8 +23767 22713 9 +5648 16001 3 +22640 7441 8 +6181 25915 4 +32002 10462 10 +11059 17550 16 +38441 29767 7 +1833 6818 14 +44469 36174 13 +18320 1830 13 +23387 45882 15 +55409 55371 8 +35820 49665 5 +45762 509 12 +8088 10823 1 +9634 55207 13 +43535 46499 2 +2557 30755 6 +7416 48747 18 +9297 35543 9 +51358 3580 12 +59425 54309 6 +31352 10960 9 +1480 36494 11 +50222 48512 6 +3874 9095 15 +299 20913 7 +22191 29497 6 +32813 49771 7 +13258 56332 8 +14648 3821 14 +10422 7258 17 +31709 11851 2 +51432 21048 9 +14732 13392 7 +3162 48742 7 +27519 23029 12 +39357 31621 16 +45526 30782 14 +41095 11807 7 +53593 21884 2 +48616 11212 18 +48259 5188 12 +52808 2198 9 +56094 3548 8 +11080 46626 13 +50063 52420 7 +1059 21026 11 +16689 31016 5 +55101 14603 6 +40131 21259 13 +30669 3464 3 +44432 4443 6 +9800 2675 8 +24704 26161 11 +28872 30096 11 +50683 45440 12 +8586 40008 14 +4612 17910 12 +18007 13594 8 +39151 54058 6 +57350 50982 10 +36032 7797 11 +31581 16741 8 +17710 53985 7 +40037 3887 9 +58703 32560 13 +14137 3838 4 +39451 55872 5 +14420 32802 7 +29422 5929 8 +2923 7952 15 +28143 19840 10 +23509 23027 17 +55189 11107 8 +2201 53957 2 +38091 37020 9 +9216 50792 11 +41190 14640 14 +24503 32940 10 +53190 40907 15 +54517 51221 13 +31428 5056 8 +41946 59026 6 +55487 25510 2 +36272 44487 8 +46602 18204 16 +40314 19209 1 +365 36816 10 +35825 30789 7 +40811 15559 3 +26012 48646 6 +38399 36852 6 +16991 52350 4 +27896 11375 6 +53733 54320 2 +5436 55537 10 +7436 15612 9 +35164 10738 13 +23464 30538 7 +38096 21005 12 +58862 33935 6 +13208 36920 12 +44801 18311 14 +7137 18184 9 +2437 17593 4 +23351 11822 7 +56172 23578 8 +55885 29066 12 +48388 46897 17 +22870 40622 9 +48110 2893 12 +59177 52485 10 +6955 39639 4 +38193 27890 9 +29220 17230 4 +44622 50075 8 +28512 53793 11 +49816 35714 12 +45158 493 4 +2389 45624 7 +7340 57058 8 +54258 4600 8 +20883 19864 15 +18513 49595 7 +32638 672 14 +50513 24771 9 +18663 27845 6 +17976 37574 9 +16918 33809 16 +14771 56490 3 +22226 56485 10 +33408 16101 15 +44918 34117 14 +14291 46118 8 +19730 19329 15 +25610 26474 15 +31170 1359 9 +16663 49469 12 +51590 49994 13 +13435 38383 12 +6106 42695 10 +14614 2606 8 +2547 25092 1 +33219 53227 7 +12855 23461 14 +23019 36687 7 +531 46298 13 +49488 15961 7 +2289 20628 9 +46692 1445 4 +40692 18155 4 +8001 11849 12 +4210 14296 10 +48524 42106 18 +39593 23532 10 +36180 8426 8 +19792 44560 8 +38952 31104 8 +24876 25232 10 +19553 28363 10 +54070 33802 9 +3749 53168 7 +15751 34725 3 +14522 5862 8 +13963 29150 12 +55962 8754 3 +27780 16324 16 +25452 26244 15 +56727 2643 15 +46588 2306 4 +36463 13730 2 +22364 53420 8 +7748 56465 3 +6444 8278 4 +8505 44729 6 +26194 13014 11 +46198 57174 14 +35276 41797 11 +9711 21575 8 +57776 19818 9 +59758 56220 13 +4807 3093 9 +33671 11235 11 +29823 35039 3 +56523 48644 13 +53494 58352 8 +43491 54641 13 +32744 35769 6 +12697 49738 12 +34481 33107 10 +27650 45131 9 +51067 14299 8 +5633 29600 12 +26634 10571 8 +45590 33953 7 +55938 5979 18 +21731 30496 4 +41196 33498 10 +15069 30373 5 +46480 59193 8 +54021 59087 13 +787 53144 4 +8456 27412 6 +40456 10579 0 +7739 28071 12 +22919 33326 9 +28966 54950 14 +15246 28333 9 +4102 31085 11 +19006 16879 6 +45247 40290 14 +16628 44999 14 +13726 28310 4 +27387 16361 11 +28468 7991 10 +38050 38656 11 +29614 16855 1 +38322 19661 12 +58795 14446 6 +18197 29565 14 +43786 18685 17 +32702 17653 7 +14403 35457 9 +5347 2490 11 +29985 54986 8 +58280 1563 7 +52890 674 10 +18189 31938 13 +32834 24247 1 +2645 29964 8 +41963 59149 7 +4410 12206 11 +24940 28034 4 +49149 6168 5 +47083 35591 10 +945 13918 4 +27410 42582 8 +9203 7704 10 +3368 29025 9 +41154 39109 16 +19205 39384 6 +54639 13220 11 +2766 1338 14 +54150 2189 8 +23808 20626 8 +55607 59107 11 +53849 20024 9 +32513 41729 4 +48340 17787 9 +25492 13886 7 +56478 19517 10 +51019 18846 4 +54295 11791 8 +48230 18633 7 +18033 50964 9 +57442 51359 4 +27242 51669 1 +5851 18455 11 +51884 56079 4 +58385 51297 18 +55336 56480 12 +17648 35425 4 +56712 43988 7 +47097 31253 6 +35716 50750 4 +19851 35707 11 +26746 40989 1 +18792 29925 1 +27402 51015 7 +30423 10607 7 +13549 54030 16 +26520 19246 11 +1851 57652 14 +49633 59243 15 +39293 10105 18 +47614 18959 11 +36135 42098 8 +27114 58655 5 +55957 51797 9 +41928 13235 16 +51216 41294 4 +49441 58948 6 +38531 17046 6 +12678 59720 9 +17211 12024 12 +57949 20095 7 +32506 52880 11 +37275 38208 7 +14900 25558 5 +44962 14353 10 +7066 12249 5 +20090 41455 11 +53015 37444 13 +35794 16322 11 +33271 21195 13 +23241 57118 11 +38204 36153 14 +50430 53550 14 +22365 46568 12 +588 20201 5 +8720 30841 13 +41540 29466 4 +20337 14021 5 +55116 19428 12 +55994 5799 9 +8556 45583 14 +53301 26191 1 +43810 1401 18 +9395 9093 12 +16343 13752 16 +58211 37499 5 +29147 29884 10 +10252 58910 6 +15166 1261 10 +36238 28222 13 +17330 12364 10 +39400 37493 13 +41751 18319 8 +57658 58519 11 +21499 48224 13 +19974 9235 7 +50089 1020 10 +3116 2868 12 +8806 41957 6 +21401 25371 10 +54263 2316 1 +10822 33663 6 +23272 12489 10 +34186 43338 8 +30798 40704 12 +3089 14453 12 +6548 39804 9 +51959 22035 8 +11973 53522 2 +55419 15757 11 +40625 740 14 +23804 42541 14 +54741 25474 9 +38007 17367 2 +773 32735 15 +33198 59397 17 +768 10392 12 +57235 19117 7 +54004 18789 5 +10785 1369 3 +26077 47136 14 +46752 14817 7 +34872 53463 14 +12679 2560 6 +10710 29234 17 +48602 56452 4 +43008 54402 11 +35188 58556 17 +24765 7456 11 +3723 43674 3 +22597 6931 10 +52132 16351 4 +49404 22869 9 +18385 26275 13 +35723 25473 4 +38284 43742 10 +55559 2976 4 +14564 14539 4 +35107 42564 9 +50540 56897 12 +7936 2021 12 +45133 3607 4 +26713 42845 9 +32385 8608 12 +12116 54212 14 +38883 38935 7 +29894 42822 7 +10379 36172 17 +23095 55975 6 +52894 9784 11 +18473 36568 6 +898 11673 13 +51911 20866 14 +3280 4482 12 +30583 52776 14 +34575 8402 9 +19287 17379 4 +4828 22888 7 +24478 55361 9 +21602 47866 5 +10599 29720 2 +6530 8660 2 +52108 5937 14 +52888 35394 6 +49216 34534 14 +32643 17860 16 +44649 2113 12 +1181 35138 3 +5197 46348 15 +9337 14052 4 +8567 54572 10 +56557 20971 15 +46076 5681 8 +54539 7439 14 +18441 17913 13 +3661 38721 6 +22967 36469 10 +46323 38089 9 +19367 25519 10 +40426 25911 7 +38944 49133 11 +5300 29179 15 +11018 59058 11 +29175 8854 6 +2660 7178 7 +16535 1133 10 +4140 31750 5 +7315 31331 8 +10773 32902 5 +3696 17912 11 +6342 42330 7 +57880 16651 10 +7419 32823 6 +10881 52280 16 +58134 29798 13 +53421 43368 9 +10472 9859 7 +26350 51376 4 +42210 164 7 +7497 53642 8 +24178 34723 10 +47898 557 10 +57063 55053 11 +32258 56968 5 +12451 54276 7 +38086 17784 11 +20103 21182 12 +5317 33254 0 +38832 20705 11 +18004 41073 4 +22611 11292 3 +7678 8576 10 +25021 45581 13 +41787 33798 16 +34385 25660 6 +53896 42840 4 +42298 11509 10 +26813 1247 16 +5556 24544 13 +38225 24242 7 +16453 14455 10 +16335 48284 6 +20588 9487 8 +4488 10956 6 +36392 43094 18 +35173 14244 5 +47029 38984 8 +32109 8933 8 +53761 34435 8 +53625 28525 8 +17121 59729 16 +34672 7314 10 +58034 49311 9 +38541 14974 18 +15619 16844 5 +31082 30922 9 +17163 36818 14 +44929 1474 11 +38387 23205 16 +42451 47773 8 +29593 2185 8 +49330 59259 11 +26824 49696 12 +53025 8568 7 +6177 18400 6 +36953 25207 8 +18523 42442 16 +34683 46772 4 +58451 36271 2 +49027 38348 6 +59823 22914 6 +19732 47036 2 +8016 33655 3 +17084 56772 9 +49165 12221 9 +53835 54689 2 +30648 25689 9 +18531 31176 16 +5601 44207 2 +56732 3187 11 +11078 48628 12 +59188 35881 5 +43078 28003 12 +18353 59156 9 +56883 15482 12 +8690 51105 16 +25901 10617 12 +4986 1862 5 +3209 42769 8 +14512 30752 7 +9591 4417 12 +23535 22371 14 +1287 12503 4 +55708 32486 6 +12171 47764 13 +17924 37763 10 +15358 17742 10 +28119 40015 4 +55484 35673 4 +38576 1280 10 +46789 17477 9 +28783 21818 8 +13337 33665 7 +3720 1940 9 +46122 26170 4 +55102 39906 9 +18038 47213 3 +1698 54781 8 +55217 25498 4 +10569 26032 9 +57177 5781 15 +5965 37197 5 +56428 27618 10 +58151 19676 9 +30546 53392 15 +35218 49370 9 +45700 45461 18 +20483 50619 6 +35503 57101 10 +48136 27016 9 +28865 27706 12 +48133 25768 9 +46265 49927 8 +15110 15739 13 +47207 5137 12 +13648 23873 4 +56011 49011 4 +11981 38881 10 +35329 9838 10 +50421 1592 13 +58786 3995 9 +7154 15287 11 +17774 50997 13 +48671 54586 14 +45645 55501 7 +274 38842 14 +54568 39698 5 +41814 55254 6 +1263 7004 5 +57121 30526 14 +23455 16092 10 +30941 7914 11 +30494 196 15 +43981 5705 9 +26151 38811 10 +10648 18386 9 +1581 23690 8 +52422 12930 3 +33773 890 8 +6595 8288 3 +23817 57640 16 +22694 51281 13 +6746 20852 9 +37835 49485 14 +46721 43941 9 +48688 7519 16 +26114 26297 10 +16010 1764 9 +38504 40301 10 +6224 51177 12 +59592 56659 11 +12889 29401 11 +12245 16874 7 +39353 30779 10 +23433 27121 15 +27310 24324 10 +2246 10001 15 +57593 36482 10 +2838 38932 7 +4065 19835 17 +29934 33649 3 +23856 3934 2 +57458 51581 16 +37760 181 5 +15133 32222 6 +20649 46791 11 +59342 3165 2 +43131 44191 4 +7262 18352 3 +35254 27104 5 +53100 45444 6 +13615 43216 3 +49594 34233 7 +49543 58350 8 +34750 24764 10 +34689 58054 16 +33596 50242 7 +9460 51305 7 +22891 8598 9 +22908 40747 8 +24943 46158 4 +28818 1920 2 +1316 1661 12 +45413 30995 14 +44098 28936 5 +24683 44542 6 +18154 18046 13 +41986 17268 5 +35331 16495 3 +35913 24686 12 +2863 53375 7 +7536 6194 7 +3991 48949 10 +43172 32572 8 +38183 34962 18 +8300 12932 9 +54158 44320 12 +38895 23373 9 +51551 50624 10 +1076 48361 1 +12506 6550 10 +5842 53291 8 +26526 43421 12 +6909 57066 13 +19704 49120 14 +50104 37366 10 +50051 54627 10 +42214 19322 18 +41206 50144 13 +59900 50281 4 +42826 48899 11 +21917 28782 10 +52154 6936 5 +11764 25501 13 +1377 39291 8 +2950 30549 14 +50381 14800 4 +2729 37256 6 +48422 21468 10 +54221 51340 2 +17892 22850 3 +28737 59161 15 +4105 41774 5 +52875 17145 13 +59339 22709 7 +296 50954 5 +43443 31232 6 +4281 34484 5 +52045 2328 5 +55061 22019 10 +24084 45395 10 +27554 58949 13 +3950 4926 6 +3890 53964 11 +46414 26805 10 +16095 9778 8 +45064 4362 13 +27164 52561 14 +25895 52191 10 +28298 44205 7 +47186 8743 2 +3185 45718 9 +53455 23651 7 +47026 56414 15 +42951 31401 5 +12769 50039 13 +26241 24576 6 +41091 41304 13 +25470 35346 7 +11128 46992 6 +42770 11679 5 +16745 4325 5 +57429 19096 8 +4852 57223 1 +44141 39059 9 +8249 44791 4 +25432 28061 16 +22200 31775 5 +26844 11104 12 +58789 8067 9 +24720 58334 16 +50212 50720 5 +3872 33292 8 +39773 47465 6 +4401 54933 8 +54974 54483 6 +1685 10730 7 +38016 12270 7 +32497 12898 12 +796 58859 17 +21640 55680 0 +44361 53606 15 +35431 47989 8 +11894 13798 7 +29759 19522 9 +16518 52225 2 +52996 6686 11 +30975 31766 3 +2156 8326 12 +28903 10625 8 +12951 24688 3 +2208 36040 4 +35672 25942 9 +31692 43217 8 +19243 29402 9 +10790 8112 7 +4165 17322 2 +39489 10986 10 +10722 34918 15 +17569 39443 7 +57449 50217 18 +54389 12810 15 +34390 6033 6 +48572 18608 11 +55615 21439 11 +35389 53548 8 +18711 51292 15 +44359 36424 6 +9310 9195 0 +49274 10340 4 +9334 3336 11 +1845 31045 5 +39381 11237 10 +5378 41255 7 +12066 59370 15 +24807 11153 4 +18686 11327 10 +38435 40169 6 +17458 55514 7 +41290 14279 10 +3489 33712 17 +18 35234 10 +26137 20288 10 +12013 1890 7 +9789 30069 17 +52796 35786 11 +19788 38107 8 +22734 28588 9 +51813 47569 5 +26281 41374 4 +27713 4195 8 +11891 40326 14 +35753 32521 8 +32599 37796 10 +25391 52926 12 +25310 42664 1 +50558 52850 10 +33454 57588 8 +4115 1761 6 +9177 24130 14 +38677 46181 2 +5143 26541 5 +30996 28937 3 +15778 38885 11 +17741 17619 6 +31174 11928 6 +35424 30457 7 +37279 55135 13 +30003 19335 12 +7547 23937 13 +12104 42504 14 +30041 43215 11 +6984 52988 9 +28722 18505 9 +43801 45780 6 +44935 19387 10 +29950 19848 12 +39277 30654 16 +8970 25612 5 +21219 20414 4 +53690 13777 5 +15241 938 3 +25385 4344 9 +22932 54023 1 +6906 23500 9 +46931 14972 4 +54844 18905 4 +57477 16259 4 +14909 1357 9 +14087 46742 12 +13652 33580 10 +52754 29287 8 +48800 23629 10 +55009 19346 10 +38269 34973 10 +8865 57168 8 +28602 1583 9 +52231 47711 7 +36349 1645 9 +44058 26009 4 +6115 1497 8 +45893 19782 2 +22775 30157 12 +33895 44014 13 +54345 45881 12 +50192 24457 8 +41309 37343 8 +14538 30518 10 +54961 15590 9 +48467 55562 11 +5592 5918 18 +25163 45622 7 +6735 11752 8 +45685 48096 11 +17407 18394 10 +18190 14609 9 +10606 6592 6 +22572 990 8 +31065 16221 3 +18946 59661 9 +45396 58108 3 +49938 42366 5 +5892 37102 10 +50575 45967 9 +53332 13755 10 +54022 52004 13 +30631 45900 9 +5471 25225 6 +31362 10994 10 +17189 7402 4 +50251 25483 7 +45744 21120 15 +37338 21227 14 +56979 38907 10 +23949 56675 12 +20050 23584 11 +41441 41008 17 +52183 29355 8 +21971 13557 9 +58857 55775 7 +33674 22311 9 +38821 36593 6 +15458 49235 15 +42389 29880 5 +55357 32435 10 +12643 52109 3 +35074 44434 7 +55252 1106 13 +2875 23312 7 +47763 18908 10 +32511 25694 5 +21804 37863 4 +23765 20665 10 +48672 970 12 +23412 7930 3 +31377 2787 10 +17808 9698 8 +49644 52902 12 +6970 4456 9 +9761 2041 13 +52026 21087 14 +42476 53976 12 +22986 50723 6 +35930 46921 4 +57175 51683 16 +15466 25330 7 +57685 22543 8 +31575 46967 6 +36769 10081 8 +35945 56379 5 +17032 56729 8 +11906 36528 14 +58067 49716 3 +39951 56205 17 +37434 21821 14 +58883 24001 11 +41589 7707 9 +35499 21266 7 +24269 25398 5 +46454 53435 12 +37465 48563 8 +57299 36596 13 +58355 31650 6 +31140 1340 8 +11925 51071 13 +2969 41775 11 +6903 30932 6 +32119 13775 10 +36407 2470 11 +3038 50691 6 +5469 49681 1 +27972 4570 11 +49161 48022 7 +4040 11053 11 +5702 33605 8 +9765 4085 15 +55351 30439 4 +11598 27867 11 +1417 12692 12 +42061 23908 8 +59703 291 10 +32597 2595 10 +25835 6974 10 +53228 40946 10 +975 33777 11 +17009 3487 1 +44585 13500 6 +31812 42208 15 +58439 23007 10 +47589 18277 13 +21896 25739 12 +26792 881 12 +34066 15229 16 +32299 3835 6 +23687 33286 15 +22651 50015 15 +39664 36913 8 +23219 36685 3 +39553 43169 3 +29455 58374 14 +28575 54941 7 +9530 25616 4 +31204 53682 15 +39158 18990 9 +37051 42988 16 +14425 5614 9 +28951 5020 6 +358 46037 5 +21847 55433 3 +15781 30160 5 +55261 19857 5 +4660 37522 14 +18922 542 2 +395 41606 6 +7329 40182 13 +31987 27245 10 +26585 43672 5 +22849 31701 9 +22408 846 9 +25486 54764 8 +45862 14838 5 +51663 27439 8 +3684 27090 9 +29774 7492 15 +9068 21570 9 +50364 35152 7 +49991 14039 5 +2417 20030 14 +37859 45747 9 +53597 40186 16 +12761 38758 10 +2778 56749 14 +9381 47142 7 +48246 34369 2 +50636 53772 6 +38612 5946 12 +56474 35689 5 +39150 41929 10 +664 38101 8 +41219 12950 11 +11037 28728 9 +41102 9972 11 +58015 43014 3 +58693 15962 9 +13996 39184 12 +12307 57319 5 +25965 9171 14 +56854 10663 14 +45226 23954 6 +37311 40347 9 +4197 57643 11 +34219 26496 11 +35604 42463 9 +27348 28405 9 +18226 6699 10 +8984 2319 8 +23597 20882 7 +32644 22084 9 +24399 43279 7 +21547 18586 7 +54605 27033 3 +16152 54035 16 +54088 29741 5 +40065 19619 18 +32647 8210 12 +25564 25512 2 +44705 37745 4 +11923 49923 15 +34793 30741 9 +52538 29714 16 +34642 1190 9 +30176 43576 3 +7017 9882 14 +7077 31088 13 +30499 37237 6 +3006 50553 9 +30497 24946 18 +51219 46802 15 +12155 47941 13 +41099 22360 13 +59194 42385 18 +446 48625 8 +13033 32432 7 +10627 21542 1 +15532 45045 15 +30733 36089 8 +11254 19859 7 +46026 4486 2 +56857 29636 3 +54151 20535 13 +55506 7691 5 +48236 50134 5 +57985 31580 8 +43632 10157 3 +56440 26514 5 +4411 11318 5 +24819 50597 13 +30994 12141 9 +11712 52600 4 +57872 26815 12 +34164 55143 7 +30985 648 9 +18440 21410 11 +51108 27452 5 +4005 1050 6 +10586 8807 11 +19324 50688 5 +49118 37600 9 +45566 35957 13 +44356 34916 11 +20565 23198 9 +6619 16694 1 +42230 46393 6 +58366 2124 17 +20904 9341 14 +9526 9431 10 +39525 44383 9 +37929 5710 8 +57918 1957 14 +1045 42335 14 +26964 57654 9 +31612 46554 11 +20662 50745 8 +30061 51125 9 +30326 59336 4 +52847 55789 2 +16729 42831 9 +45378 2062 8 +48164 4274 7 +53373 21634 3 +21991 52585 4 +30275 19269 9 +39660 7607 17 +22802 4786 10 +46485 55435 12 +13717 17963 16 +19780 53386 16 +41791 11263 15 +28582 57881 15 +3407 6471 13 +8424 43858 13 +14977 18224 11 +19399 19430 5 +13724 22994 7 +31026 29380 6 +10789 35578 10 +3211 38201 12 +23951 18153 16 +36972 7905 7 +22875 49743 15 +19939 24061 11 +41293 45892 13 +20106 37596 15 +7343 21106 12 +2032 20358 13 +26215 59348 16 +57718 44655 13 +7601 20064 3 +25121 3510 16 +36670 57922 17 +4716 24628 3 +41205 34168 2 +26861 57017 16 +2842 25794 9 +59889 5081 10 +6783 32389 9 +1458 22079 9 +22374 28571 6 +55171 57233 13 +23070 4205 11 +37357 12699 15 +38121 2982 16 +23811 2133 14 +49818 25263 16 +1003 27820 1 +40806 10493 7 +29450 2195 7 +95 15396 1 +52540 56214 10 +29607 35514 8 +771 26997 14 +48499 32345 9 +12541 11070 14 +10152 30630 7 +35749 47255 3 +28609 25357 6 +34845 6872 12 +54261 48222 18 +39471 37122 12 +26242 41204 6 +49511 24532 12 +20098 31184 13 +51701 50614 10 +58203 9572 3 +34430 8214 11 +2639 37092 5 +38436 25980 1 +59332 33764 16 +13806 10345 10 +26591 53989 9 +48111 40686 2 +58868 45973 9 +14587 12696 13 +4012 5588 4 +57942 51303 5 +25472 35489 11 +20285 20923 11 +1226 11402 8 +46258 41430 14 +32726 37129 10 +43571 584 6 +9045 25256 9 +12112 47784 9 +4806 23195 13 +31014 56664 10 +43943 56974 6 +59516 52316 9 +14421 35000 7 +15461 35725 6 +38068 8025 4 +36336 46197 6 +45209 5665 6 +37498 51979 13 +34123 53594 10 +57068 53743 8 +9925 6696 15 +23249 37634 9 +7833 1972 7 +49884 14499 9 +24982 24538 11 +46108 12637 10 +20613 18066 9 +54634 35517 10 +57815 49693 8 +23346 12172 15 +31884 18771 8 +29547 17169 9 +56010 7521 4 +5726 42127 17 +41654 12395 13 +52702 18442 4 +14263 2742 12 +45062 14320 12 +32011 31171 8 +31037 210 6 +51601 43663 17 +42825 32010 6 +12061 26886 16 +5760 16063 6 +23181 4837 5 +48715 41447 0 +12123 8528 10 +46015 48660 5 +2295 16317 12 +10328 37162 12 +13563 5357 11 +41613 46927 17 +19684 35638 8 +59476 42190 8 +3308 2489 5 +53133 31666 10 +44814 43494 9 +57859 6497 5 +58660 6496 3 +40844 12175 9 +38587 58030 10 +58055 38724 7 +9085 51700 7 +52402 47399 11 +2057 51993 14 +35796 26236 8 +24736 38244 16 +17970 20159 13 +44498 25746 9 +57830 29991 4 +35748 40788 6 +22252 15432 6 +43417 44278 3 +17743 51648 10 +43768 56240 8 +17726 55565 6 +48315 6988 12 +12846 46977 11 +13508 58607 10 +21095 16146 9 +23823 16543 0 +15636 53969 14 +42842 55193 4 +9531 14372 11 +38802 25716 7 +18506 13593 15 +2727 15963 13 +48180 26160 6 +23549 48575 11 +8646 30394 14 +24967 38360 14 +17430 41252 1 +692 27946 7 +6661 50841 13 +37144 18927 6 +58275 39148 9 +10643 47677 11 +34878 36455 13 +307 24313 16 +52 799 12 +37784 4229 16 +45558 13733 7 +47483 43987 8 +47138 2682 13 +50800 55630 9 +29909 6425 3 +39948 37987 15 +815 35363 9 +12937 15309 14 +38808 9820 0 +38024 25597 9 +20404 2897 7 +25629 7968 6 +13219 25067 9 +28175 54160 7 +38268 38869 9 +29679 23258 7 +10921 43762 6 +40657 59047 8 +7079 52626 8 +17271 25495 11 +26070 18468 2 +10623 26726 10 +16308 19022 6 +58222 30519 17 +49666 36630 11 +26773 53795 13 +33700 44685 9 +19079 50620 15 +34362 30279 11 +10421 547 11 +2142 28060 12 +31950 56046 2 +53604 1150 4 +23913 17745 7 +55707 12824 2 +18624 34830 13 +14593 11578 10 +31530 27963 13 +29569 50590 11 +30030 35879 9 +50568 47531 4 +52248 31958 3 +51882 20471 1 +28679 43179 12 +28792 49881 5 +33971 40104 7 +40984 31872 13 +42036 16602 16 +40988 11188 9 +27030 52407 15 +43210 28932 11 +52562 23528 5 +7832 39762 12 +6849 9768 12 +20518 29843 15 +47395 53767 7 +3025 12369 13 +26146 18337 2 +40519 8369 4 +16705 30001 7 +16624 25108 14 +5906 25624 16 +14431 55502 4 +52363 39952 12 +31495 28185 7 +22287 22334 16 +28273 10082 9 +12628 40249 7 +50583 252 6 +75 53768 4 +22309 1634 11 +11458 48402 6 +6236 23347 14 +8642 31610 8 +5318 22303 3 +24036 3870 7 +43694 1528 7 +9186 49430 4 +39217 47623 3 +40824 41101 5 +27011 3066 9 +31323 38179 1 +17459 46401 5 +16100 9802 13 +3306 20927 7 +52413 763 11 +46537 34252 11 +25774 19180 1 +59457 10611 8 +36208 12513 12 +34655 28756 7 +47586 59735 8 +5573 56819 5 +40551 51433 6 +57463 13675 9 +25286 13715 13 +45947 44743 5 +54878 52786 11 +35248 28688 10 +2577 26788 10 +35963 14689 5 +4866 23311 12 +56992 32523 9 +1818 32550 4 +41298 54714 6 +54217 33404 13 +40951 37662 7 +48869 11515 16 +33576 22680 14 +53431 32461 7 +47437 22180 11 +22332 49143 18 +38635 29029 7 +8908 30229 9 +4353 36677 15 +55966 30459 5 +16064 31508 8 +53781 35306 14 +32564 41619 3 +49335 1038 18 +14017 27658 8 +18466 27743 12 +39115 32465 1 +429 52130 0 +41692 55759 14 +39713 23964 17 +6885 35844 7 +50802 24710 3 +26120 11439 3 +4836 8771 6 +54661 9984 10 +28274 49250 8 +21745 40664 7 +57366 610 7 +1677 23526 10 +15116 39526 10 +29729 11403 8 +1136 45087 7 +57386 39732 16 +25652 19144 18 +35659 50535 8 +17634 15843 9 +31148 29829 10 +48309 51153 11 +5547 44306 4 +50694 37800 8 +23894 57585 6 +50536 44321 15 +45935 36433 7 +32246 29489 7 +52934 2849 6 +21733 25269 9 +34688 4061 16 +25653 3758 5 +11707 22962 11 +32238 48027 5 +23936 47855 4 +23744 18009 18 +44978 11420 4 +13770 50081 6 +44939 51659 10 +13479 48135 14 +12396 3987 13 +56878 48583 7 +37236 16622 15 +42737 6979 8 +32281 30646 12 +27331 28410 7 +26223 27162 11 +28145 12269 12 +20581 42670 9 +5609 16756 15 +2480 51009 14 +37527 6290 13 +19015 4765 9 +2833 27540 11 +46871 40763 3 +16851 20584 4 +3274 47886 10 +46228 26095 3 +22310 52654 6 +42689 50650 11 +20200 57844 9 +44630 8415 7 +2404 49734 9 +52593 45543 2 +17583 13338 6 +47198 47190 4 +7474 20248 18 +17368 12815 9 +35525 36629 9 +20623 14637 13 +34652 3255 13 +53284 55955 3 +30320 5083 0 +7452 9348 10 +11914 22913 8 +33325 13333 15 +46741 24115 10 +36284 56627 16 +18805 54732 10 +10391 29361 9 +42518 18799 1 +7073 14200 8 +49718 5721 12 +41733 42744 1 +33524 17412 11 +57711 15010 4 +33270 57166 11 +22110 11879 13 +20940 8148 14 +39880 21907 7 +38302 13213 7 +21989 26473 6 +21841 18120 10 +55067 32841 7 +19392 46563 11 +17076 22966 12 +17486 43141 13 +17747 14255 8 +28708 12698 9 +40943 50264 10 +16440 22824 17 +58635 40452 10 +53776 48121 4 +10474 55766 15 +53700 49986 16 +13945 14285 13 +9665 28749 15 +34519 38961 6 +7552 23829 3 +24971 11955 6 +42609 7119 6 +30891 51811 3 +34158 2959 3 +17865 45048 8 +47272 47874 10 +30915 18855 10 +16635 22598 12 +53465 23864 16 +1422 30335 8 +4170 58082 7 +57717 40503 11 +54232 22220 7 +24266 22843 10 +28615 6742 10 +40821 10224 6 +3678 40855 18 +15620 2700 8 +52573 8698 15 +21612 15971 0 +30091 10155 5 +19539 7860 10 +26794 44193 5 +27269 49655 13 +13564 43354 11 +24979 22187 10 +47533 22839 7 +2657 25194 10 +38188 16290 8 +54012 6920 3 +14833 28282 9 +8443 54602 8 +9290 58956 13 +28956 1495 7 +56268 34039 9 +29479 26320 6 +54390 47570 13 +56275 10948 11 +53446 37415 4 +52313 41509 5 +5675 53434 3 +26310 1384 9 +9118 7424 10 +47253 37463 8 +47402 41987 7 +521 54265 13 +55355 8809 0 +11954 37169 4 +41182 56147 6 +28813 52072 16 +26036 20284 10 +41381 4839 8 +32549 58231 10 +19297 8169 4 +17300 20884 5 +11687 48074 8 +58770 5931 12 +51874 996 11 +15070 58483 8 +8677 6708 4 +16314 52617 4 +8562 47175 11 +21105 44055 13 +52462 8329 18 +3348 24567 10 +47787 36242 5 +22260 27034 7 +28236 40617 9 +14229 24541 0 +34054 34676 7 +33560 19168 7 +20727 25699 9 +42830 50282 6 +43398 34476 13 +49800 28988 7 +42307 56448 4 +30071 26307 3 +24114 45698 6 +16413 27002 14 +42517 22279 12 +55592 1074 5 +23630 36755 8 +22998 45976 10 +56431 26992 13 +16363 40731 5 +25276 28353 14 +25450 41512 3 +37560 43043 5 +45448 7901 12 +21628 6960 0 +18646 47881 6 +57965 36775 6 +33063 22695 9 +42806 37492 5 +52611 3210 9 +29538 9250 12 +47482 22387 6 +37315 27705 15 +20209 27015 14 +54870 43007 9 +28355 22842 10 +1433 36870 13 +52590 48099 4 +33398 7078 2 +36810 52581 12 +15393 21807 16 +20936 49126 8 +26795 21636 11 +20952 2894 4 +11623 54407 15 +50009 51248 13 +1134 70 3 +54401 43820 10 +50293 42725 8 +15699 38571 8 +23374 54455 13 +14658 12688 11 +29256 12139 11 +10461 22613 14 +35225 15680 9 +10096 53904 8 +44048 11114 12 +20037 44194 15 +17297 47233 10 +39999 54763 14 +57285 8348 8 +55776 17512 12 +21685 53556 11 +41114 46837 9 +37096 39930 10 +20154 37539 14 +6893 6637 11 +13440 15473 9 +40227 53805 6 +163 36690 9 +8724 16517 8 +43149 40218 0 +49623 32878 10 +24932 18053 2 +53829 13477 13 +10441 18718 5 +51840 33358 18 +38169 35614 10 +16242 35140 11 +17429 41598 9 +55443 22021 4 +44153 45640 10 +50971 10188 2 +5506 59021 14 +53687 28597 10 +37958 39849 5 +19385 55904 12 +14994 49080 8 +46966 43077 4 +54879 1501 3 +45932 50773 4 +21710 43225 13 +52309 35991 10 +23228 2092 9 +15301 1574 14 +29618 8581 17 +35062 12174 13 +6664 35949 8 +55878 53844 10 +15314 8613 8 +49572 46307 13 +28674 57921 16 +41754 52777 6 +40655 28982 10 +18433 21665 10 +44696 14209 8 +46253 9615 8 +38199 38888 8 +23083 5264 11 +5753 9959 6 +39171 24109 6 +40306 7377 13 +36192 29413 10 +3285 40695 10 +35688 54346 11 +51038 47484 11 +40931 16992 12 +54921 48834 4 +9110 33670 1 +3283 18926 7 +28123 13265 12 +29142 46139 8 +10748 49271 10 +33825 21932 5 +20421 41181 11 +16167 1293 7 +53410 17036 13 +18559 29292 17 +9539 25280 14 +40934 19118 10 +47071 6087 14 +55466 56707 7 +9657 22033 12 +4846 17351 3 +56876 27683 9 +5872 15561 6 +29384 38285 10 +45167 38154 10 +44781 21335 12 +4215 32212 10 +6752 25969 16 +5013 19682 2 +26227 29421 6 +1227 25106 7 +37098 4967 15 +30670 16824 15 +32701 45980 15 +14706 18595 16 +1644 2245 3 +54710 7978 4 +26966 53861 7 +28657 22291 11 +29540 51543 3 +42529 52785 12 +57240 47982 6 +4408 51338 10 +34311 12514 10 +21265 48366 13 +35046 13788 6 +33447 38057 8 +32125 51913 13 +22116 27117 14 +59718 10250 16 +37820 35042 13 +47728 34921 16 +15159 21285 9 +17122 59628 2 +49505 54408 8 +37089 8641 14 +17762 6169 12 +11349 17048 7 +27763 3177 4 +1371 11721 5 +6076 23291 1 +13162 56160 14 +579 33343 6 +44887 17500 3 +12526 27433 11 +44609 16649 6 +25535 25364 4 +57687 3391 4 +36988 30888 13 +20657 20771 16 +8013 57532 11 +47316 19112 4 +31732 52424 12 +47169 52208 3 +55384 19199 14 +1707 27088 10 +45666 4903 2 +29875 51696 9 +16521 54231 17 +43569 3086 14 +26998 34050 8 +20734 11680 15 +14561 43037 10 +42666 31290 6 +33468 54673 8 +20697 41170 9 +35784 14057 14 +6515 41560 10 +44731 20755 5 +58071 28302 8 +29008 21947 8 +51954 2721 10 +27596 21985 8 +23451 13550 9 +28292 40991 7 +58314 27300 3 +17313 8394 5 +25309 28364 10 +26614 6233 9 +11804 21682 4 +42004 43073 11 +50908 44784 11 +2703 9733 9 +42007 32881 9 +52194 14880 9 +35007 30851 12 +36727 36921 9 +51137 57552 6 +27680 23124 2 +12057 55422 7 +45397 38967 15 +43259 41470 16 +36261 17610 14 +33634 7484 3 +28309 14226 15 +54089 11912 9 +21546 34928 2 +3486 59690 6 +47473 50481 3 +38632 27218 13 +21525 47347 7 +19838 50198 15 +1759 34931 11 +22675 19980 8 +11005 6267 10 +40114 11601 9 +37243 47099 10 +57225 33707 4 +53128 48700 11 +54210 45582 6 +33462 20556 2 +59814 59757 11 +58357 52260 9 +8691 52857 7 +12773 14139 11 +24944 25520 12 +41956 19238 4 +7894 16345 1 +50154 29478 4 +18764 38275 4 +37619 41075 4 +35265 35928 16 +26334 27169 10 +19554 39018 1 +29182 21172 12 +57983 28723 10 +4637 58406 11 +10745 3079 10 +12819 10324 14 +50174 23586 9 +17370 57565 13 +136 12912 7 +52789 59006 13 +4793 21504 2 +17151 34813 9 +53154 54770 13 +50406 15972 14 +35453 17052 4 +16761 49905 5 +51020 56491 10 +58184 40420 6 +31134 19120 10 +42149 33333 11 +1673 29921 15 +9237 44701 15 +33617 37171 9 +38178 27373 14 +19523 10874 6 +3178 7161 5 +58645 30680 6 +15663 47084 6 +28844 37007 4 +13907 37871 14 +31811 19489 11 +34060 43182 14 +13443 27089 4 +29620 37886 9 +51594 26209 4 +43481 36087 9 +1344 47377 11 +32199 1784 14 +58376 7579 4 +39896 53836 8 +32205 28617 11 +35692 28913 6 +15995 43197 7 +41213 8085 15 +46232 11186 13 +17409 8355 3 +12559 19365 10 +11287 10590 8 +7140 762 8 +58749 25735 6 +59099 2997 9 +25613 18493 5 +43554 32363 7 +1322 53178 8 +49435 46045 9 +13938 58763 6 +59660 53705 3 +41317 29543 9 +7286 32028 7 +35709 3080 7 +21928 7982 4 +35463 45630 6 +51938 48448 13 +51430 15936 12 +22477 14224 4 +25057 48904 4 +49756 58505 13 +40294 57591 7 +34701 51436 4 +32257 36130 9 +50979 59357 5 +37488 45625 17 +15891 27757 10 +2762 28665 10 +11270 7309 12 +52000 38936 12 +13346 50321 13 +27347 32710 3 +2204 25930 9 +15357 47818 12 +36660 7117 14 +792 43874 10 +940 16446 14 +29744 25894 8 +2174 12561 8 +51024 6441 0 +34558 15202 9 +3195 14136 6 +53014 19377 3 +39056 53215 13 +41624 16846 12 +23762 738 2 +27353 15878 6 +10602 21183 9 +2376 20013 13 +18852 5804 7 +20036 51645 9 +32696 10375 12 +29246 23069 5 +27178 16093 6 +58411 33057 10 +57310 56358 7 +55818 795 12 +39322 39288 9 +16143 31817 11 +35519 26193 11 +14631 46776 9 +20713 33088 2 +32525 23380 2 +3659 32651 9 +15119 6117 0 +35336 12285 7 +31689 42675 3 +23010 6843 14 +43388 44837 11 +48717 45707 10 +8729 9949 12 +40960 14558 11 +42183 58569 5 +34276 33260 10 +21024 48548 11 +6666 32323 12 +48151 3476 15 +14309 59549 7 +46472 38223 7 +14311 43751 9 +3377 52361 1 +45191 23637 12 +29813 56166 15 +33055 44646 10 +49156 15716 4 +28001 15428 7 +10841 33229 10 +49571 58606 14 +1797 25665 6 +55168 45221 7 +53396 30084 12 +56204 42014 5 +11819 13657 7 +33975 18912 7 +14268 28958 3 +58000 2669 5 +292 31023 12 +26564 27547 4 +59221 49759 14 +45782 26014 4 +33021 18296 18 +50923 15068 8 +5320 56209 16 +47296 30317 7 +19621 58495 7 +24439 13000 12 +33274 41358 2 +42221 29060 7 +628 45067 14 +12320 58398 11 +43961 1690 5 +24539 15982 5 +27363 46807 9 +15011 21480 8 +51846 16196 10 +22256 35106 11 +18794 23280 1 +11821 242 10 +8000 33114 3 +38405 47396 8 +40486 55077 6 +52608 9603 12 +39727 47900 8 +40848 18604 5 +16870 40860 9 +10920 475 3 +3620 4036 15 +43814 39523 16 +39908 24403 11 +44327 34893 4 +13912 49175 12 +875 42286 7 +15513 54010 7 +24037 7107 8 +7020 32833 7 +57720 53540 6 +3972 17023 11 +15548 15978 7 +13238 56794 7 +30761 55685 10 +21922 36671 7 +56737 1202 7 +49537 19268 11 +39878 18491 9 +14889 8389 13 +56801 31927 7 +46179 4487 10 +37624 33373 3 +6261 59374 10 +54894 19797 7 +51711 20083 7 +26573 55866 7 +27067 4654 4 +23612 1652 8 +6606 24498 16 +3737 41072 13 +16826 16592 13 +57098 10256 9 +23471 30469 15 +41768 54684 17 +10568 7724 7 +30510 27120 10 +12256 25429 12 +57731 20546 17 +1278 23858 10 +44170 13022 9 +51073 5706 4 +3763 37040 3 +4152 48256 9 +39080 22766 8 +1719 16978 14 +6269 5582 4 +24863 35479 10 +55511 18639 9 +14666 17888 7 +58104 31451 5 +37567 22038 10 +32278 22951 9 +30981 52913 17 +47500 13166 11 +1356 44989 11 +28012 4462 12 +39656 50455 7 +55244 7530 14 +26858 2746 4 +45716 5876 6 +58978 12207 10 +40815 19789 6 +27688 16454 16 +23460 41855 11 +18962 20309 9 +16856 3591 7 +55628 54988 8 +24425 36428 6 +19881 20858 14 +18173 14153 7 +19486 14125 3 +41209 32409 4 +57665 12094 11 +29508 43100 10 +41501 18073 9 +41415 47703 15 +12215 7321 6 +1543 17524 14 +43485 11776 11 +29198 11810 15 +35872 1301 4 +34858 37176 9 +50611 937 7 +11630 4512 4 +15783 47701 8 +30629 15946 9 +49669 9762 17 +43108 28332 12 +14055 16055 17 +3090 30829 10 +58246 57738 7 +57775 35002 10 +16425 24965 9 +41373 20396 8 +56908 865 8 +43357 50548 13 +38508 17047 7 +27776 19748 13 +50300 56994 6 +28445 48115 17 +13067 58200 9 +59247 51111 5 +1187 30351 9 +650 56107 9 +54564 19454 8 +26644 53381 3 +48088 39167 11 +28808 37826 13 +8030 284 14 +44496 54378 11 +7063 52841 8 +34101 23499 4 +43392 1511 13 +8039 8283 10 +1352 25560 14 +16841 14234 7 +51494 58600 6 +29801 40552 11 +12030 39837 7 +28607 14996 2 +22832 9294 3 +25123 3622 7 +10858 19907 8 +3399 54541 6 +33744 16670 10 +29258 21820 9 +23974 14970 9 +18358 1303 13 +46456 2502 11 +47634 38170 10 +4277 14010 18 +36410 20276 5 +45811 2350 10 +7320 30923 8 +11848 23324 5 +53694 28223 8 +4820 23269 9 +37292 43435 9 +51196 906 7 +19514 41299 7 +46057 42797 4 +5026 55785 6 +26235 27961 5 +5803 33154 15 +21225 34633 12 +46024 19350 14 +57462 50094 6 +29091 37008 10 +3530 5522 15 +26186 4158 15 +57842 27411 6 +54666 51810 11 +37837 15646 7 +34856 18138 11 +8485 44960 12 +20318 23764 9 +2708 5019 1 +29911 49090 4 +43334 21865 13 +38959 41134 11 +12978 17045 9 +10695 17654 12 +5016 6928 2 +24693 42699 8 +35739 54839 5 +30974 18859 12 +56982 37027 5 +57252 2338 4 +37902 13793 8 +1079 5593 10 +46549 54317 10 +20777 30455 5 +12429 7126 15 +44130 47457 9 +30935 2141 16 +30330 59666 16 +2275 5976 10 +1370 28800 13 +24089 11027 2 +45651 29788 5 +32654 49863 7 +14011 9835 16 +8185 17805 3 +15578 14688 6 +40786 17075 15 +45140 35867 6 +20968 25086 12 +12384 58765 4 +4295 2988 4 +52945 58872 9 +44246 39431 13 +29188 45681 6 +4755 27288 8 +52330 18467 15 +19026 14837 6 +20621 50871 16 +10099 7844 11 +56090 17673 12 +28097 29035 11 +3198 40075 0 +34388 43457 7 +12639 51609 13 +54291 26434 5 +32302 52113 13 +39892 2984 9 +39021 46450 13 +54847 12857 14 +44868 53723 16 +9906 27778 6 +32827 38570 4 +43592 51941 13 +32530 32856 3 +19027 1396 9 +44593 41612 4 +42682 22148 17 +47625 58152 12 +3546 56773 10 +58424 58235 10 +35845 53281 15 +53253 14338 15 +31806 14807 8 +25296 39613 12 +55294 56287 11 +53489 6540 3 +59775 28863 10 +2377 24517 4 +10755 12481 2 +27203 7127 7 +54592 3267 16 +50301 12196 7 +43247 22269 2 +33317 52715 5 +3319 57100 12 +47973 9970 10 +46814 58633 14 +3499 28193 11 +1416 4683 13 +27719 1746 9 +33224 49392 2 +26823 32423 5 +1355 22105 13 +39424 6241 12 +118 48152 6 +57953 27518 5 +27398 25505 10 +439 38503 12 +36888 40926 8 +4925 5203 7 +41354 30513 11 +22158 9204 1 +13004 30305 9 +17707 6631 15 +57758 54279 10 +21929 2542 10 +30340 17568 4 +11689 49253 12 +14009 922 11 +42240 15409 5 +32070 33331 4 +3481 47380 12 +52685 5816 8 +33448 21615 7 +14845 23857 9 +49104 52044 14 +14093 29488 8 +58273 21874 8 +59769 41404 6 +15395 52737 14 +30138 40881 11 +22541 56074 12 +43794 10469 2 +6528 51274 10 +15823 35165 5 +38316 27704 10 +49981 50922 6 +9091 18798 12 +27377 52582 7 +30477 23492 3 +33307 46872 15 +24713 8840 8 +2575 42520 12 +58779 14527 11 +18185 12504 4 +15767 1 5 +26525 24550 16 +5004 52738 2 +840 38055 3 +31639 6123 3 +11818 14150 9 +55058 50743 12 +26653 51185 13 +23709 49781 2 +26432 46436 5 +45530 18359 8 +10228 37477 8 +28259 31404 8 +16746 128 7 +50305 36863 16 +47149 15014 10 +22144 28085 6 +8468 2374 10 +40362 32810 1 +56027 12234 4 +39453 49333 11 +39979 59083 9 +51270 3863 11 +34913 4068 8 +36008 32787 12 +44993 11039 5 +18786 2146 12 +41808 30015 12 +2259 9282 10 +4058 56537 9 +14720 52646 13 +17792 38186 16 +10754 6998 7 +6286 17372 5 +35086 5312 10 +10808 16366 12 +6882 33801 15 +19795 14135 7 +59591 42512 15 +1444 9650 10 +27156 50058 12 +32122 32793 7 +55549 48289 12 +51199 51678 8 +1508 26987 12 +36193 49318 8 +26211 39802 14 +50924 55606 8 +22902 6234 18 +55804 40897 6 +59556 34656 2 +38569 29735 7 +55218 5639 12 +30490 15384 12 +39687 49727 9 +5462 39286 9 +3249 1434 15 +14961 43227 8 +54743 56184 13 +12038 44640 8 +2369 46255 17 +21797 29771 9 +11583 10698 9 +36156 9550 8 +27649 51684 7 +56279 14173 17 +48675 13809 13 +19488 50122 12 +44880 11652 12 +24893 56243 2 +43426 40092 9 +17476 17581 0 +35606 16306 2 +58595 13734 8 +20210 13986 9 +28361 57041 1 +29275 14198 10 +54632 24205 10 +19767 35853 2 +37 33669 0 +14384 41955 2 +53837 28183 11 +3137 44446 6 +40765 46011 5 +9771 50711 17 +53914 31186 9 +46592 1828 3 +50019 25749 6 +17878 16981 17 +34047 8526 13 +37971 17937 15 +7543 31044 9 +33818 11685 7 +39759 33389 2 +9102 6476 14 +27747 38528 8 +56573 35982 0 +38775 58096 18 +57621 44608 10 +52684 3595 7 +57877 13157 4 +43060 53841 17 +1615 34024 11 +36531 37963 14 +47150 17842 4 +59 25743 1 +59536 28721 6 +39446 38398 1 +52028 17280 3 +20597 26342 9 +50978 29864 11 +59542 37414 13 +24882 39832 15 +10410 45488 11 +15223 45443 7 +19643 25649 8 +31168 13314 8 +43545 10749 11 +28925 12303 4 +16712 38862 9 +52303 55811 16 +35048 13 7 +11096 47151 17 +7276 9723 11 +28453 6156 11 +48918 54286 8 +8816 55777 9 +18351 13289 16 +27840 40640 8 +21419 1285 11 +2550 53741 7 +10732 51743 2 +46029 3589 3 +12927 34838 9 +47385 50844 11 +48998 35906 8 +7553 22987 1 +34019 16868 9 +34003 16715 9 +9246 46577 6 +31042 47806 10 +22663 22302 8 +37952 46362 17 +4711 290 6 +31765 59438 16 +8376 23932 13 +26437 26226 14 +43408 54353 8 +34510 25724 9 +39700 46473 12 +40737 42344 17 +44466 11484 9 +31792 56714 8 +39893 52389 14 +18494 59887 16 +20204 18317 16 +44217 24204 9 +7654 52285 10 +34254 11040 13 +14254 34607 12 +25436 4416 6 +43776 5070 11 +43445 1411 4 +53356 45621 14 +59179 7319 8 +50475 7061 11 +10772 24850 12 +22024 43635 5 +7149 37038 2 +23135 26466 9 +11528 52864 10 +55272 19912 15 +30187 7352 7 +51021 9709 10 +13174 48270 9 +41997 12515 11 +36955 7826 5 +43540 27069 5 +24292 18327 8 +2073 43527 2 +22518 4543 7 +50835 58219 7 +58156 37708 8 +48553 39630 10 +42757 42294 9 +28130 51628 7 +38843 36011 10 +47384 56747 7 +22392 59104 10 +17079 22670 10 +58266 47442 2 +30908 36344 10 +19159 45683 9 +54421 45781 3 +36076 58064 6 +2169 44114 10 +23747 22865 7 +29127 3013 10 +7346 40248 6 +56273 53680 10 +26674 15726 8 +38081 34747 3 +17543 12812 8 +34258 32737 14 +29791 44470 10 +41458 55560 9 +14681 46133 16 +34497 27640 7 +11058 11187 1 +10227 25803 5 +57874 42076 17 +5589 59935 16 +11384 306 7 +3429 46333 4 +52385 46603 3 +50556 25124 7 +54484 3194 6 +40795 36419 11 +38624 15728 14 +11218 1881 8 +10057 9985 8 +51857 14204 11 +5355 17630 8 +17024 50112 13 +56266 23410 9 +31721 25807 10 +19824 22905 13 +15734 27086 6 +54611 8714 10 +52518 58234 7 +23172 9737 2 +34036 7943 7 +4656 41795 0 +32069 24230 13 +29932 48725 5 +10342 28318 2 +16676 58886 4 +5267 48859 9 +32121 31199 9 +3271 15467 13 +46483 30838 11 +30315 34223 12 +32269 31249 5 +28523 32319 12 +9500 5841 8 +27711 18200 9 +32276 14668 3 +59080 52761 18 +25784 13764 4 +27949 2626 11 +51382 57803 8 +54537 56692 12 +1374 27954 7 +41801 35727 13 +32250 797 6 +9717 47023 2 +7026 55220 10 +36717 12372 4 +17806 10563 1 +24251 33185 5 +17721 14399 14 +29326 22272 12 +39920 2508 9 +44919 25594 8 +38980 13170 12 +54822 25718 2 +27025 47766 10 +12735 46734 10 +32565 49529 7 +45755 47382 7 +12205 46422 7 +58331 18149 15 +41979 25986 17 +17983 15456 13 +19032 9392 8 +1648 14875 11 +13527 51716 11 +32522 54427 10 +40200 3493 8 +34624 7169 7 +9221 4528 8 +7550 10711 16 +42073 35956 12 +34566 41977 5 +8382 15021 9 +43813 23572 4 +48813 8737 8 +37939 40708 9 +52218 19811 8 +2366 27170 1 +50447 57890 3 +41048 8666 11 +2120 1944 12 +245 23371 10 +7397 13970 9 +25070 1776 15 +41902 21520 11 +11584 11215 9 +6883 22002 3 +50145 12389 18 +55637 3917 4 +23697 34116 8 +12791 11535 4 +56264 22826 9 +14240 4730 13 +13019 8161 3 +9831 57548 11 +32825 16284 10 +2293 55500 7 +41999 30364 1 +52939 5432 6 +33145 5845 5 +7255 30462 14 +492 727 10 +59762 58807 6 +48324 20196 12 +30253 7450 12 +35116 58713 6 +34034 40353 13 +28839 19774 2 +32009 59291 5 +53623 34583 16 +52093 32601 11 +23735 25141 12 +25753 56999 11 +8450 27061 10 +55900 43641 11 +56816 39415 10 +24681 53951 9 +12212 4451 3 +22855 54691 16 +48505 11158 12 +20691 5650 8 +41554 40346 6 +17316 28304 11 +7068 33831 5 +16285 464 0 +34863 22465 14 +9142 45094 2 +17302 3839 13 +8273 21516 9 +28305 38686 0 +55010 47032 11 +40953 8205 15 +2872 55609 2 +29666 40050 10 +18517 5798 9 +12455 11930 12 +52412 2258 11 +57116 11519 10 +964 56868 5 +25078 22480 7 +57726 4126 7 +20798 52513 7 +34557 53138 7 +5785 6967 1 +44923 7741 11 +2122 47850 2 +30130 19000 11 +16515 49853 9 +24373 49780 3 +27294 39124 11 +53590 24314 12 +31497 45637 7 +6514 5899 13 +57377 21690 5 +59308 3695 8 +32804 57020 3 +38166 14748 8 +45568 47799 3 +17325 29762 11 +38991 27206 3 +1349 14661 6 +12067 23985 14 +56253 3223 14 +18978 2003 6 +24042 17234 12 +42239 36292 6 +40680 41505 12 +43909 10417 10 +53249 9421 10 +58512 51228 2 +3694 39347 4 +24463 11455 7 +33699 39354 10 +23716 11676 3 +50919 34831 7 +35726 24229 8 +28530 4161 18 +9584 48689 7 +6824 34271 13 +54865 54850 9 +9483 23354 15 +27366 8981 9 +37439 11788 6 +22501 33588 15 +27514 32191 8 +35799 38582 6 +58675 28382 7 +5064 45177 10 +35047 44426 9 +51851 54190 11 +20338 41079 8 +50471 18010 6 +45760 1686 6 +27405 25557 11 +15442 9866 8 +42039 48683 4 +1960 4069 10 +58117 46555 8 +59947 41100 5 +46760 27615 11 +32927 45575 15 +8580 12995 7 +58676 34759 8 +38697 18657 9 +9860 54577 11 +823 10963 16 +30209 25993 4 +53801 49142 12 +52547 20187 8 +56108 34094 12 +43341 48589 16 +37759 17669 1 +21429 38701 16 +35548 37562 7 +16858 40956 4 +6363 14400 11 +27583 46449 3 +58529 54505 17 +13889 28975 6 +57604 14888 11 +3522 31633 8 +46730 23152 12 +38709 39973 10 +54355 28352 10 +47102 42184 3 +27968 59840 9 +19413 15854 3 +28132 51293 9 +17855 57354 10 +10659 48922 4 +57433 50565 4 +14162 7559 14 +52642 36438 13 +10019 49886 13 +37736 11205 5 +16375 1569 12 +55666 35597 2 +27133 47479 3 +52467 16815 18 +4630 43741 17 +25925 10064 12 +23005 46731 9 +36345 45317 15 +51986 12794 9 +19596 12093 11 +14674 58902 7 +45888 31622 17 +40820 32507 5 +17431 16842 9 +34525 43720 9 +59576 48403 5 +18290 9319 5 +13529 28729 15 +25550 17821 6 +43157 33678 12 +49432 25587 17 +38132 39834 13 +6691 5535 12 +28244 12841 8 +24005 58225 11 +35898 38180 6 +54896 50223 13 +496 13728 14 +54325 49958 9 +51271 35818 13 +47349 43611 16 +56062 19359 18 +14651 58416 5 +39350 3617 15 +48240 22206 7 +28769 11621 3 +43844 7381 10 +56 51644 9 +38715 13836 5 +1870 52766 3 +51990 9006 8 +26754 5830 6 +57394 34504 11 +2587 50362 4 +11964 604 7 +31185 39161 4 +19135 38308 12 +39875 58020 12 +27305 3587 1 +55972 20632 2 +57891 58605 12 +40598 50474 4 +6644 19053 7 +39338 41851 9 +35973 43405 9 +17452 6795 8 +37531 24150 9 +50493 46828 10 +28018 50226 10 +54659 41655 9 +2930 56863 12 +52297 35337 13 +15733 5392 9 +13115 2411 9 +39806 26793 13 +49526 41056 6 +4530 53510 14 +34446 16834 12 +11367 1028 9 +11116 58482 4 +47931 30638 13 +48395 56594 9 +33244 16193 3 +49791 26268 4 +13848 35406 9 +49782 47215 6 +2038 760 8 +39594 29030 2 +34486 39843 1 +43845 59460 10 +13106 52200 5 +5879 23855 10 +48280 37923 9 +37961 47383 10 +37474 1512 9 +19536 59205 12 +18950 38486 12 +46341 36370 16 +51279 40266 7 +46238 21767 10 +5286 18297 4 +43018 48144 15 +20269 13587 14 +54671 30573 9 +9092 58860 10 +55908 18148 6 +47619 7373 0 +34281 44283 8 +14489 55152 5 +19775 48235 9 +35552 29085 3 +28806 47199 6 +20595 56167 9 +29007 38347 10 +9652 2112 5 +35847 23294 11 +28174 34195 12 +32469 27429 4 +44793 38246 12 +50266 45079 9 +15155 20202 14 +18991 19311 4 +53968 56977 9 +9620 55512 9 +15661 55304 7 +25487 43818 4 +25951 47302 10 +38233 44026 18 +31080 18238 14 +13494 746 10 +37841 20490 4 +15966 9063 8 +57728 58677 10 +54618 406 7 +3014 33843 8 +53578 5883 6 +45178 41889 15 +7383 10409 4 +22494 38679 10 +14221 16508 2 +30862 53317 17 +48064 40912 7 +54699 45821 10 +26485 22963 4 +36226 48060 12 +48331 43331 10 +50416 58880 12 +30839 2423 9 +49900 33214 6 +58277 3925 11 +45697 52401 15 +1970 170 14 +48035 58288 10 +19519 24831 10 +47610 19884 3 +54928 19874 10 +58548 54277 6 +24459 6572 15 +50218 8157 2 +47739 51725 8 +50078 21261 12 +53359 16677 8 +16538 35907 7 +14092 51446 6 +31697 21694 17 +41329 48 18 +17319 17338 14 +52322 20308 16 +5337 20056 7 +51244 58522 14 +24521 21926 16 +529 23113 9 +53142 50126 16 +22983 54200 11 +14190 28398 10 +39798 31077 12 +27321 59948 16 +37680 15780 11 +30437 29989 17 +43255 16178 3 +7414 26723 8 +56126 15803 9 +47567 45845 10 +58930 15868 5 +47472 9158 6 +50412 11596 8 +2646 21115 10 +35904 53351 5 +27842 45335 12 +57434 2598 16 +53441 2154 8 +56590 16160 8 +32880 41513 3 +2488 15516 7 +26406 23741 1 +2392 8212 6 +9425 33050 16 +17392 13904 9 +15641 23481 8 +38262 19694 6 +59550 44347 5 +20254 57459 13 +53992 30822 11 +19737 11253 8 +24426 30522 7 +45279 4099 11 +20044 55897 11 +40481 14645 11 +40422 47403 14 +16551 52651 13 +21647 40945 11 +19759 21840 14 +41704 6492 5 +31757 31648 9 +55948 2915 7 +46208 3121 12 +36330 39346 5 +46990 31821 10 +19111 59649 12 +31482 55527 6 +34551 233 5 +30391 47489 9 +8004 56385 12 +41862 27996 6 +6153 973 14 +29881 15916 10 +19126 21114 14 +31222 10810 14 +39134 14971 9 +39697 23496 11 +37842 45429 9 +24568 9542 9 +106 56222 8 +34692 24381 7 +43199 36776 13 +10159 57939 6 +49846 14214 12 +2816 24244 7 +12721 53445 17 +8582 56631 4 +7953 40165 9 +50779 14629 5 +27399 16643 9 +36020 31661 8 +57057 30080 1 +59500 13908 7 +51188 29462 6 +57288 44236 9 +55498 25809 8 +19734 54234 7 +48375 7437 15 +39412 22182 6 +24833 46429 9 +58409 1982 9 +39510 55608 9 +40349 52958 10 +39856 59190 5 +47441 2882 3 +5075 48579 4 +48507 42608 6 +50812 41037 4 +43159 47039 10 +56704 55150 1 +16059 11540 15 +53753 27645 7 +34889 29267 3 +50629 29794 4 +38661 37495 6 +1780 58038 8 +48880 3920 7 +20100 47696 2 +52697 12718 5 +8265 24950 14 +42970 959 6 +45145 33033 5 +58858 32382 2 +58684 51564 5 +955 14774 10 +48565 53788 6 +12456 57691 6 +9799 43456 11 +29546 54617 6 +41858 44454 2 +53888 24203 6 +23383 40654 13 +9368 1128 5 +26600 2210 15 +33103 1705 11 +22960 7391 4 +23173 47114 9 +51155 20130 13 +4446 57673 3 +34674 4113 10 +49065 12814 3 +42979 43618 15 +2029 42946 12 +2649 37332 12 +12482 7614 10 +683 17713 9 +36653 7028 13 +6898 35589 11 +20832 49694 9 +23253 36675 12 +55211 17858 17 +48387 25454 9 +30812 36610 14 +18760 12573 1 +27499 56317 14 +33998 51812 10 +18146 23981 14 +27644 1088 10 +23424 37267 13 +47008 18108 11 +2819 49 7 +43652 55702 10 +26768 12947 6 +353 26517 12 +18095 5566 7 +53760 14192 12 +8928 11246 17 +10361 57716 11 +14497 7510 11 +51437 4095 17 +39366 5261 12 +33727 12017 5 +6001 16906 1 +28918 41281 14 +53131 52994 11 +15937 37884 7 +17663 19697 6 +38156 21530 11 +35398 9135 10 +23847 16043 2 +5884 26810 6 +10294 13683 10 +32487 31629 10 +7504 52259 9 +33201 9759 11 +35305 4033 9 +38428 19308 4 +4564 4067 4 +58627 40777 12 +1164 39959 6 +41796 14402 4 +33921 47935 1 +10699 13604 3 +43353 33747 7 +10333 5488 5 +53028 33445 11 +41007 13088 12 +59715 22666 7 +25661 26478 11 +700 41104 5 +655 57937 8 +29424 14264 8 +14287 49642 9 +39260 20997 15 +57112 34469 10 +49450 21431 7 +47738 38674 15 +11643 21389 13 +560 25361 11 +13468 6802 6 +18973 42331 17 +8902 48528 17 +34205 49083 10 +58688 37657 12 +32751 36492 10 +2348 55470 10 +12914 50082 15 +48740 46889 15 +11728 48120 5 +36915 41780 12 +2040 45679 9 +10530 13911 0 +43119 49943 5 +2269 12209 1 +8620 24423 15 +43929 14405 10 +6512 29861 17 +18840 5791 11 +53255 6500 3 +23180 32794 11 +19398 21754 10 +4053 49963 11 +11009 49015 8 +11264 23340 5 +41189 15017 2 +8444 21091 16 +30776 19314 11 +11616 49607 10 +58796 53207 11 +33197 43517 5 +31737 31880 17 +33277 38321 3 +5745 51662 15 +24676 33379 6 +32729 34829 14 +12274 43827 3 +42803 8434 6 +32716 15212 16 +44840 14949 7 +2304 17969 12 +5228 7558 1 +34959 29717 9 +24183 42002 2 +50267 46866 13 +1137 7459 4 +41794 33986 5 +5309 13979 13 +20761 33849 6 +32115 14365 12 +17194 43180 11 +47892 17290 16 +38912 16610 9 +51651 52956 3 +295 9993 12 +23303 9664 4 +37523 39883 12 +31403 54804 4 +6489 3310 12 +2407 52073 5 +30775 29754 12 +22125 15420 13 +6400 3753 4 +24634 10591 7 +26389 26323 0 +4279 23275 9 +35983 23794 10 +37032 7434 14 +30712 21755 15 +9405 28138 13 +2640 29452 16 +32451 593 10 +28349 22979 6 +30910 34216 13 +12516 30569 2 +45474 36644 6 +16391 14206 8 +21229 40405 4 +7966 57635 6 +57014 10264 3 +33554 58399 7 +32629 28972 14 +36591 14968 9 +12340 8220 15 +5472 16069 15 +39143 39943 7 +12052 31685 5 +43230 51600 7 +21487 53334 10 +22202 20514 2 +28004 8145 13 +14901 41640 3 +46406 12405 8 +48221 1909 8 +12655 33853 13 +45248 37272 10 +44133 28442 6 +53405 55270 8 +22230 10124 17 +47540 51492 14 +23171 13472 11 +25102 45979 9 +31163 11845 4 +7667 17605 10 +8931 27918 15 +27448 33725 10 +47724 6375 10 +18141 48712 12 +28655 11869 6 +15056 16831 11 +45161 25523 11 +46928 4684 15 +48832 34242 12 +17464 23554 3 +43468 9173 1 +9916 55881 6 +50046 22415 10 +52676 5454 12 +1253 44313 4 +49205 49897 5 +26605 20636 17 +5997 10430 11 +24256 35221 11 +52135 44632 4 +13373 43272 10 +44517 10372 6 +19915 40312 0 +9864 21709 4 +14866 14459 16 +58348 47449 14 +11505 39936 6 +46956 47887 2 +8884 35797 8 +27794 1186 13 +26395 55860 11 +51632 44441 15 +2224 16443 8 +47035 54457 3 +38914 30154 7 +24648 54506 5 +20399 28787 6 +26393 43387 6 +6384 4845 6 +11743 8270 5 +38930 52469 17 +23472 5907 4 +47226 41527 9 +9346 44711 13 +1782 21170 9 +14417 49833 10 +45654 45468 12 +37247 59101 17 +7261 30256 6 +47962 54797 9 +56860 49950 10 +14622 38032 13 +18380 9039 4 +21134 11672 12 +4662 27424 6 +53470 10146 10 +37379 28894 6 +22446 47812 2 +40899 39070 10 +49146 42306 13 +29899 2715 7 +23440 17068 15 +39073 33690 14 +51112 8514 10 +20780 8903 4 +41140 25976 13 +24512 26797 7 +29228 48973 12 +35866 3802 5 +22805 24911 14 +53211 9360 11 +16236 40808 9 +54136 25954 13 +32639 55180 15 +19702 23885 6 +54153 36397 8 +24454 24509 10 +16004 9188 14 +20578 31818 17 +5119 19081 13 +41393 8336 14 +57222 26313 6 +23721 19190 11 +39302 24907 11 +7280 47047 10 +22060 12032 16 +18020 43633 8 +40101 2656 7 +10033 19187 11 +218 28933 13 +9330 24196 8 +9224 15849 14 +15125 50944 3 +17428 39836 7 +3637 32088 9 +3738 39004 10 +48473 4540 11 +16718 39436 6 +47793 17720 16 +48021 10692 9 +46841 50465 3 +4153 23541 2 +824 41297 14 +13919 59187 2 +2986 45534 12 +12225 10211 8 +16951 4007 13 +54156 10325 5 +56762 21773 4 +23972 3560 6 +10783 44079 2 +1827 44951 4 +44025 390 11 +52576 7506 9 +30020 23511 6 +11091 46633 8 +58039 50194 6 +10209 42640 13 +20864 42548 16 +41090 55202 3 +14105 50172 12 +54495 41615 15 +22280 15419 8 +44806 49012 15 +40185 28323 3 +45199 8146 14 +20553 32059 8 +31665 13936 11 +21031 5772 7 +6333 4132 11 +51318 14461 9 +45159 58931 10 +1517 4610 2 +33827 15864 8 +2720 34126 12 +3239 32024 6 +49858 41022 8 +45779 36084 2 +15167 5744 11 +15089 8657 13 +26947 38565 11 +46067 10585 9 +21339 6965 11 +22677 33989 7 +1365 56862 7 +40954 16145 9 +25770 10577 15 +622 20275 6 +30002 27487 7 +32395 1228 13 +41730 22873 10 +281 59245 5 +49656 56753 18 +44061 21297 17 +10382 33152 17 +25056 21591 13 +34137 40247 8 +16171 4771 12 +22937 19344 8 +38446 13722 2 +47193 20948 11 +33542 15251 9 +24727 18373 15 +42676 53186 12 +22044 30428 9 +25755 52426 6 +26874 33647 6 +31754 47568 13 +53674 19649 4 +41757 59976 12 +5810 22433 15 +20198 10634 12 +39067 53135 7 +5725 18932 13 +59907 3631 10 +35255 28522 8 +4666 56088 7 +28128 36147 6 +53755 30445 6 +13066 37978 1 +28941 16829 18 +25161 27550 7 +1295 27174 12 +37998 13672 15 +4080 30516 10 +46023 3236 16 +21414 37417 10 +58455 46111 7 +7360 11343 6 +31772 33303 10 +3721 50457 9 +56887 24984 15 +24125 57971 9 +34206 32951 1 +30659 21633 9 +9056 42163 7 +21816 12045 15 +22707 24880 4 +55937 39865 11 +15038 58260 5 +32615 18086 15 +42893 38561 2 +6103 52229 4 +39928 22016 10 +7663 30693 16 +40944 44276 1 +26382 27537 8 +21168 1617 7 +46782 58887 2 +54939 18264 2 +29332 14149 10 +46215 34936 9 +27391 59847 9 +31246 21383 12 +26256 31424 5 +11393 32378 6 +50207 44198 2 +11904 14141 15 +44556 58498 9 +58120 51339 6 +40757 27657 7 +52098 57779 7 +13099 33483 12 +12267 55100 6 +54406 22244 8 +41033 2413 17 +45419 30858 8 +55056 1756 3 +16388 2110 10 +5387 56030 14 +43303 59166 6 +9443 6736 4 +39153 14404 7 +4272 39748 13 +56054 13285 8 +33382 26601 14 +15476 15117 11 +38157 36179 8 +12120 45028 11 +10092 28888 5 +59651 21361 9 +21937 59326 9 +32529 26776 15 +57478 7744 8 +3623 55069 8 +33200 34546 11 +49639 24162 6 +17101 45613 5 +7259 9113 8 +4524 28288 10 +26872 8564 6 +39325 36295 6 +2917 52667 9 +7937 33266 7 +29835 32185 13 +32485 44856 3 +42989 58625 8 +10524 16163 9 +17840 6846 4 +17198 22639 5 +51089 40113 15 +16578 36843 11 +1734 23028 3 +9379 6901 10 +25833 34981 14 +40628 41690 15 +26893 432 10 +40161 2134 4 +32587 34563 14 +25182 35575 10 +55899 50625 10 +19138 34842 12 +31761 24394 9 +24294 8018 6 +54456 46934 3 +11634 34911 2 +5537 10576 17 +41747 18788 9 +44068 42143 10 +14395 462 7 +55954 2607 10 +5774 42205 7 +9446 17813 13 +58192 15586 9 +52414 26314 13 +3725 59668 10 +37799 29711 12 +27270 31314 1 +14912 1626 6 +17164 26687 7 +7312 19969 9 +42901 54446 12 +58150 36820 16 +54912 52571 13 +6018 40094 9 +40499 22132 11 +32101 10601 5 +46264 39840 9 +2767 55898 5 +31018 174 10 +26939 36416 7 +1930 6059 3 +57666 979 8 +13095 23791 17 +340 19674 8 +53189 32379 3 +23313 45758 5 +44493 43662 1 +31303 17903 15 +9074 35750 9 +58743 41419 1 +34549 5832 15 +47598 2586 2 +57614 20484 5 +32180 31341 6 +34741 34671 12 +24090 38551 5 +34663 12797 10 +2273 37956 10 +28029 17019 14 +3877 5022 11 +54172 25042 7 +42009 24703 12 +43809 22665 11 +49337 32365 7 +5464 48158 2 +48734 58400 12 +7122 34428 12 +45542 33859 15 +33112 45309 6 +46475 17116 11 +12809 40357 8 +23850 6753 10 +3974 13935 8 +16595 11796 8 +48863 4616 4 +36296 52351 10 +1048 9957 6 +43138 11459 13 +54342 40450 5 +44568 10380 8 +2440 13934 8 +42439 43911 9 +10988 30343 8 +44518 27879 1 +40571 57948 9 +6459 16792 15 +12593 30928 8 +28780 33213 4 +19116 49990 11 +15342 22131 10 +21973 35814 7 +17144 33424 15 +13178 26290 14 +1687 20453 12 +18127 19929 14 +57272 43092 1 +48703 21708 12 +10796 50653 7 +38669 8003 7 +24161 41645 13 +11021 27489 4 +43300 46113 2 +4145 45281 2 +17621 50275 9 +8726 48238 8 +14798 44959 5 +49510 31155 9 +38052 27483 10 +57501 7588 9 +26894 43246 8 +15350 53858 10 +4879 45423 14 +11738 2101 2 +23542 30415 17 +56121 19662 15 +44113 1697 2 +31210 12727 9 +29834 58720 7 +10385 16881 14 +5430 58901 15 +4157 17335 2 +18402 42123 13 +24949 38550 16 +19530 42291 14 +31598 24188 4 +44667 47658 7 +10943 49750 5 +1840 53210 11 +38691 21672 7 +35166 33844 4 +28165 28579 5 +6805 16374 7 +26808 41053 7 +10416 56533 1 +5815 7698 5 +19610 20961 6 +53220 1098 10 +58715 44092 9 +1945 11457 9 +50734 21055 8 +18257 13723 15 +967 28976 8 +4748 49101 2 +33034 5649 9 +35615 44417 10 +18735 55591 8 +56443 10788 16 +47470 51449 4 +53013 53280 11 +25983 5324 5 +57347 11516 13 +643 23634 6 +36379 54814 8 +18767 52780 5 +43042 5626 15 +44045 15289 8 +20084 11675 9 +51688 46465 12 +30153 56985 15 +10395 22115 10 +14413 7240 9 +58010 38300 13 +49408 37779 11 +45567 17765 13 +50124 47072 9 +34229 49534 8 +58004 205 10 +39192 4492 10 +51963 8229 7 +41352 16959 5 +2710 39550 7 +51767 26155 10 +30880 41949 7 +12309 7157 17 +35527 21706 8 +51130 26666 13 +25620 7753 8 +9382 7523 9 +55909 47732 9 +18461 51133 3 +45511 17641 8 +45266 13927 9 +19563 56756 13 +17461 24626 6 +2207 7973 4 +26232 2740 10 +26119 18648 13 +8968 1559 12 +46536 10446 14 +33311 14081 11 +42449 42758 8 +4014 55996 3 +34240 17307 11 +48979 31907 13 +27119 14594 13 +9700 58644 8 +29460 2233 9 +56791 25836 3 +59364 14531 8 +49992 21613 13 +36099 42423 11 +20539 26497 14 +18254 37266 8 +608 14211 15 +49162 52831 4 +35890 58233 11 +30007 44567 4 +1085 24225 1 +18017 6394 2 +38485 37432 3 +52724 40833 6 +28576 17332 12 +17984 26250 2 +275 20006 9 +58992 39208 9 +22620 40382 5 +14007 18355 11 +17327 6029 6 +46792 23015 9 +28413 50265 8 +9286 5255 12 +14762 21107 13 +46587 41966 8 +30568 34809 12 +39252 21472 11 +55991 51052 0 +21732 56651 7 +52460 56600 15 +26745 7433 4 +48452 32288 9 +11206 56553 11 +24835 1176 7 +39396 103 11 +5414 3197 6 +20449 8979 7 +19909 27219 12 +24772 17283 12 +21392 39809 10 +28148 49466 14 +29392 11508 7 +9340 43665 5 +1518 9438 15 +20905 21071 9 +57851 36393 7 +55129 34477 9 +31521 50497 11 +45925 44182 18 +8423 24380 7 +21880 43209 5 +12166 2835 8 +1241 47899 12 +45765 14683 12 +18287 10131 10 +50599 26329 6 +12938 27184 9 +45612 44893 3 +30412 34306 17 +31911 14370 6 +14897 45537 1 +36873 6728 15 +48008 30661 10 +11478 30532 15 +35272 53067 8 +32229 38615 8 +39034 24892 9 +40981 28960 17 +49955 12444 11 +24700 20226 14 +55496 4217 16 +32334 59906 6 +3430 41750 2 +50729 10130 2 +19541 39505 9 +24078 15137 2 +13922 4052 10 +27857 74 7 +57054 37340 14 +9465 12084 12 +25887 10638 5 +3940 21914 12 +7971 31976 6 +35942 6165 2 +29839 9533 8 +45855 57091 1 +11103 34098 17 +40759 12005 11 +35416 19505 9 +24529 15652 6 +56796 21465 7 +38608 50006 10 +4843 59244 2 +54752 36454 6 +12798 1157 8 +59062 33618 10 +13139 11180 9 +11495 16434 8 +29300 6121 12 +57432 10958 14 +3503 29279 12 +23279 29790 5 +39552 34978 9 +33791 16228 10 +19378 41510 13 +57973 58075 10 +31216 40623 7 +44461 10253 12 +39257 23363 13 +56113 53959 5 +45982 28217 8 +14780 59927 18 +58377 13073 5 +21970 32961 5 +17157 44651 9 +34315 7013 12 +51427 29415 9 +34282 42155 9 +55896 45403 8 +5077 48464 5 +54780 19313 11 +30663 57346 8 +38577 59854 6 +58268 10805 5 +45593 29541 6 +40090 12680 3 +33903 11149 13 +24813 14806 12 +4529 9193 7 +50748 35611 7 +56761 23706 9 +39444 35151 6 +53913 42437 17 +24992 44966 5 +6140 45082 15 +1836 42526 6 +28741 36584 10 +12707 33918 3 +5844 56608 0 +6533 42493 11 +32482 53675 6 +11474 31664 12 +55977 56752 8 +13783 3691 5 +10368 55817 2 +8362 17905 7 +28828 45598 2 +9560 34801 16 +2222 32470 11 +22488 49568 8 +59742 51164 10 +55551 58802 11 +41675 23564 8 +41594 2830 10 +14068 4467 9 +25279 27349 7 +39662 54248 11 +32479 14832 10 +59494 19584 7 +37233 50443 16 +32610 59969 12 +41870 14324 13 +16833 32081 10 +3114 54075 10 +51347 11024 12 +49186 43126 4 +20896 55003 13 +46919 30708 5 +5183 44317 9 +15769 33054 6 +2593 34917 10 +32574 19641 9 +11514 26264 4 +39841 59216 13 +14394 9407 12 +54198 8790 11 +5094 37995 4 +47762 7275 10 +4169 28606 9 +51343 37308 8 +1378 3734 3 +5341 591 11 +41520 15316 8 +20917 45701 2 +26741 30192 8 +18500 40381 17 +56874 19877 16 +53745 47066 16 +45842 43887 9 +40307 52445 11 +52500 37313 14 +1214 31910 15 +39509 50181 6 +33041 851 13 +57191 43933 3 +27632 50561 8 +18620 16094 6 +26663 8802 18 +34614 28216 8 +56170 15444 13 +50825 51194 9 +44577 35706 15 +41488 33381 6 +30735 2061 15 +54328 18206 2 +23728 57727 0 +45690 9595 5 +21449 54812 9 +40671 36787 9 +38365 40882 13 +2281 22812 6 +17668 56134 11 +44628 58092 8 +23959 5233 15 +24330 57120 5 +55412 13753 8 +3291 58247 3 +56326 58735 10 +32399 18363 3 +26580 16115 16 +5269 10298 13 +27096 44250 14 +43081 45256 5 +9706 24666 8 +39186 16089 6 +26945 49697 8 +6884 48122 12 +45012 37468 8 +39535 23422 15 +31079 33676 7 +18407 35369 6 +42482 43970 18 +10896 21313 10 +32694 8009 12 +5492 8301 18 +14947 51946 7 +47908 51280 10 +26732 4822 7 +32755 56743 9 +6326 19525 7 +16378 20131 7 +6881 20321 7 +31547 26763 11 +28524 31197 3 +31049 2309 5 +14915 6992 11 +21652 29297 9 +38494 16027 2 +51311 25345 12 +21864 59910 12 +22936 21135 8 +31705 7031 9 +44697 34745 8 +16597 36306 17 +20947 6617 1 +55582 49479 4 +22317 54094 1 +3616 23306 12 +49673 49272 12 +23270 23827 9 +50775 18344 9 +27729 25912 10 +33604 46340 13 +11269 10498 5 +41682 21354 5 +34344 31696 9 +35210 43196 9 +51245 3711 11 +30309 26265 4 +57853 50740 13 +981 44226 9 +53400 38510 16 +36989 26934 11 +38831 2412 6 +48046 15806 11 +42011 39370 16 +5259 5362 11 +21568 53149 5 +6005 55870 3 +29584 36133 6 +10565 23820 4 +6144 16849 5 +9667 5850 5 +23849 8751 16 +33672 29733 9 +53199 59067 9 +35800 36472 7 +34798 21673 3 +54980 46641 7 +55973 19127 8 +45934 49347 12 +2925 58941 13 +11774 49304 2 +39368 29483 7 +1509 26116 14 +16394 30060 14 +341 50591 7 +32130 41515 3 +30492 30082 7 +7617 39156 8 +45302 2194 9 +10511 57653 10 +31550 20054 16 +56963 27975 11 +12022 22777 3 +31258 55148 7 +28045 27761 10 +48260 52397 5 +15638 20825 15 +41389 22066 6 +26049 56399 13 +37992 10812 9 +2610 24283 13 +8578 37115 10 +54795 3214 9 +7034 44195 10 +18666 40259 17 +9524 46719 8 +19124 47816 8 +24865 19400 3 +19763 6112 13 +4931 42287 5 +34965 1274 14 +44154 49172 13 +15022 24028 6 +52270 52559 14 +1603 59285 10 +42349 26532 8 +49617 5680 16 +12051 15563 13 +48714 13969 9 +16359 5221 5 +25697 36684 4 +37124 2068 11 +10331 52379 7 +41910 50777 11 +33226 18777 18 +59338 43581 11 +36560 991 4 +9131 10370 5 +22763 45880 8 +24786 14585 3 +45063 1884 10 +55638 22917 6 +50965 18481 13 +56591 55691 16 +22313 17205 14 +27643 50751 3 +52197 29153 9 +17336 53711 5 +2337 29371 11 +18877 9016 10 +43139 6768 5 +6188 21940 9 +56174 20055 2 +9126 23810 9 +58721 42489 9 +15847 46783 5 +57037 37781 10 +14941 25862 15 +26863 171 4 +37753 38783 14 +5166 20297 6 +39720 29841 13 +31563 37151 6 +26354 23041 9 +54676 41010 5 +83 11429 6 +35311 28622 1 +32142 21798 4 +25054 24431 2 +44979 27633 0 +48726 54040 16 +2205 57762 5 +18540 4764 2 +52247 16954 3 +31443 19349 15 +20559 24375 15 +54796 49814 5 +41307 9637 7 +55176 26142 3 +39174 42196 17 +53659 53313 15 +5406 26991 5 +3398 12550 9 +56334 55918 11 +14717 32709 2 +8543 40507 10 +51634 12853 13 +23990 41677 16 +18279 9183 7 +2903 53928 7 +15745 37393 10 +20267 21355 9 +15440 25231 5 +1585 7000 17 +34272 38945 9 +3761 32343 9 +31001 37997 2 +48999 29564 6 +7727 18879 14 +20843 26961 1 +42876 10764 12 +32509 48475 16 +17539 21993 3 +7253 42765 2 +35628 6268 9 +9132 50833 18 +39807 20941 8 +49840 15347 5 +58144 58965 4 +55285 57408 4 +47164 17403 5 +43617 6964 11 +5366 24895 5 +18511 18103 10 +39473 1393 11 +27393 26168 8 +29393 42303 11 +47358 39064 15 +8097 59316 4 +32591 19607 13 +48494 43231 12 +19865 7516 9 +51626 9797 8 +8364 55312 10 +25591 21181 6 +14627 50017 14 +53536 30827 0 +47826 14232 10 +2395 6829 10 +20041 43082 8 +1267 30451 16 +42365 14450 11 +16473 53451 6 +2459 59940 6 +12871 15363 4 +28566 6871 9 +2387 43857 12 +57199 37672 11 +10419 30724 8 +28668 49079 8 +49925 17889 17 +8290 34591 15 +26851 48586 12 +9029 1927 4 +50102 11164 16 +30726 43930 10 +37195 43306 14 +45619 13327 11 +13049 11228 10 +50746 17687 16 +15406 18722 17 +28761 12470 14 +8887 32321 8 +47371 23051 7 +53475 56869 6 +32082 34301 15 +13796 39758 5 +19497 42940 10 +56386 20350 16 +21930 35088 11 +5829 4508 12 +30009 22654 6 +30300 179 3 +4 17225 10 +90 4303 13 +54758 29915 7 +17939 11629 13 +57232 46765 12 +27118 35435 9 +37024 23076 4 +59253 5172 11 +10479 7555 2 +29403 43655 7 +20354 22370 8 +41121 41015 5 +44831 12652 4 +19976 43753 8 +23495 23640 5 +43427 35283 16 +41848 29811 7 +45249 6504 13 +44825 19133 9 +40201 15515 9 +5038 43505 10 +52489 24783 4 +9315 1947 3 +50294 55462 14 +56746 25339 13 +31973 20060 2 +41096 44964 9 +15204 40210 8 +52903 32235 18 +26025 10178 16 +18819 3468 8 +56518 9815 8 +24951 26330 7 +37287 55564 4 +48199 13989 6 +48537 37951 10 +36371 50685 1 +43027 36395 10 +20957 53102 13 +27849 18969 15 +53934 44391 8 +23302 34627 3 +43065 57848 8 +51720 8006 12 +3168 2535 11 +2090 15861 15 +47124 57002 4 +49641 46516 1 +22216 10592 17 +49275 52123 15 +8051 15600 1 +45988 23702 16 +16048 55801 12 +24273 16636 5 +23321 2179 9 +6948 22959 11 +48713 40244 15 +50248 9375 13 +19182 25871 15 +53464 1447 10 +59716 7985 4 +3768 17171 6 +2058 43287 3 +36408 9582 8 +30376 31439 9 +41109 53132 6 +55317 36446 7 +14120 58778 16 +19583 25845 4 +23393 50187 9 +27396 29947 10 +14783 23915 12 +49912 29116 11 +15931 7773 13 +30019 21581 12 +37200 34280 10 +15842 9000 9 +14307 32635 8 +35865 51192 10 +45751 40325 13 +34413 59659 6 +13949 6420 12 +25761 9491 12 +40246 24331 2 +10863 20136 7 +6338 3615 12 +55157 57190 3 +4780 5612 4 +6790 13625 10 +41887 23012 6 +13260 12493 9 +22133 14471 7 +7920 31169 3 +4639 49524 7 +35158 3128 6 +26328 49823 9 +50724 50116 17 +25792 24211 16 +34942 8821 12 +47854 47948 13 +35382 46723 7 +33029 19761 9 +11145 31182 12 +3827 48695 9 +3681 23694 11 +184 59529 4 +35216 38798 7 +28911 14554 18 +2184 32562 3 +24830 17882 14 +55242 1868 0 +24774 49841 9 +58481 44984 7 +39442 31844 15 +32628 45506 10 +33817 26015 7 +5441 8780 15 +44805 45749 11 +13797 53600 16 +6739 392 6 +2823 57024 10 +5826 29428 7 +26471 49671 13 +3161 47503 7 +53341 26367 5 +52223 6228 1 +21444 54799 8 +55465 21541 13 +21863 16460 8 +59483 14962 8 +43892 46709 9 +3217 28998 6 +42656 49227 17 +51070 16903 7 +21573 57540 7 +13385 5730 4 +34032 42100 7 +29581 39146 9 +57530 31902 11 +47268 37135 4 +43080 32916 12 +23670 27728 11 +41539 12610 9 +34608 1298 14 +48530 26963 13 +19130 19384 11 +16845 47593 11 +44372 21221 3 +54233 17318 8 +33891 50967 18 +27635 39383 13 +53107 22067 5 +30418 55076 13 +8406 46468 4 +10330 47301 8 +15686 47659 13 +16121 30276 5 +38318 26954 15 +8513 15366 9 +17753 41117 3 +28118 7749 11 +9911 445 15 +42043 45914 9 +49615 14791 15 +10257 7715 7 +33684 7016 10 +56455 31965 10 +17947 47088 5 +47608 51802 11 +53194 46506 4 +13250 58753 8 +46717 42956 11 +49922 25004 7 +46124 58327 7 +48635 12849 11 +14297 46522 8 +26247 32991 5 +56286 59163 16 +30049 18569 12 +11409 26368 5 +34065 36458 8 +27970 30521 15 +13666 25211 16 +1794 39233 9 +19612 25817 7 +55651 9668 2 +51163 58380 12 +58062 37530 4 +17952 14550 10 +18378 23092 16 +51476 1362 14 +14161 20587 3 +49030 46947 9 +13405 53860 15 +30805 52666 16 +21875 42671 6 +2358 36027 15 +30411 483 10 +37688 55717 12 +31409 55074 7 +11790 6190 7 +58969 34905 14 +40616 17385 11 +17012 8860 7 +52714 44777 9 +34874 4994 1 +28215 45166 14 +47315 20726 12 +5986 8366 15 +36342 26339 4 +23832 38114 11 +57444 42120 12 +17359 10070 16 +14284 35803 10 +44836 46564 11 +24443 39819 8 +57722 40429 5 +13430 7089 5 +16051 25529 4 +5870 7158 3 +54335 28493 13 +2714 31454 4 +42724 52908 14 +30191 6987 2 +59851 34061 15 +53972 89 6 +27370 42473 17 +7896 14854 6 +235 12407 4 +36587 17566 11 +39351 21165 18 +31426 35811 10 +26429 44694 5 +3459 45591 9 +38220 34046 13 +51243 47636 14 +14575 59144 5 +14151 4622 14 +16704 26378 12 +15797 53611 7 +35 38922 9 +51912 6865 10 +46594 185 13 +49196 27698 13 +45287 46440 8 +12210 38098 12 +2125 41571 9 +47235 37059 2 +53977 34584 12 +10597 21957 18 +29273 1072 12 +6263 24419 9 +35834 47400 8 +51182 46611 10 +42110 17321 9 +27182 188 9 +54720 34581 6 +39780 3352 9 +40396 10068 1 +42195 52095 7 +20411 58843 6 +40712 30242 15 +12544 58237 6 +10198 23550 14 +48161 17160 13 +13567 52976 9 +17563 28268 8 +59600 14491 5 +27903 51568 12 +41678 58002 4 +38453 44674 13 +34193 22851 14 +14822 40278 15 +8087 12371 2 +33070 40226 8 +20264 52528 16 +40675 31431 10 +58289 34447 10 +43727 22976 8 +51660 31360 4 +49407 16482 10 +22623 10935 15 +30868 5428 5 +8401 14929 8 +16234 4350 10 +18597 54102 7 +32738 56779 8 +56423 55179 4 +57294 46374 13 +1163 24444 4 +46373 2964 12 +46196 33981 10 +38613 8035 17 +50600 43484 2 +10133 56989 4 +47246 35381 5 +41212 42226 11 +41519 22871 13 +45958 28726 10 +21467 53758 11 +48192 17588 11 +27128 39094 17 +52040 54144 14 +24486 2661 9 +52753 28195 12 +2999 41318 5 +10977 53239 11 +23365 2564 12 +40229 10939 11 +29944 8 7 +32583 33257 4 +27260 37054 12 +44689 6825 7 +44860 55788 8 +7884 30684 6 +11368 47469 12 +1896 45202 10 +23187 5037 10 +684 43359 8 +11014 21117 6 +54889 32993 12 +12223 50752 11 +10279 52487 5 +41487 28584 5 +8594 33695 9 +48523 19985 0 +7692 19019 8 +38956 54463 16 +52610 3484 9 +26547 40062 9 +59989 52258 17 +29867 17286 8 +21774 48491 15 +51098 46212 6 +39900 36267 9 +41262 25857 7 +34963 55467 13 +42266 49690 17 +44852 3305 7 +41259 41016 5 +46557 44293 5 +8173 43386 11 +11734 1923 10 +902 17798 16 +1788 31005 13 +16605 34299 9 +15196 1928 10 +29304 28893 4 +20594 59455 18 +3193 5993 15 +22644 27986 8 +44557 15170 12 +38509 59965 3 +39985 4271 8 +23513 30028 9 +46012 30022 10 +25996 18343 6 +44969 59482 8 +3098 49125 13 +3686 56930 13 +18725 35178 4 +43096 915 13 +18533 2280 10 +52375 56877 12 +54420 34852 7 +8529 23149 8 +44707 11967 6 +666 5184 3 +24420 56657 12 +3876 53414 5 +49736 19944 12 +10037 17537 15 +18738 38231 9 +28069 48900 2 +17971 23844 6 +13256 22006 11 +22081 10231 16 +55504 32225 10 +13331 43102 5 +33832 1535 3 +56277 56272 12 +12391 46330 15 +18731 51138 16 +36323 47968 13 +38837 35490 5 +56473 42763 9 +29438 25626 0 +16926 8919 6 +43147 14502 18 +18878 48793 1 +52839 12586 10 +44779 23860 6 +3092 37382 4 +4844 48708 14 +59036 23703 2 +59881 53631 8 +7273 54083 6 +50444 35587 1 +50220 45975 6 +9672 2635 8 +47843 13450 4 +11648 54760 9 +1765 3664 12 +46199 11552 3 +23146 22395 7 +44477 16357 8 +14568 55807 17 +38053 21851 11 +29156 9858 10 +3814 59804 15 +26438 50350 1 +16449 302 8 +42677 12512 4 +22700 27712 1 +3142 9228 7 +57149 59358 8 +59398 24778 8 +52596 39829 9 +42157 27898 2 +40361 1534 12 +32775 7323 8 +24198 26093 4 +30380 13859 10 +8803 50417 10 +55978 52588 16 +46978 27769 8 +47519 42601 4 +10542 18420 4 +47494 2048 12 +19178 54323 18 +52985 25039 4 +8309 54883 5 +17777 36420 12 +54558 7529 16 +20025 52140 12 +51072 28306 13 +59240 50771 5 +36817 39403 5 +46097 45458 12 +839 30217 12 +37649 57097 10 +56885 18383 14 +1629 15338 8 +53163 21552 5 +7385 2908 8 +20800 50932 9 +55322 56890 10 +42128 49160 6 +1783 11622 9 +38818 55251 5 +32093 25723 13 +35791 16411 9 +46781 15055 6 +7173 2074 10 +53056 15533 5 +33466 49622 7 +11383 56889 16 +57384 8606 3 +10366 4747 9 +26646 9623 5 +1484 18925 13 +16541 15655 14 +7366 9829 7 +31504 47760 6 +53946 12230 6 +23372 25099 9 +36648 9205 13 +29699 24092 11 +4811 39050 9 +37071 47224 7 +49018 12359 9 +42982 56003 11 +5155 54981 6 +30983 31465 6 +21614 22462 1 +44100 25607 12 +52952 32942 13 +31740 35567 7 +31392 53918 6 +18708 29585 4 +13561 2805 15 +45428 30491 16 +8347 52690 9 +52672 5439 10 +29886 12884 17 +26365 32868 10 +10336 18454 11 +48378 50021 9 +17365 6212 2 +8102 57571 3 +1321 20517 3 +51272 28748 13 +42418 42866 9 +51969 28989 16 +5244 22121 6 +6543 15735 7 +37943 32570 8 +36355 44462 5 +17698 36059 14 +26322 54957 14 +31900 41435 10 +44829 3425 9 +7593 15529 12 +39997 4182 11 +57307 4396 9 +34175 27917 6 +27327 38304 7 +25321 10414 17 +3330 47992 9 +32100 5617 14 +9716 44545 10 +5162 9631 11 +2569 23684 11 +3508 12281 11 +4717 23934 4 +26931 2784 7 +26523 55224 17 +41296 12538 4 +22934 21458 11 +46648 10302 14 +15872 48398 9 +22499 17967 11 +48820 8441 10 +7839 46530 6 +12808 40460 6 +46722 49265 2 +42921 54841 12 +43297 20369 1 +32291 26809 10 +17775 31031 6 +28823 58835 10 +21452 8195 9 +12326 51567 5 +37346 4670 11 +44931 18863 10 +38433 30713 12 +2934 26376 10 +34309 24928 12 +36041 49059 7 +45870 34730 5 +30920 19450 3 +30953 25033 7 +7675 21495 12 +2249 34710 9 +20984 41974 12 +31339 50358 15 +25920 59879 6 +7788 30947 10 +7060 49312 4 +44580 19069 5 +35402 55640 7 +10509 55969 7 +15417 708 14 +502 15913 14 +37001 13118 15 +38110 29505 5 +47428 42584 6 +29325 38337 8 +2410 17393 9 +42719 12650 6 +37167 56110 4 +37515 42537 3 +46324 11372 11 +56388 55307 6 +31698 5631 11 +2765 23017 4 +59199 39457 1 +39681 16480 14 +54835 22822 5 +44220 16017 6 +31980 49737 1 +38366 36735 11 +4923 18284 6 +40738 9607 10 +38878 6367 8 +24406 13569 3 +29399 32311 15 +38333 16053 18 +416 15695 1 +25325 11970 9 +19631 38782 9 +19149 27679 11 +40428 39792 6 +50660 45850 11 +42390 19501 8 +22293 55892 3 +18972 51549 6 +59286 12539 9 +36112 51035 3 +42357 33289 12 +1271 35196 11 +8561 27605 15 +200 36219 9 +22847 25400 9 +35371 56683 8 +38978 43849 5 +27766 57102 13 +27031 4496 3 +28307 6478 10 +14885 40793 8 +54382 15051 15 +28035 29097 4 +31103 9082 15 +56912 15895 8 +23089 40700 9 +14019 58460 5 +54538 54014 12 +52257 2990 9 +51390 33807 3 +59219 32206 12 +25530 33002 5 +33175 16373 9 +1740 41894 10 +10149 10679 10 +52209 8480 5 +18960 8042 9 +57772 12065 15 +37005 9817 17 +52032 17884 3 +19312 43399 13 +54953 40150 3 +32035 59958 3 +58428 33368 6 +46183 40553 8 +11236 13046 7 +45524 48577 16 +26637 28031 8 +41466 15505 2 +53159 23035 15 +26487 28176 10 +30555 21832 6 +12657 14655 4 +32848 31749 17 +9837 55675 5 +348 11842 13 +56133 19090 10 +41367 22058 10 +9047 49431 3 +51633 37458 10 +32331 58816 16 +1549 55398 2 +8328 13803 14 +38620 37715 15 +7206 3647 8 +4652 35966 12 +19058 13277 3 +16323 25805 1 +10108 42338 11 +31073 24048 15 +10677 42569 8 +38384 50643 1 +19233 59955 7 +58970 7179 11 +11857 32393 6 +17580 33183 4 +49682 34310 7 +45560 22298 10 +18417 29591 14 +29181 31553 10 +59213 26846 16 +51266 10463 14 +20721 34580 2 +8699 41215 12 +9001 13247 16 +48546 26995 9 +20072 51546 3 +20544 55987 5 +44119 45153 10 +28227 48833 8 +43501 49155 12 +36227 13242 6 +14288 29429 11 +23672 12284 8 +39280 7994 12 +10548 15330 7 +57834 16032 7 +56932 11523 12 +6944 24228 9 +10737 35093 7 +42478 23318 9 +12922 12859 6 +35790 45308 16 +26028 11686 15 +51664 47777 7 +25179 12886 16 +34290 32032 8 +51421 28592 12 +18390 54218 10 +34345 35633 7 +57801 36794 4 +17931 6373 10 +25334 52174 5 +24145 19062 9 +52121 41291 5 +56065 16470 11 +50516 37711 6 +59725 40373 4 +20953 14529 3 +23338 34302 3 +57723 55588 7 +30893 33313 18 +46997 16407 5 +19219 38998 8 +26671 22943 6 +20329 50425 9 +34067 4545 11 +11858 29337 9 +22910 40030 4 +56643 14486 7 +14383 21901 9 +15449 43906 11 +58174 23659 12 +22911 26198 12 +5901 35758 11 +16151 9915 3 +22039 29172 16 +49342 39231 3 +55595 57626 11 +13795 26719 16 +39185 55049 3 +5027 15029 14 +3801 33904 5 +35227 2323 8 +22383 22397 8 +58665 10161 3 +36868 39590 6 +34422 15144 13 +40715 57941 11 +66 40862 12 +46876 50534 8 +32380 3041 10 +48704 55333 17 +32284 55683 11 +44333 12653 0 +29136 34318 9 +45325 36450 6 +8703 51120 1 +24996 7141 9 +47727 24743 14 +52458 36655 14 +40932 1212 7 +54731 19368 10 +20571 44968 17 +10667 33009 4 +1723 48003 8 +46903 33095 7 +54051 56306 10 +17513 41994 10 +40391 40482 10 +56813 10650 11 +54285 7033 10 +56052 20612 9 +58293 4313 17 +48981 13902 7 +34644 28260 12 +1554 21770 8 +1965 12465 9 +58594 44869 5 +24131 626 16 +39114 3806 5 +22320 22369 10 +22614 3880 6 +41319 11969 4 +47176 3913 17 +25162 52272 7 +11299 20336 10 +32290 58646 10 +15888 35586 2 +31084 45186 2 +2538 1117 6 +2026 5379 11 +18707 15093 3 +20786 18724 6 +3824 10255 8 +24285 2461 14 +35975 6128 9 +39121 28834 7 +43728 51598 0 +16180 43896 13 +7696 44175 8 +25338 35914 9 +6933 18763 4 +33575 7938 6 +34832 52775 11 +35434 41596 4 +58435 39862 8 +46984 48447 5 +35141 42800 11 +44314 7047 5 +34157 51529 3 +27938 51158 15 +9510 46442 10 +24083 27251 9 +39290 16289 5 +37004 54570 11 +51386 2408 9 +29542 26098 9 +12836 2114 6 +44788 53500 2 +35443 45195 5 +5429 45407 4 +7685 58102 0 +32490 19169 6 +4286 49365 8 +19587 229 2 +54693 6004 6 +16759 55044 3 +10369 14734 4 +14727 42702 8 +53426 4241 6 +40238 486 7 +17488 37812 3 +52929 28703 13 +47238 52696 2 +24306 38408 6 +59835 55823 14 +41689 53290 3 +14801 3700 12 +34736 54613 5 +43348 26128 10 +26914 53893 1 +31057 53282 12 +26419 34156 4 +31147 49364 13 +41743 25829 3 +51583 20002 14 +49893 42544 10 +56703 57850 15 +1630 43879 5 +1245 59639 5 +54609 12725 1 +46982 46244 13 +16803 7562 6 +17706 51755 7 +49417 17073 9 +56945 41815 6 +26279 16186 8 +18773 5758 9 +55346 8558 16 +48842 58871 13 +9825 17964 10 +39178 37481 10 +37806 38480 3 +48802 59302 15 +5240 14196 6 +12224 33696 13 +42154 9941 15 +42322 22235 10 +50234 3743 7 +13381 49486 4 +5557 541 12 +20784 1242 6 +36186 27753 11 +17219 41080 7 +3969 6279 3 +57200 23126 14 +4714 34516 9 +26744 19341 10 +2867 41413 13 +5403 11705 8 +3477 912 9 +20661 47978 7 +59468 34179 8 +53480 37809 4 +18822 27602 10 +17251 637 7 +28194 26859 9 +4059 1138 15 +9812 51975 4 +17143 39289 14 +19266 35717 3 +27434 4835 4 +56367 57093 9 +8693 50603 6 +37271 10175 7 +3892 9351 15 +48774 51964 13 +12874 51722 7 +37991 5236 5 +43171 1919 10 +22643 33399 6 +11256 28406 7 +36396 29526 8 +40764 36804 14 +23404 54917 8 +15679 41847 7 +21554 55086 8 +45633 58319 10 +44230 16497 11 +22203 6182 5 +13286 3939 6 +17577 48658 12 +49913 46186 9 +36000 38563 11 +14893 42858 12 +6996 53101 4 +55277 18300 2 +12328 47832 9 +27590 47439 8 +43281 9878 13 +1786 38807 9 +44599 13953 2 +1670 30189 6 +31768 53498 6 +12347 28752 8 +48377 28677 16 +8742 39846 3 +46639 8521 8 +59898 1022 7 +8573 7743 4 +5164 16018 7 +59605 23902 7 +19105 33528 11 +23436 35439 14 +17233 38474 14 +44534 7369 11 +8484 2690 14 +36762 55119 14 +24323 5874 14 +28805 23001 8 +55481 8853 2 +20144 55889 13 +36331 34085 3 +23097 22737 7 +6171 41778 11 +24360 39548 6 +29697 42716 2 +31237 19004 1 +44839 48649 14 +20158 49545 15 +41726 47218 14 +49471 25144 5 +17796 19555 7 +11703 29472 16 +47406 12089 3 +14235 37418 11 +34707 33905 13 +52116 45227 12 +42391 45368 9 +31149 45328 8 +26812 50819 5 +58431 12896 10 +54642 23939 3 +33392 42963 9 +5581 40040 4 +52279 54551 6 +31905 51596 8 +19139 56223 3 +32769 23846 6 +56790 1696 7 +39244 12480 13 +43460 22925 12 +11000 48972 10 +17598 53843 12 +29536 8619 17 +59041 15451 2 +17118 20933 10 +20566 54751 14 +7489 9978 4 +2056 4489 7 +18602 31371 15 +56061 51342 10 +6821 30751 9 +21645 29797 16 +12137 47820 5 +22565 11557 14 +9058 589 15 +43913 57655 8 +6894 20774 11 +11112 22323 7 +55385 47141 5 +10758 33810 12 +29956 20810 10 +38209 27978 5 +9728 4204 11 +9940 14351 8 +52008 45411 14 +20740 40399 8 +11733 58601 9 +22590 56636 12 +9982 35282 8 +22656 30913 8 +41879 11035 9 +32358 10837 9 +16819 37832 7 +13153 13613 11 +56785 49026 8 +49670 9827 12 +51795 19439 5 +43214 7602 10 +55575 10744 11 +55578 55906 10 +36418 24369 12 +21594 23103 5 +13475 57075 7 +34667 10148 8 +57355 40814 16 +40517 3981 5 +21551 32836 11 +16191 11549 8 +12203 45068 16 +19177 51414 16 +44587 6973 4 +40666 19603 17 +24707 7190 9 +22757 29660 9 +46383 7564 12 +45244 57605 11 +7606 51225 12 +2274 29010 18 +54284 1442 5 +30970 46030 11 +35955 48957 10 +26203 10847 11 +24573 48108 10 +4194 42846 6 +21688 15237 8 +45912 2126 10 +2030 48735 11 +37564 41377 10 +38598 19744 7 +51639 36462 7 +41449 8248 8 +43847 6971 6 +41565 3632 3 +39324 45605 13 +36124 21390 4 +14756 33806 10 +48841 25088 4 +31040 58759 9 +45492 14103 11 +49830 37449 17 +23967 49438 16 +18842 45237 10 +5692 12220 6 +35987 25859 16 +12264 12972 4 +17204 44468 9 +59178 5497 6 +23886 27908 10 +33231 19085 10 +47991 22294 1 +41175 26052 14 +15604 42217 4 +9690 9036 7 +6300 6167 12 +53508 37506 8 +4708 24025 8 +37633 33864 4 +41178 14231 16 +9699 4121 3 +38060 42836 10 +10995 32670 11 +54979 11041 1 +25085 13976 8 +21393 37907 8 +40448 19278 10 +54363 19034 10 +46726 22981 0 +16751 5921 9 +2339 46161 12 +24559 31583 9 +8078 58531 6 +16021 342 5 +10794 25071 12 +34333 56958 7 +30289 47450 12 +52507 55544 12 +22052 8377 14 +37397 36231 10 +33697 32598 7 +27127 4138 8 +10327 38717 6 +26779 55432 5 +43642 54845 5 +44244 39362 10 +40276 9638 4 +9024 11312 7 +28996 2028 9 +57816 9206 7 +10830 25839 3 +4424 36285 11 +46759 24809 5 +53978 3204 10 +59146 42789 8 +49410 26237 5 +18790 48653 5 +24586 24111 11 +24480 27969 1 +29627 3475 12 +686 780 8 +45827 56763 8 +47817 17716 6 +20750 47342 11 +53930 14708 7 +32747 24745 12 +44844 19691 1 +3500 46755 11 +805 46249 12 +12201 28051 15 +2956 40971 10 +36166 25606 3 +48353 27250 6 +13469 4355 9 +5912 25708 10 +52700 17869 8 +21232 37754 13 +785 9238 15 +25293 15802 6 +33952 57719 10 +45181 14266 11 +12817 35934 4 +30698 13532 14 +49547 7218 17 +758 36643 15 +17694 28092 13 +11137 27959 11 +54228 57898 11 +23988 59414 1 +53338 52446 10 +28686 20770 4 +12248 58340 13 +53907 30254 8 +21627 3043 10 +7984 16977 8 +26715 1283 8 +58442 37350 12 +38908 46433 8 +25299 37528 11 +16468 56895 8 +16150 41715 8 +27694 23926 4 +6219 29559 6 +41896 47390 9 +40593 28474 12 +49076 50759 6 +4935 46387 10 +42436 59802 12 +52050 24174 12 +19294 48525 6 +27446 56775 0 +49390 50966 5 +54738 32594 18 +32767 43416 18 +45764 26785 9 +51903 35649 6 +49138 16601 8 +51377 32741 2 +1904 42545 3 +20845 34651 10 +50142 53963 7 +41083 23654 13 +27307 29372 4 +56671 31660 8 +32695 16961 2 +58135 9213 10 +34365 59707 8 +55719 36104 10 +49904 10477 11 +25681 8926 7 +8201 18196 3 +52604 40505 11 +45649 28267 13 +38580 8490 5 +39617 18436 7 +53126 10371 13 +26132 16537 11 +8817 36973 15 +17043 40942 9 +41503 46055 16 +56419 7011 4 +50731 30313 8 +28648 18104 16 +2372 14595 7 +42187 31064 11 +39445 6372 4 +29191 5824 8 +53982 31430 0 +10154 7058 10 +50587 20234 8 +39944 4534 1 +18568 38911 4 +40746 59167 2 +16973 47003 16 +50608 14335 5 +31406 2679 9 +44351 34680 11 +25525 57494 7 +505 50210 10 +42947 35848 9 +45008 11396 8 +40652 32771 17 +21185 16217 6 +34127 18299 6 +58318 23761 9 +24112 7828 9 +20868 38874 3 +44879 31785 6 +40875 42124 17 +48305 27304 16 +49546 27598 6 +12970 27461 8 +4389 26989 3 +5277 17406 7 +39276 47357 12 +5177 25998 12 +13063 22810 17 +47524 41271 14 +20086 37912 12 +39855 3844 6 +31746 3871 7 +14958 35366 4 +30419 18616 16 +42427 6374 12 +31313 42355 11 +48142 10066 8 +28149 42497 18 +4618 35073 10 +27726 24094 10 +39007 6578 7 +32693 54565 7 +8449 44634 5 +2373 22921 7 +51193 5523 16 +47834 56823 6 +20327 25719 2 +39611 12471 15 +2530 51484 9 +48824 57056 11 +29690 52029 10 +31423 50231 15 +58865 42882 10 +14438 30885 11 +37072 5089 12 +58777 8047 16 +9174 6655 11 +3364 18324 6 +50837 3776 3 +49491 53062 11 +14955 14919 5 +42063 56284 8 +30186 42912 10 +52074 25532 13 +34153 6256 11 +39771 43113 10 +23056 35288 3 +54942 18997 3 +42794 55300 5 +24552 28983 5 +44513 42216 4 +7145 49747 13 +16805 9249 3 +33064 11517 11 +30878 9998 15 +4516 46643 11 +43049 56230 15 +31568 37761 7 +40513 52925 9 +46257 40605 5 +55085 14463 5 +24359 48939 13 +41746 17374 8 +25440 17827 14 +14719 1546 11 +11346 54616 5 +2357 26819 8 +54097 24923 8 +21438 42263 6 +47617 44877 13 +56227 47543 9 +44414 52022 5 +52090 53384 12 +42121 12548 10 +24088 8783 14 +5289 20667 11 +35999 19549 3 +45791 18457 8 +57189 1679 10 +48441 15639 4 +57314 17010 9 +5783 14504 17 +55731 9666 6 +31791 30961 8 +55923 55677 6 +38822 24049 11 +59857 46547 10 +44353 41480 16 +38369 29145 17 +48311 44772 10 +38750 16483 2 +23050 51668 14 +46942 31706 16 +55308 27383 11 +3795 25500 14 +36429 50372 11 +76 46995 6 +31559 21352 10 +55187 34145 7 +27076 42150 11 +6125 37288 9 +52283 28676 17 +1201 9298 4 +25044 13784 14 +18823 53063 9 +41323 9960 7 +59454 4583 4 +6315 49203 12 +30804 16147 18 +15525 18409 5 +41798 33897 7 +36594 32783 7 +12433 23793 6 +36922 26110 18 +17130 51715 9 +29951 4718 7 +50957 46119 11 +21976 26862 10 +35441 53169 15 +20397 32390 11 +12990 48947 3 +29132 4548 15 +1926 1597 1 +9865 18470 5 +4979 16671 6 +33126 24655 16 +20023 6889 10 +51649 47655 3 +53783 32986 8 +38705 24827 13 +10002 48489 12 +9318 8653 2 +36881 41391 9 +59393 4474 10 +54411 11979 3 +31542 15617 11 +46346 40725 9 +39463 33973 5 +10849 37010 4 +27592 10574 9 +57480 26820 5 +46856 50133 11 +13339 31017 5 +13023 29360 10 +43876 25148 10 +35724 54557 10 +33926 42401 5 +46152 11119 12 +19371 39933 11 +26702 21810 8 +21813 25682 6 +12984 33443 17 +54422 18590 4 +23970 53241 4 +38868 3045 2 +58924 20524 11 +40634 57437 10 +55186 5493 5 +41049 27087 14 +19812 42817 7 +55026 6615 8 +14930 25405 13 +35253 20548 8 +59103 40716 1 +10528 5734 6 +10966 58126 5 +18536 33766 14 +50557 11272 10 +34105 5655 4 +8950 15174 7 +22361 45130 6 +32808 10766 3 +48587 21938 6 +4494 34820 12 +43440 14178 12 +6189 26163 9 +47467 16728 8 +36633 45485 5 +40721 1238 5 +21775 7191 7 +57489 47285 9 +25212 51616 12 +40642 50456 11 +40178 1838 2 +52410 13147 8 +6270 40066 6 +38217 18856 13 +52564 25928 4 +29034 36752 8 +34501 43019 14 +43543 45887 15 +2559 37456 17 +40217 42096 8 +34713 46829 8 +59813 57380 15 +45084 53839 16 +52360 35700 1 +18443 7800 3 +52149 28162 13 +40967 32850 5 +20408 1588 11 +50205 44445 0 +28339 14406 14 +37598 30197 6 +13674 32713 10 +7999 19600 12 +49731 253 3 +31970 29919 9 +15692 16402 0 +51924 39990 10 +36042 28179 8 +43610 17896 4 +48839 5072 6 +23243 42630 9 +12015 59162 9 +21824 43146 16 +56693 35808 12 +16451 12236 10 +55070 29211 2 +879 53017 7 +25193 26073 6 +59065 47885 15 +45695 51472 9 +22122 59321 7 +51404 31975 3 +41628 6823 5 +35705 12616 12 +58885 29687 13 +11898 18650 7 +1590 20606 3 +32658 2725 7 +28059 39387 9 +54662 51181 13 +32005 49180 8 +32153 16972 9 +24662 10196 11 +27889 10121 8 +30384 35330 9 +20856 1047 9 +44027 22047 6 +57698 26370 8 +40562 17062 5 +49328 530 8 +42752 17956 0 +19076 2209 9 +12050 36688 16 +5474 52179 8 +40260 15725 9 +14709 81 5 +23683 34184 11 +39173 8475 4 +14308 24597 9 +47167 24258 4 +58012 26003 8 +28663 58469 17 +21646 2380 16 +31191 39428 17 +23048 12277 17 +23377 32948 13 +56290 32261 16 +57920 17339 5 +33164 53645 16 +29080 34304 9 +40416 39159 3 +26469 11760 7 +35822 37385 8 +19891 56375 6 +14724 59345 7 +8584 1009 16 +20108 11115 6 +19070 9184 4 +32633 7972 12 +6486 9558 10 +54413 5093 12 +26090 43497 8 +39600 56866 8 +22258 14035 14 +45517 33717 10 +15773 4302 8 +58803 39462 6 +46533 53269 6 +36679 25999 15 +52143 544 6 +53877 29295 12 +6253 14239 13 +33092 34899 8 +20978 52572 10 +751 25659 10 +33046 15294 9 +57242 41287 8 +2155 26906 17 +2651 38314 10 +24964 11593 16 +22253 54164 5 +56342 47765 14 +4553 31098 4 +11657 30614 1 +27916 3028 16 +51569 29311 2 +6837 18612 9 +54439 25126 14 +57324 15296 10 +55863 24354 10 +49849 32876 13 +22856 38207 3 +53151 27432 11 +17320 38460 9 +47185 330 12 +22135 12446 4 +15234 26867 7 +46955 38020 13 +4723 17613 9 +55210 32604 14 +13810 15608 8 +22555 54013 4 +44266 30127 0 +459 52871 6 +26054 16919 16 +6819 56087 9 +45594 1467 12 +40688 6598 9 +51096 49297 9 +9520 48846 7 +46716 50292 5 +18233 23252 9 +17879 35132 8 +59023 689 0 +3926 12992 1 +35670 47690 2 +31181 15369 10 +43566 30056 4 +2869 11083 8 +46768 43749 13 +2263 25939 5 +54947 40333 3 +31215 13350 13 +17226 17718 16 +44759 40461 10 +24718 23565 9 +17033 953 10 +29557 6329 7 +41396 49621 8 +42295 11876 11 +37953 30809 1 +41194 48016 12 +30203 18728 17 +54930 14796 12 +9907 38851 6 +23563 47802 6 +39259 51316 18 +31624 49794 14 +29412 11136 12 +3346 53537 9 +58309 32167 2 +59786 33442 0 +31071 55974 9 +15367 17013 17 +4125 18902 12 +9207 48905 10 +6490 25962 10 +17361 22205 13 +21621 7235 12 +46243 10021 17 +42588 23115 4 +30551 5603 14 +12360 54179 4 +49976 13421 9 +32607 15722 9 +19816 30297 3 +23802 4478 14 +57447 5668 13 +7890 4064 10 +11926 4710 10 +20908 22088 13 +52803 50712 11 +45261 50580 11 +56401 11669 18 +42426 14290 13 +59437 40335 6 +15247 11840 17 +27459 55124 12 +33890 4297 12 +55579 38175 7 +28920 10126 10 +50581 41270 11 +22678 3462 12 +1727 24386 5 +34434 6639 9 +14100 5335 13 +197 1246 10 +11877 29481 9 +19735 17440 8 +51258 864 14 +50626 36822 9 +45805 35864 11 +31123 39540 11 +45643 31413 11 +4228 40698 12 +35581 51428 7 +42510 34873 4 +26661 17763 10 +18542 42750 7 +18346 38367 13 +25603 44817 11 +10362 36960 10 +29518 19652 2 +18168 24047 10 +33364 56739 4 +38657 18971 9 +56672 57998 9 +82 20107 10 +6807 1643 15 +15851 17340 8 +26625 42589 8 +25404 18556 16 +18042 46952 0 +27179 16620 8 +22949 28521 1 +19052 22306 8 +37664 58794 11 +32526 56442 9 +36718 28383 11 +57930 46727 4 +43360 48411 3 +46596 59109 9 +15890 16484 0 +50048 53702 15 +47284 15833 12 +53073 20895 0 +17686 37375 11 +12144 43473 4 +49045 5533 11 +11118 8635 8 +37764 31670 15 +41105 58722 6 +10425 35111 2 +59834 34071 14 +58138 27801 13 +1279 26579 15 +32434 19242 11 +53641 43778 5 +52206 27040 10 +41250 59436 10 +14469 48559 12 +46093 17517 10 +25424 3911 10 +39745 39529 9 +56538 30341 7 +23625 45121 6 +29369 13387 14 +42074 55750 7 +30231 40083 12 +58957 36066 13 +26696 58044 3 +48849 7053 12 +33868 41444 5 +8058 40365 4 +56297 40070 10 +372 31703 7 +22063 33736 7 +17381 58952 0 +46703 12343 4 +38390 9284 8 +29695 53234 7 +28670 33173 7 +33215 51859 16 +19353 2621 5 +4233 59204 10 +20699 32571 8 +40861 38392 5 +19658 4301 13 +23729 13018 16 +21021 24488 13 +25135 31296 7 +10537 33140 2 +11592 22325 7 +39256 1613 6 +26981 2670 5 +54904 21532 10 +31512 39991 10 +15436 40342 14 +54515 41166 4 +16546 58049 5 +5505 52382 1 +21184 24987 8 +23214 58754 1 +48292 19451 8 +58111 25879 14 +54838 12584 2 +211 25351 7 +44091 57600 11 +57086 42890 11 +48876 1462 13 +17684 10906 11 +48923 23983 13 +28287 25982 13 +53541 55993 2 +41993 23232 12 +16061 15471 1 +12943 8879 11 +15918 15446 14 +46438 47006 6 +35917 20181 9 +19718 8211 11 +5153 33667 4 +42799 45347 4 +27149 54803 7 +23356 28943 12 +44160 2243 15 +27171 11391 2 +57991 57490 13 +16421 40299 13 +950 42066 13 +53870 51475 7 +8418 46565 8 +22815 33298 2 +34405 23686 9 +24127 41841 8 +7064 44086 10 +32208 43536 7 +51510 26910 9 +55339 10388 11 +50618 16802 10 +59118 52783 15 +37704 313 17 +27767 43953 13 +40496 17489 6 +51731 37944 8 +46527 7998 13 +21883 11667 7 +6583 41228 8 +36795 29896 7 +21418 41585 7 +57558 14063 2 +1985 36377 9 +15623 59623 12 +34691 17228 15 +44909 37283 15 +13678 30637 13 +21959 33919 8 +2883 36212 7 +40632 55020 10 +16867 17104 6 +45774 1762 7 +46640 38294 8 +6956 39626 13 +11866 45540 6 +15939 40660 11 +45659 21180 7 +47197 17809 7 +22641 19873 8 +8610 42372 6 +46110 48415 11 +39946 21179 8 +55950 45099 4 +5682 45417 6 +1044 4962 1 +31847 47177 13 +38241 31427 4 +21677 38273 13 +41576 19559 10 +43016 59784 1 +45433 39581 1 +55401 49349 11 +3287 21004 12 +34846 53862 5 +37752 23389 0 +19724 19521 5 +13926 21030 6 +41145 12408 11 +49181 23018 4 +37161 2140 12 +4243 27730 15 +2302 33715 14 +29305 55036 10 +52889 20319 3 +41532 49864 10 +30235 38116 11 +38155 8867 8 +44924 37563 6 +26080 42924 16 +41159 24429 4 +5193 37405 4 +50883 49002 9 +50396 47637 10 +13820 38305 3 +49306 40056 1 +14488 43547 5 +36613 26889 13 +30511 34077 4 +524 48040 1 +41926 42579 5 +15618 9094 8 +49008 9255 4 +59450 1594 3 +679 18647 6 +23259 19508 9 +29354 37455 6 +35851 35826 7 +13525 27494 6 +22684 14954 16 +12099 59273 14 +4592 29174 12 +25378 39049 5 +11291 46490 10 +32956 34404 16 +44325 16914 1 +42059 35310 14 +48006 27279 9 +47209 640 13 +34797 55639 14 +42218 2634 15 +35195 56177 9 +1694 20745 12 +41976 3422 9 +23859 22753 9 +35358 39821 16 +13357 40203 8 +30729 18295 3 +734 51580 10 +42996 32280 8 +15644 11354 9 +12505 46550 8 +41899 20219 10 +55990 43436 6 +25185 2483 2 +46938 42044 8 +35687 417 11 +31022 22199 7 +32662 28395 12 +5258 37556 9 +27298 28354 15 +5058 23225 4 +9364 54677 17 +18580 47918 17 +28712 40370 15 +11909 42533 17 +7090 7072 6 +29671 2478 8 +45949 50725 10 +57431 29773 8 +7799 3334 12 +21484 52653 14 +16950 38512 6 +2253 36968 1 +8295 4035 7 +15866 55486 11 +17256 35186 8 +14274 52633 11 +54225 57023 11 +42631 11012 8 +49956 57215 16 +34359 20161 1 +21735 1773 8 +53821 21939 12 +15753 42379 5 +44996 3082 9 +39584 54178 12 +47239 38665 8 +6683 26387 6 +52182 29590 4 +34974 23929 11 +11443 58755 8 +17961 14476 10 +254 2663 4 +19315 56019 17 +8600 23144 7 +54376 7556 7 +24416 51003 9 +19623 56604 8 +56352 17507 2 +51631 44418 2 +803 46187 10 +4724 41454 7 +44858 52390 13 +23136 50117 8 +45376 56554 11 +13335 17872 1 +30688 2254 12 +55387 49316 11 +12665 23830 6 +4561 34678 9 +26222 3120 4 +33024 2051 1 +43089 6422 7 +17868 217 8 +30696 1568 13 +510 4130 8 +35541 37700 11 +17590 14824 11 +50501 57525 1 +6198 36931 8 +45844 8985 8 +31730 26136 15 +35041 34 5 +30642 36857 9 +56607 16255 15 +31636 34709 7 +53104 49809 6 +45539 39437 1 +8889 31243 8 +4347 46521 6 +8864 37482 3 +15827 54498 17 +33363 47629 5 +5319 36139 7 +30470 16611 17 +21426 7455 9 +55875 6431 10 +50011 28882 4 +12046 17369 7 +45169 10774 8 +40209 24249 8 +9922 58530 7 +6730 47229 1 +20579 16447 8 +47565 53884 4 +11056 10363 7 +52855 29461 8 +20663 46998 14 +46635 32247 12 +57089 23298 9 +1627 50682 2 +54385 35731 14 +12983 13830 3 +43833 51477 10 +6975 45665 7 +49509 32369 12 +40137 24030 12 +52849 4175 8 +11641 4772 5 +48481 255 4 +58812 59462 10 +18766 11646 7 +54050 36798 15 +16436 42027 12 +29831 41241 13 +31694 44017 11 +41158 52865 0 +41595 55981 6 +58194 34699 4 +8929 56629 12 +30647 23317 11 +14248 43774 13 +4539 58661 4 +45642 24635 6 +38282 30785 15 +58263 4519 16 +53262 27418 11 +30399 25216 14 +12987 23818 2 +42108 46836 12 +13843 50374 8 +35118 12714 14 +40096 43370 6 +42607 9583 6 +13966 465 14 +40028 51949 10 +19379 17155 10 +55894 16403 7 +4323 30903 14 +52474 21072 14 +32127 16077 14 +24153 42233 13 +5911 20943 12 +48354 39342 4 +11544 17132 8 +38549 11799 7 +3949 41545 9 +58649 31731 7 +43920 46860 9 +17958 27774 18 +5338 54098 17 +45224 29960 10 +59672 10466 5 +11907 41738 16 +23814 33463 4 +50402 55735 13 +11937 13611 8 +1037 9422 4 +3762 22573 10 +49221 59073 6 +3411 5373 1 +19700 10084 6 +37231 30874 14 +42279 9241 9 +4307 59260 17 +29604 53535 8 +6201 44749 9 +18061 55956 9 +7645 47130 13 +51257 34993 9 +6949 9561 7 +26878 43830 12 +17276 43205 4 +50968 33631 8 +44686 5615 8 +905 37817 6 +51169 40858 4 +41712 56972 14 +54163 14449 12 +4034 16658 15 +41629 40321 5 +28618 41260 11 +31868 25082 3 +30432 30119 7 +28914 48440 9 +43529 44338 8 +43498 40152 8 +30033 31172 6 +4853 32934 10 +53561 12907 9 +39782 12117 4 +35619 53221 14 +12909 22893 9 +38711 20811 16 +623 27416 11 +45053 50303 13 +8221 23552 17 +7586 34357 12 +30452 18654 13 +38983 2522 10 +9892 3379 7 +44019 56665 4 +35200 43707 13 +32499 2270 6 +55634 35967 9 +12982 25217 9 +32535 57686 1 +25910 36307 8 +58402 36745 8 +3390 1456 8 +20826 51657 13 +10886 28637 6 +17677 51758 2 +1609 58996 10 +40980 42639 13 +39188 19820 11 +11065 39637 7 +35422 8745 12 +47317 33356 15 +40397 15984 12 +43559 30530 10 +1415 35626 10 +2755 2167 11 +22834 8243 10 +2326 41042 10 +41187 31576 7 +53346 32875 9 +3833 36665 8 +24395 12523 12 +49452 43132 5 +10887 6197 4 +13638 14199 12 +42596 48344 11 +10223 16401 8 +46130 56472 10 +52998 20092 12 +2434 49428 14 +59025 20547 12 +3386 51816 6 +29112 25849 13 +35685 56764 3 +10074 28357 3 +41723 23580 5 +19462 46423 15 +30946 42403 14 +14633 45472 6 +57885 59810 12 +19007 28837 10 +51606 9886 8 +31678 50681 14 +1792 31474 14 +29118 29308 13 +19035 44548 6 +12558 59772 11 +51309 11339 6 +42382 26373 12 +44270 37854 8 +7037 47336 4 +30919 36053 12 +14387 13086 9 +8070 17479 1 +23754 10645 0 +11965 32528 5 +17360 14813 6 +9040 23928 9 +10800 1213 10 +57938 22796 11 +52129 23153 10 +5637 21216 3 +21511 45688 11 +18917 58427 10 +11444 36215 9 +13704 50627 10 +26986 55888 2 +54220 20345 11 +36102 32337 11 +17138 32255 11 +50858 8143 11 +28324 53915 11 +22355 35522 7 +58876 40629 9 +23067 32356 9 +8276 11825 18 +42119 49206 8 +9965 6623 7 +31274 4805 5 +1493 48566 15 +59317 55236 3 +53333 43788 15 +38629 11049 8 +3845 43286 9 +7230 44985 15 +18706 21891 9 +14559 2672 8 +9398 1655 9 +7866 29882 10 +38330 50872 9 +43444 12751 8 +19460 1636 15 +44687 55992 18 +57487 32466 13 +55216 55985 9 +41414 48603 16 +46777 48964 13 +31267 34574 10 +17085 31836 3 +17853 36198 3 +55746 41711 9 +8320 13007 6 +20464 56986 5 +42329 4177 9 +12885 38994 8 +36033 52660 3 +27740 54414 8 +19640 18001 4 +14589 46149 5 +13875 1533 4 +28881 21104 11 +44876 35940 9 +37578 54154 12 +43836 44492 3 +52301 52010 14 +50569 26628 13 +28725 42909 3 +10646 36163 9 +35296 877 13 +31607 31015 8 +7458 51671 1 +1360 55716 10 +38301 42316 5 +41872 11161 15 +54171 50517 8 +56625 51902 4 +10916 12218 3 +57301 38894 4 +23417 13929 4 +18535 24565 9 +41437 13302 5 +33515 53564 11 +2625 12524 9 +24062 27939 13 +45348 26807 10 +43861 9467 15 +35860 25451 8 +19707 51703 13 +22527 31284 14 +53629 41762 14 +9071 18000 8 +49836 4826 18 +46920 30464 8 +2388 28804 9 +13967 58908 6 +11761 24303 10 +1798 46608 5 +132 36471 8 +26841 38892 9 +18578 24284 7 +23653 39682 8 +41085 15812 8 +52762 14923 10 +5922 39163 12 +3401 28338 13 +59848 23104 6 +15026 31206 9 +38670 28208 4 +52411 8342 3 +13410 49358 16 +41093 18203 8 +39242 10287 15 +45678 29499 9 +30749 52756 5 +21956 55805 5 +33900 47572 4 +43999 33608 9 +6024 18974 9 +6274 46172 9 +23537 48338 14 +58894 6799 11 +11373 46969 17 +49092 18628 13 +12820 4901 13 +12075 58410 7 +20696 15813 5 +25602 17326 10 +31988 6622 11 +12807 48593 8 +46329 11139 12 +45360 21619 9 +55319 9440 2 +18519 38639 11 +25413 25683 5 +57259 52464 12 +4082 42245 0 +34245 16897 15 +58243 25668 13 +34864 4776 2 +5691 54825 2 +37638 14379 9 +15308 15870 7 +47715 5195 10 +30636 3357 11 +45808 44116 14 +14597 55050 10 +44528 8644 8 +18369 31494 9 +3437 36282 13 +16292 51438 7 +28987 40264 17 +55066 50340 9 +25164 48011 11 +17387 6828 11 +23271 3497 15 +54768 32865 6 +25358 58270 11 +35798 23784 8 +36725 17039 8 +44604 6088 15 +25181 8997 2 +31863 52647 7 +52344 3289 9 +28636 55407 17 +9796 8400 5 +9958 52241 13 +7491 13047 7 +11194 40892 12 +12568 30564 1 +42299 39670 12 +23044 54131 5 +53505 19145 10 +14487 55738 12 +58850 50313 10 +38942 37116 9 +35267 24824 6 +36046 8741 10 +26828 39972 14 +27803 13892 4 +50843 36140 13 +4399 52104 12 +31753 16657 3 +50709 22702 7 +34604 57632 4 +7442 35339 12 +56236 24633 2 +34194 1880 10 +45170 25549 8 +3117 59683 9 +54194 53543 13 +27277 6097 9 +17120 23957 8 +32844 19436 12 +46038 1167 13 +29026 41698 15 +36432 30641 10 +51344 8868 13 +25891 11859 13 +38041 38239 7 +40126 11344 11 +5919 52059 6 +4139 47430 14 +53319 26508 0 +18745 12186 7 +15945 48845 4 +55740 55055 6 +56990 7883 9 +44977 36207 9 +46925 46428 12 +24698 36425 6 +37445 36131 10 +46445 26636 6 +25409 55806 3 +45548 42819 8 +26522 44269 3 +46131 35876 9 +21536 24120 6 +15659 18503 12 +15154 38386 9 +39043 49609 10 +46509 12999 14 +34095 52492 3 +5496 8359 15 +41476 6335 11 +8436 6833 11 +51900 50072 1 +48620 11485 15 +28540 37778 8 +38410 57479 10 +2503 17244 11 +13917 25783 8 +2401 12752 7 +20888 17891 5 +27692 38292 18 +42899 50352 6 +50188 23139 10 +32724 24922 9 +9007 2910 13 +34939 26251 12 +33965 1135 6 +20403 42152 16 +13442 18338 11 +21561 51249 5 +14673 41028 9 +45052 35710 6 +5154 18924 3 +36588 27313 8 +23215 44199 8 +14685 33519 9 +59807 50896 11 +57761 34821 6 +21443 16681 11 +8616 51726 9 +22785 39229 10 +6670 24706 6 +4253 1294 16 +152 6124 3 +54315 17975 10 +1041 27479 15 +43696 11055 10 +7689 31322 5 +8510 21582 12 +14582 14250 17 +43613 27665 1 +48380 59388 13 +45833 19338 3 +21814 5135 6 +44614 1221 9 +19602 58827 10 +51186 55012 6 +3993 31334 14 +471 38080 11 +53762 26218 9 +19911 21075 11 +49013 36093 3 +27139 6276 10 +38009 5971 7 +34270 842 9 +29465 19073 16 +18641 36266 2 +5809 57278 3 +26039 10516 15 +47844 25475 13 +42847 59766 10 +6888 40722 8 +22516 24795 6 +8053 54833 1 +1861 7622 3 +9406 57793 5 +43998 20167 10 +48539 24050 8 +9955 6184 11 +18172 15359 9 +8799 40024 7 +51977 40251 6 +3110 55636 13 +56836 57460 2 +55550 41401 3 +50141 47822 3 +59363 29975 9 +25582 49631 13 +15996 57388 7 +36335 50328 2 +712 11162 0 +26238 17072 2 +930 13608 17 +27186 15486 5 +19546 48630 15 +59217 18341 8 +22049 33387 11 +32585 4810 6 +37725 9521 9 +10829 2436 4 +27567 39399 7 +23662 36659 4 +56307 29906 10 +15123 57729 11 +3518 55594 8 +36148 262 11 +9867 35163 5 +31889 42595 16 +47162 23375 10 +12793 18218 4 +44187 16125 12 +18232 17304 0 +55518 22086 16 +14349 31052 8 +13942 7965 8 +5618 45989 3 +13962 33914 0 +41379 34197 7 +12543 55114 13 +22922 34781 6 +16776 56502 17 +6305 49436 11 +24346 46996 5 +36326 33222 7 +29750 5010 0 +6930 40497 8 +52372 41463 17 +41735 10315 13 +6816 10187 16 +45628 38581 11 +4813 19719 6 +56312 20766 9 +16023 37022 11 +14505 1347 14 +44802 56852 10 +52071 29977 12 +4198 33751 12 +36563 45746 15 +36245 31417 14 +30133 9888 9 +8574 49212 8 +14283 44891 4 +50508 34595 3 +47917 29209 6 +20819 2347 15 +26293 49490 9 +19068 28664 9 +37513 57755 3 +9876 59129 17 +1155 37286 2 +29718 27856 3 +21853 20425 7 +44857 24133 2 +6669 8093 4 +56233 53667 8 +8725 35559 9 +46613 50698 13 +54730 50079 12 +15079 54771 15 +8307 42804 8 +50641 40212 7 +564 24867 7 +26331 15860 14 +43729 51252 10 +10464 36995 3 +58914 18110 11 +7239 23740 10 +52023 20912 3 +7076 8386 16 +22692 54491 14 +12611 22577 12 +4088 48862 5 +53831 22204 12 +26045 44127 9 +52727 26760 12 +56310 31107 14 +35557 58367 10 +24755 56179 6 +48118 13210 8 +56207 35937 10 +3304 31416 6 +14707 19741 10 +53124 25352 1 +33446 910 7 +6714 5955 5 +7405 18836 8 +51296 17715 8 +1308 30319 13 +22710 50582 1 +6034 8360 3 +48355 50064 10 +4111 39741 5 +31925 26374 10 +2536 45819 5 +30561 35732 11 +40751 53512 8 +42768 56820 4 +34553 41456 4 +22995 11265 12 +43378 58444 14 +21701 20189 14 +35036 57467 16 +18067 25919 5 +52463 50384 16 +32706 22591 17 +24453 32805 14 +15430 6877 9 +46065 24152 11 +17071 41327 10 +8772 35770 3 +8247 8761 8 +16827 39255 13 +21925 23682 17 +41306 55856 18 +6240 21686 17 +23506 29015 6 +11221 34325 8 +21474 17756 9 +17895 18267 8 +9034 20717 4 +49954 24417 3 +28986 14794 8 +59913 28820 5 +50007 15645 11 +43656 38964 15 +52438 5822 4 +52817 16376 1 +38602 28459 9 +1275 2527 9 +31509 13008 4 +52362 12864 3 +1111 13198 7 +6323 26516 10 +40577 5666 6 +39729 57765 10 +1425 50828 1 +26451 40518 10 +57952 33581 12 +49042 14341 10 +517 40743 11 +6074 55326 11 +45585 57351 11 +38119 55932 11 +20505 18414 17 +34266 1426 13 +28835 29333 7 +42824 14242 14 +30757 50022 5 +36541 49940 12 +4332 34018 10 +6027 37077 14 +29073 35995 4 +34035 48418 8 +32681 50439 16 +4864 38800 5 +19698 2511 14 +3460 27953 12 +25027 26427 1 +44681 30212 3 +2391 57840 11 +40099 47651 3 +46318 54925 16 +12190 56477 12 +15721 45852 9 +50152 46358 9 +15880 15502 13 +37222 13367 10 +43950 15977 10 +29493 27470 10 +22626 25650 7 +52393 57395 0 +52569 27160 4 +47939 8565 5 +8501 21175 15 +16626 20243 6 +49767 14067 13 +52185 39551 4 +22034 44013 8 +16910 59561 12 +54029 20178 4 +14364 1981 12 +33376 39555 6 +53855 43034 16 +26675 47849 12 +14926 47311 10 +21908 31979 6 +51489 48596 11 +39884 34475 5 +59979 58966 4 +26873 10451 10 +33155 26583 17 +55235 37112 13 +23330 1701 2 +49039 15838 10 +45579 3469 8 +45451 504 6 +27626 58878 12 +53114 31292 13 +39028 47889 5 +18298 14016 6 +42364 756 16 +49753 58621 8 +20021 30372 3 +56497 55963 5 +13012 24563 10 +5046 38371 10 +7426 22667 11 +51973 36525 8 +50180 15494 15 +25985 41261 13 +59249 4794 7 +58087 15262 4 +32042 30570 5 +57343 35599 6 +24017 14611 11 +57305 47720 4 +10844 7203 0 +6775 19184 8 +3944 58918 14 +46229 48890 10 +23669 22053 13 +59359 31126 10 +57178 29576 15 +10104 3564 6 +14761 19987 10 +34006 47212 11 +12390 56525 15 +35037 41380 12 +7055 29732 0 +43256 17919 11 +24785 42102 7 +9981 24097 6 +4761 59013 16 +31549 9659 10 +6443 38835 1 +38129 53106 9 +40646 20824 9 +54426 39971 15 +58159 49565 18 +45588 85 9 +42790 33993 12 +24293 54222 10 +26303 49310 7 +50719 35207 11 +23251 49361 13 +6767 34346 10 +4705 24953 16 +30553 51994 10 +7960 14670 7 +43597 22329 7 +43269 13814 8 +27059 26897 11 +17006 17242 8 +50436 42857 13 +23660 25690 1 +19534 38326 9 +56446 24255 11 +21512 37630 5 +10760 9061 12 +43181 40398 16 +31876 46272 15 +49058 17064 7 +44290 3240 5 +29477 6301 9 +53547 19710 9 +54485 2381 8 +12632 54607 2 +27625 35374 12 +1914 30625 8 +42888 21610 4 +14547 29278 10 +49584 6239 13 +23123 27045 13 +14842 52877 15 +55874 46725 4 +28207 52234 8 +19800 35775 7 +31254 55756 10 +40831 49041 3 +59480 50974 3 +12415 42300 5 +11310 244 10 +30402 15877 12 +21553 7236 7 +29816 50541 4 +59521 18882 6 +2539 8492 9 +24849 29367 6 +7732 54464 2 +47412 34471 5 +45479 7697 13 +12546 26053 6 +43852 195 11 +3482 43521 17 +20939 35988 9 +28335 5463 10 +48948 41348 9 +54869 45736 7 +31865 52543 3 +8525 52502 15 +8081 6759 8 +18949 52977 14 +59084 40528 18 +20574 51878 12 +32144 30690 2 +52332 18387 14 +42462 48198 5 +14823 21025 9 +43985 45122 13 +59598 1139 9 +55408 5282 9 +36324 49270 8 +41584 23122 11 +52198 23101 9 +34086 20526 2 +34376 41632 11 +44053 53439 8 +29036 1166 16 +43516 10473 10 +647 40224 8 +9053 6517 9 +43992 30535 10 +34107 3849 8 +1986 10857 15 +1440 9026 8 +19996 40633 5 +8689 10112 10 +30728 40878 8 +32851 10541 5 +23770 46174 9 +34518 32779 9 +28056 25406 3 +19770 13831 3 +120 48745 6 +8027 40170 12 +19087 20660 10 +44022 11888 9 +45723 28718 2 +26864 19970 10 +52834 54159 7 +21300 47214 15 +57866 19401 11 +28269 2604 3 +3782 21923 8 +56460 8334 2 +51373 28500 7 +24802 36623 9 +10063 28180 14 +27541 4020 10 +38263 2792 3 +44382 5838 9 +10512 48554 6 +18447 54907 12 +24797 199 9 +53453 17701 7 +37620 43593 8 +38600 11834 9 +39512 21905 2 +57224 40214 11 +33023 50016 1 +1869 46973 15 +40647 52497 16 +27921 38622 9 +26871 45090 9 +47388 18117 3 +54009 18174 5 +5941 19863 6 +58281 4681 9 +45434 12339 0 +24282 27587 6 +33834 23096 11 +37387 30414 14 +58171 36465 8 +44215 54460 8 +12368 34021 14 +33860 37850 14 +51499 38873 0 +56593 28707 11 +7598 50704 10 +41461 33867 14 +37333 4324 17 +36150 31937 7 +12830 38655 9 +27893 10753 8 +9522 58671 14 +26857 56926 7 +59046 46289 3 +45264 50224 10 +4120 10320 5 +36903 53261 13 +24790 6857 2 +21020 37580 11 +27436 59776 9 +31828 4948 14 +34213 49872 0 +28028 56851 5 +40118 19632 9 +33318 15674 12 +23177 42632 2 +57962 37801 6 +30931 25448 3 +46418 41857 5 +5491 21899 4 +9031 32663 5 +25819 51301 8 +33218 37248 8 +42495 20672 9 +25252 47373 1 +17885 10194 3 +46663 50191 10 +17140 21461 11 +46574 2135 10 +27670 12565 11 +34459 45456 14 +59825 12129 5 +39601 2671 9 +44756 55517 14 +27906 39919 9 +29062 52668 2 +57348 53952 16 +27472 10636 6 +28492 38102 4 +3576 713 5 +52535 56935 14 +30488 30087 5 +19158 11956 5 +11711 17423 3 +35379 32001 8 +31391 54709 9 +35449 44024 8 +48119 51365 1 +29802 30440 15 +34796 761 9 +40909 6577 10 +58844 34969 16 +16158 6457 16 +58577 2853 10 +51996 59682 14 +36057 50814 5 +53609 40933 7 +5849 53485 14 +47591 14118 7 +53660 45038 12 +49398 24534 12 +48645 14210 4 +53323 27613 10 +31007 17272 7 +45992 40941 13 +55852 18003 10 +55953 14891 12 +7583 40927 11 +15191 43496 7 +8160 32756 2 +9103 44563 6 +16128 21787 17 +3055 23769 17 +10319 58561 9 +10225 45741 4 +31620 1651 9 +45441 4964 10 +58892 21718 12 +29833 19258 7 +48987 48237 14 +3945 9296 4 +33249 40149 17 +32177 10042 12 +1172 57950 8 +20438 58846 6 +41970 33020 5 +27585 40337 4 +10142 9064 14 +17255 48791 3 +32750 6350 8 +52554 56355 2 +11769 54938 11 +49382 4176 12 +668 28596 5 +7029 33248 6 +54208 59180 8 +39113 32777 16 +15631 34378 12 +46855 30048 4 +47410 11147 6 +44824 39391 17 +49964 21132 1 +46349 21008 13 +54975 17158 9 +19275 8066 17 +7263 56725 8 +29682 43128 7 +40783 44874 15 +42761 4969 10 +54995 33081 12 +48239 50854 15 +47310 15555 2 +9359 4682 8 +27754 50487 4 +10689 39003 7 +21210 6917 5 +34687 33301 11 +5930 500 4 +17165 23841 9 +38424 27930 4 +17575 11363 9 +2115 10048 8 +30461 21695 6 +58881 7328 11 +18147 2455 9 +28345 6307 7 +15609 51566 10 +20450 11729 9 +40080 42775 10 +34409 42626 9 +46729 59932 4 +15875 54329 7 +56056 35127 17 +47341 16490 14 +35471 8552 10 +13145 57171 8 +16202 6015 7 +36526 39694 18 +41335 16201 11 +20033 58800 8 +20467 5778 11 +31263 4589 9 +2931 25878 7 +49724 533 9 +29753 40488 16 +40948 39501 11 +12996 16900 8 +25298 50360 16 +24631 54828 6 +5574 47267 8 +46865 38139 1 +47283 4336 12 +15377 12618 7 +34630 40829 3 +904 30537 16 +54418 6774 9 +30840 30083 8 +34648 20681 6 +4532 18049 3 +24723 57247 10 +26969 16420 14 +37257 22743 11 +16435 44561 4 +41106 10781 8 +5934 57079 14 +58854 39243 9 +16987 31483 2 +25772 49889 4 +590 6091 11 +52486 25596 8 +36476 12132 1 +45304 26413 14 +34740 16927 6 +38491 19629 15 +44792 54081 10 +6966 23045 4 +36771 40859 9 +804 20007 6 +42534 19949 10 +47163 25702 9 +44004 33936 4 +30172 54396 15 +45597 23798 10 +6132 37113 7 +191 25186 13 +47050 39910 6 +22528 18050 8 +29563 14390 12 +34868 14146 14 +30909 5550 6 +632 4696 8 +43285 55520 13 +9754 9270 16 +42563 35176 9 +30720 37789 6 +15993 12379 8 +49589 11239 11 +41492 3413 3 +17040 32650 5 +48077 12229 14 +22956 27148 1 +42645 5398 9 +23826 34148 7 +44425 3189 9 +49725 26743 11 +6469 9430 5 +40701 37654 16 +50151 35509 2 +30099 59602 16 +23534 28616 13 +22701 20432 4 +55648 38778 17 +58369 20377 12 +59328 5055 11 +26962 33733 11 +4874 33400 7 +17606 45860 4 +54573 30195 6 +15850 4527 3 +22449 26783 9 +54305 13341 10 +51326 5713 9 +53517 8132 16 +56637 45036 6 +5789 15201 4 +27017 56614 10 +25686 56022 14 +34449 22414 6 +48872 53200 10 +11631 6698 6 +20339 575 16 +18476 28199 8 +6513 41809 6 +10741 53232 8 +46156 24263 16 +25785 44884 10 +55087 38680 13 +5370 43989 4 +27828 59441 10 +39988 21293 8 +8299 17923 8 +8323 39543 4 +25243 36530 7 +7919 53790 6 +17139 18123 8 +8378 8546 14 +28730 19091 8 +48798 209 3 +19153 59884 15 +31305 47154 12 +26105 35484 12 +59637 18838 12 +48281 23837 7 +16548 53240 15 +41626 46706 13 +35891 38434 11 +21316 34896 12 +27775 46007 6 +37911 20921 8 +44213 58379 7 +40177 46940 11 +4895 58060 11 +9749 6610 13 +47640 17744 11 +18425 16503 3 +2658 19425 10 +33487 18428 7 +45298 42616 12 +43105 12952 3 +789 20743 11 +32135 13311 9 +30695 43029 4 +4802 35131 10 +23309 27187 8 +58343 54501 4 +46994 16893 14 +20157 12064 12 +36790 21201 5 +38218 41021 12 +8847 17623 15 +35121 20886 7 +54500 24482 14 +48179 56224 8 +54168 18929 7 +22417 20994 8 +36534 21379 8 +28573 27099 9 +58593 50496 10 +27947 33830 15 +31842 27255 14 +6060 42781 6 +12629 800 18 +44598 43758 3 +50531 39898 4 +39680 21641 11 +16057 56521 9 +6688 8663 10 +32915 12926 5 +54167 11545 10 +30314 43695 14 +48544 8446 10 +46584 35547 16 +30021 31372 9 +18720 13551 12 +11762 2099 18 +11976 50250 11 +42521 19569 8 +8976 22030 8 +41535 16249 10 +33032 29651 7 +38257 8987 8 +54704 29451 4 +54846 46941 13 +9486 53669 8 +17526 45340 6 +57413 55809 13 +35181 4172 10 +5735 30660 13 +39704 47148 12 +17875 25710 9 +32707 34474 10 +27046 28283 15 +27113 25329 4 +18321 19854 3 +37422 19218 9 +45753 43167 14 +48794 3549 4 +25249 18939 5 +40487 2355 10 +23223 9461 10 +30896 4003 6 +51304 20795 14 +55879 3619 8 +32151 35720 0 +9367 20223 12 +704 41361 15 +53866 47488 7 +16409 41945 14 +5792 54664 5 +29271 53796 9 +50721 35487 8 +48100 18365 11 +55394 19585 10 +38745 49512 9 +48619 30980 12 +6749 9879 8 +43895 26129 8 +54992 28308 13 +54902 47367 8 +52708 59606 5 +38313 11534 12 +17867 46247 9 +58758 20379 7 +56544 10726 16 +52097 33633 4 +18131 41324 16 +7783 39385 13 +40543 13186 14 +7284 15329 7 +40992 26738 8 +2485 48249 9 +51456 52510 15 +30556 31329 9 +42388 20592 16 +49282 45055 4 +48978 39219 11 +18797 37118 12 +24792 12839 5 +50525 8855 1 +51284 2826 9 +59551 8139 12 +627 1096 10 +23151 26189 11 +23673 54149 13 +26827 41608 4 +6963 49383 15 +5998 41648 10 +7479 47716 8 +32527 7778 10 +33740 36504 10 +44016 10651 16 +11624 37201 5 +46754 15691 12 +38694 41398 8 +3529 3629 6 +33910 29389 12 +46005 42861 5 +4924 50023 2 +15335 33411 7 +11522 30448 6 +2271 14493 16 +53570 24068 8 +35070 59655 11 +46961 23737 17 +13713 54399 7 +45710 14064 12 +12942 49483 9 +50201 37836 8 +8969 53166 9 +9581 35317 8 +40378 16029 10 +29643 53707 7 +16682 13120 10 +7380 6704 7 +8060 10747 9 +21402 37913 4 +34757 50390 1 +35006 2589 5 +42949 29017 4 +19102 18813 10 +18746 42983 14 +55826 38071 12 +790 23787 7 +2623 18734 8 +20646 50959 12 +9571 40339 15 +10853 16581 15 +54601 2650 16 +45143 12965 16 +53617 12420 12 +50424 25625 9 +31641 41282 4 +54445 17816 7 +25840 10217 13 +35515 4101 8 +51307 24 9 +41897 45897 6 +44046 59057 9 +7252 2978 15 +55022 39337 14 +29770 8332 3 +13037 26221 10 +8805 34511 7 +21486 16386 11 +44334 16331 6 +26774 30914 9 +55253 38349 7 +38363 36902 11 +41161 48838 12 +7 19164 10 +11106 35738 7 +34103 20391 9 +18132 32606 2 +44438 27397 18 +45495 37532 3 +37472 22096 9 +138 6635 11 +6063 36352 9 +42740 30544 6 +23191 19202 9 +28853 30934 12 +52118 52440 4 +18057 56933 18 +43586 59451 10 +3633 30539 15 +48692 51518 12 +52038 53469 3 +54585 14592 12 +21162 48963 9 +28023 16890 14 +49892 45563 8 +37476 50900 14 +55328 21032 13 +42602 3314 4 +26011 29047 6 +21518 26912 13 +15818 15746 16 +59767 2078 9 +16869 33829 7 +27813 41127 10 +50753 10028 11 +23679 18828 9 +2420 3639 8 +13694 33790 8 +6173 27809 11 +4777 38685 10 +33160 40643 9 +45359 37244 3 +12788 45693 5 +50764 47656 13 +7130 34025 0 +2512 5467 10 +37936 24401 10 +44052 50589 5 +1905 4990 15 +24588 23148 15 +38995 55509 12 +20729 9565 12 +45931 15817 9 +577 48827 6 +48515 58155 8 +12173 20775 14 +43640 30118 13 +41724 12312 7 +42273 46523 5 +10856 50776 14 +35197 40323 8 +45252 45671 13 +58990 15209 15 +44212 50408 16 +50179 25870 16 +57167 54685 5 +44937 9328 9 +45677 30122 3 +29809 30107 2 +20599 53395 5 +43438 43458 4 +6209 41411 4 +43752 28652 10 +10100 38194 7 +36143 26740 2 +39917 55882 14 +42336 17830 9 +51950 44564 10 +40328 12730 16 +7546 13002 14 +6288 47098 13 +13995 22468 9 +51065 27669 13 +55530 263 15 +32872 42929 4 +42088 32348 8 +5238 15008 6 +57856 10266 4 +58212 59543 8 +10192 1334 7 +46270 15334 14 +25989 38162 1 +9154 40480 11 +54076 54186 16 +56748 49238 13 +34181 10153 6 +3742 44035 12 +386 29638 12 +10169 48865 6 +53650 6557 7 +37069 50339 9 +2104 29501 4 +42341 33628 3 +29721 54860 13 +57831 51261 6 +19823 58804 16 +46764 30250 15 +19319 9534 11 +58657 30686 17 +40061 19975 11 +41143 45400 6 +1872 43878 5 +5185 30501 16 +54859 25420 3 +49122 24716 8 +49368 10496 8 +44375 28980 13 +48154 55534 17 +56093 29089 11 +4250 5411 12 +42082 23295 4 +24277 56014 11 +40106 56677 14 +59996 34881 10 +5275 38084 6 +35841 27052 10 +56305 15909 10 +54580 26270 4 +1032 38457 14 +46214 28368 11 +10393 49668 8 +38605 57044 7 +59985 49552 6 +8068 57721 11 +20897 11636 4 +50379 56927 3 +9371 53654 6 +52915 23757 8 +23079 46417 3 +26761 43208 3 +45732 53276 12 +55519 18676 13 +1924 40974 4 +11017 36756 7 +6509 5331 10 +24533 35115 12 +3665 21826 16 +25797 31489 12 +5924 9339 14 +20772 48352 4 +44076 9721 2 +7522 17342 15 +13310 42939 9 +29280 3582 10 +33931 57251 14 +8900 47434 6 +21281 8667 7 +4913 47790 2 +8390 58267 4 +34920 1769 9 +12737 13851 16 +34517 27414 9 +23516 33472 14 +23066 15930 8 +27115 31602 6 +58624 54120 15 +43345 25230 3 +37214 45151 5 +24065 4351 9 +35191 22404 12 +49284 43551 10 +15339 45001 4 +4413 10031 8 +37566 6068 8 +40239 26083 9 +852 37720 14 +35630 55554 7 +2217 54864 5 +21308 38860 1 +22018 8024 9 +58836 24370 15 +19613 48097 10 +3579 18209 12 +15541 8814 15 +37811 48983 12 +57309 49141 14 +11947 48861 16 +53327 8989 5 +33460 17676 11 +35198 25202 3 +41128 18721 15 +9215 11702 1 +33035 38574 6 +6679 26079 17 +13928 12467 7 +52110 25675 12 +25941 18887 11 +16067 53457 12 +9239 53764 15 +2007 5687 10 +4370 10398 16 +24355 14839 6 +40387 22485 16 +22786 37218 10 +58458 20032 7 +54985 55448 7 +1774 20809 6 +34355 42867 0 +28369 14690 10 +33504 8936 10 +37235 18903 12 +6222 14040 4 +21231 14821 9 +54593 32988 15 +9905 29140 13 +11492 40284 5 +33492 59320 6 +46805 42095 7 +18430 33924 7 +30333 35202 7 +1624 1123 3 +41904 59987 5 +23196 46815 7 +8551 8440 12 +17695 28509 5 +2023 1911 7 +24924 55665 4 +54773 42885 3 +59491 31522 8 +15960 6151 11 +23376 58198 9 +41543 33855 12 +4736 43035 9 +41996 48677 8 +25209 42309 5 +5801 8318 9 +42904 26606 12 +32867 23465 6 +56918 27878 8 +24012 1755 4 +38968 10244 6 +30027 3952 8 +7989 49473 6 +13667 16078 5 +39014 279 10 +21981 36125 15 +34371 52991 9 +52914 52712 14 +43815 45342 4 +37606 48194 9 +54394 37970 8 +28048 47511 12 +29276 6621 12 +23643 38471 6 +8977 41577 8 +33146 30059 7 +35172 8234 13 +19231 20173 9 +17018 54176 11 +20429 14852 6 +22356 32195 9 +12428 30446 16 +37726 48562 12 +44148 18748 7 +21420 39181 6 +321 21909 3 +12271 15138 7 +33314 27825 10 +59094 10807 5 +54983 55276 10 +20909 22411 9 +41557 11347 6 +2655 40147 6 +4568 31070 11 +15001 37767 5 +46399 484 9 +40711 4400 8 +18028 52801 2 +7285 5359 7 +29077 22845 10 +8631 56099 3 +30136 3828 7 +48041 5451 7 +33858 27601 14 +42145 34949 11 +12353 29319 16 +19469 59453 11 +59636 24523 14 +58717 57582 13 +1876 20416 6 +44231 19940 4 +23013 33056 6 +28101 46237 8 +27143 9155 11 +9594 11893 12 +23553 35251 16 +9441 44457 3 +56814 48405 11 +13712 32637 8 +11661 15554 7 +19104 15749 8 +33947 56955 1 +4906 27437 4 +17480 31734 7 +30325 19286 12 +10022 50651 8 +13045 18142 10 +3657 13486 10 +4002 53490 7 +19468 23238 3 +35868 50341 11 +14427 46142 2 +24341 28693 4 +27509 54079 2 +24558 56654 11 +17135 18302 7 +1470 38660 3 +1506 54393 7 +39882 24140 15 +41054 23156 10 +19711 25198 13 +13692 31019 10 +167 39909 12 +43667 43856 14 +1735 57401 5 +29071 33724 8 +15741 24175 10 +17309 24410 14 +2845 23755 9 +29969 566 12 +25275 38728 15 +10991 25428 7 +48459 42796 9 +50087 20640 11 +34623 44809 6 +10702 17434 9 +20124 35231 9 +19990 2465 9 +11022 41295 9 +44238 23973 7 +54854 39297 3 +46904 37310 14 +31250 52224 3 +31831 45338 9 +49874 26266 11 +19606 51054 5 +3074 35180 7 +40857 4902 18 +31693 3914 16 +17900 46021 10 +33438 57410 3 +5108 11758 0 +52096 9694 11 +31594 50480 7 +2160 43351 2 +55306 26590 6 +58585 44753 7 +21357 12681 2 +28430 39492 9 +18853 30608 7 +10941 10292 11 +53962 50949 7 +41481 43822 14 +26002 37345 7 +29621 12799 0 +51769 51399 10 +29084 56263 11 +22498 3859 12 +47880 20410 10 +58525 5390 12 +24095 38543 3 +30819 12687 10 +21967 32774 14 +57587 45453 8 +10553 57862 10 +50355 40281 9 +58895 30599 11 +6859 57257 9 +23779 20615 6 +20593 27378 6 +20724 47643 3 +20487 25401 4 +13324 32722 13 +51031 12903 15 +18592 14621 8 +34935 35357 8 +11532 22278 2 +21441 12973 6 +47702 48033 7 +22077 28991 14 +50859 6712 14 +17034 40402 10 +15856 52537 8 +56536 14703 12 +41410 2012 7 +4638 21124 8 +43965 52298 3 +39582 30542 6 +2773 34944 6 +38324 19429 13 +18265 20816 7 +17382 57236 15 +31640 28136 9 +37440 20436 4 +33723 54703 0 +3600 13117 8 +17702 47613 16 +14166 29490 3 +50795 50960 10 +4922 3523 9 +46332 54643 7 +16297 15677 7 +13699 37349 14 +47683 49808 13 +31986 2574 16 +31954 43402 11 +10171 8707 9 +38814 19119 7 +57030 32795 9 +34130 148 11 +43552 23418 8 +54881 16902 6 +46314 38886 6 +21715 54105 3 +41315 13300 5 +7305 52242 2 +41639 9734 4 +18872 17001 14 +47785 16984 8 +18816 44393 9 +56050 11465 9 +1438 27136 3 +12675 8427 7 +58663 36401 7 +59525 33547 7 +6647 28427 1 +22447 28732 14 +31208 19214 14 +28362 57912 7 +18247 26463 3 +34151 35236 9 +39474 20590 5 +41874 40582 6 +33787 36985 10 +59263 13079 11 +35167 55547 5 +59337 7277 7 +23426 52619 9 +57337 6870 8 +43223 33955 13 +20323 28969 7 +20443 3441 1 +515 23771 11 +48461 24345 1 +21271 18179 1 +10275 43127 11 +13284 31063 7 +41124 29143 9 +1016 20963 9 +7132 13548 7 +11716 56146 11 +37517 7865 8 +47208 51217 8 +45696 35065 12 +11985 48266 7 +38161 17203 4 +56198 6102 9 +8722 15822 12 +29611 10214 5 +39705 44315 9 +34485 51503 12 +46609 47858 14 +19476 27957 7 +38356 3869 8 +5757 20702 9 +48103 16279 8 +32427 40792 8 +37148 1637 11 +16387 3439 8 +43439 48148 11 +34571 29214 5 +22484 4471 11 +15603 46786 6 +45484 52568 9 +44584 12153 7 +52348 866 6 +54188 47907 8 +52228 25016 8 +17549 7615 2 +25214 25072 8 +13080 7166 5 +16600 11197 7 +7627 3446 17 +7212 34912 14 +31374 38951 8 +12631 57397 5 +30890 6700 4 +20867 14444 7 +58393 12500 7 +54836 52347 5 +1097 19113 12 +16132 29440 16 +43257 58973 2 +19134 47776 7 +5871 59045 5 +37066 39139 14 +18314 18140 7 +9934 46705 1 +18680 35601 15 +23776 47769 7 +45999 26385 9 +40026 22817 16 +4314 33105 7 +47078 46842 1 +46175 53098 9 +42779 20757 10 +4571 8233 7 +17178 46116 10 +31109 27850 6 +33720 24746 6 +45520 42766 12 +20954 36357 11 +14300 29235 6 +48736 17638 8 +48312 43974 10 +19221 3769 14 +6628 41132 10 +55747 40302 10 +29674 7432 4 +39889 48488 9 +17153 1608 9 +44990 10433 8 +43268 14381 10 +23540 31488 10 +13271 7002 8 +50147 28440 13 +59794 45234 13 +29578 19230 3 +3651 13531 6 +8993 7018 2 +55855 33196 8 +55453 34453 10 +2014 4021 15 +35802 32190 9 +38444 31382 13 +7353 457 4 +32869 5343 8 +29941 35274 9 +2425 50657 14 +36382 4995 8 +14265 30808 15 +1207 20829 9 +9267 23675 9 +6842 44684 12 +38093 28777 9 +41045 30204 14 +42604 32656 8 +22964 44087 11 +20402 43064 12 +11357 35421 15 +18841 51372 3 +17217 43991 8 +22807 33822 11 +261 25552 9 +44261 11010 12 +46677 30029 12 +40650 1013 14 +15087 6568 8 +17170 46922 12 +35888 32111 12 +55188 28854 7 +42950 21275 7 +15616 7133 13 +55764 21751 9 +499 18637 12 +24605 15324 6 +9437 38650 16 +20995 48302 7 +47679 42648 16 +59744 27032 9 +54229 48325 3 +42820 24107 13 +27848 6737 9 +58824 11865 7 +5746 58922 15 +29378 40748 5 +20647 22941 13 +53826 23314 14 +31946 2159 6 +23591 48131 9 +3169 44637 16 +36081 44463 8 +58229 7294 6 +16464 25481 11 +5694 40263 1 +20417 56387 15 +9619 8828 13 +2592 50574 10 +26649 55438 10 +15054 45512 10 +23364 40635 3 +3570 57006 17 +1678 21397 5 +35842 12192 11 +25994 57465 9 +39005 20001 17 +54145 19293 8 +27357 57943 13 +5435 51821 13 +13740 34331 10 +36882 48940 9 +48650 13931 4 +57821 3672 7 +21556 21294 14 +3315 51406 10 +15041 14108 7 +27727 46140 9 +34190 6465 5 +26171 2814 10 +41337 53559 7 +46749 31354 6 +31333 17548 10 +22878 2069 6 +7730 8062 12 +38707 9162 8 +47699 17639 7 +7415 53527 6 +37873 8973 13 +13948 17445 14 +53944 55933 9 +26149 48710 7 +8113 24612 2 +49798 3156 11 +13702 53774 14 +14236 26703 12 +12452 53883 10 +6761 54528 9 +56698 6352 9 +51425 15073 6 +48779 5442 7 +25166 29110 9 +52854 1432 4 +2707 57934 7 +21730 34343 14 +22745 52811 9 +29137 44374 11 +19853 22041 13 +50628 9402 0 +8823 30607 2 +56960 23428 9 +14227 40176 7 +45322 18134 15 +50135 3139 12 +26426 3451 15 +37692 844 5 +47353 38106 9 +4629 20102 8 +42851 13467 9 +23075 8413 8 +28906 474 6 +53779 53399 7 +50068 21254 3 +58925 821 8 +29825 56328 2 +8788 44435 3 +12295 35199 15 +30649 55381 3 +59755 6858 3 +50926 44297 12 +10564 53681 8 +51323 22545 3 +49137 4433 7 +56175 37831 8 +46388 31452 7 +16131 48748 8 +39248 28542 11 +23555 1162 14 +43928 59752 10 +34654 2938 7 +6393 59573 15 +17934 44719 12 +11176 56829 17 +11198 39592 7 +10717 46345 11 +5073 15714 7 +51917 31681 2 +14088 20992 9 +47774 54579 7 +46572 56650 16 +28489 45316 4 +48990 56078 13 +36853 42733 9 +55936 17907 7 +49055 43803 8 +21324 57128 7 +56585 54476 6 +27744 5970 4 +50067 42624 11 +25444 1789 14 +33755 50342 16 +6827 39618 6 +2188 17127 4 +408 40316 5 +42941 23326 12 +27762 15414 17 +1131 39554 2 +32492 30025 11 +4484 19310 10 +21866 31458 7 +36764 2652 3 +17442 41256 5 +17008 50710 15 +41938 13890 12 +20844 7711 9 +56393 24687 13 +17589 59774 13 +5660 53330 4 +57936 12257 11 +39304 48024 5 +58615 7477 16 +40220 25511 8 +28112 17955 9 +49210 7835 1 +45358 14787 11 +10964 835 5 +56633 3227 10 +6249 23405 10 +58078 53110 15 +57736 5697 8 +58915 39450 16 +37746 5284 6 +17682 28484 10 +1516 22884 7 +6019 2088 12 +5175 25693 11 +57022 32568 1 +24818 46809 8 +55722 34835 12 +51351 19754 10 +56754 7396 6 +8544 33759 10 +21488 20145 8 +7045 17494 9 +20286 10259 6 +43676 53292 10 +5575 420 12 +52515 115 8 +34628 50119 15 +32889 6582 14 +16985 37504 6 +59662 22637 5 +52635 59961 12 +58326 37018 14 +52268 29523 5 +11089 55645 15 +57694 59024 11 +6079 58315 4 +55185 14579 15 +48622 38226 8 +9455 46462 7 +17349 220 2 +35927 43518 8 +22127 59160 14 +53167 30318 11 +9532 8669 18 +47733 2781 15 +12150 1898 6 +24607 9910 5 +35475 43120 18 +52329 12409 10 +18109 26501 8 +26317 53005 7 +51265 46732 6 +54195 34119 11 +16283 27150 4 +34253 21998 5 +2585 59266 3 +58009 11289 13 +4890 3797 5 +10203 8294 0 +21274 49883 2 +22136 16691 9 +51325 20238 4 +34794 45107 14 +21113 50550 7 +43295 26716 10 +29291 52947 9 +202 37948 9 +58526 18588 10 +27295 26087 10 +46309 24637 10 +14532 33405 14 +16414 2451 7 +10426 22225 8 +15581 55927 4 +7794 59862 7 +36793 34458 18 +718 36277 4 +52455 16014 4 +22209 16800 7 +8760 37780 1 +18027 13988 14 +22984 39330 9 +28271 22997 9 +25484 10376 5 +49828 10813 12 +23108 51984 7 +1531 18696 7 +43513 38230 11 +37922 9280 13 +41920 925 15 +10684 45832 3 +2214 20294 10 +56128 46838 8 +25029 41687 4 +10351 6158 12 +757 30306 8 +58207 31167 7 +53501 28378 8 +20491 9569 10 +20220 35169 9 +14188 7462 4 +10350 34188 7 +56824 32373 9 +29223 17298 13 +23142 28641 8 +47227 44471 9 +49678 36239 4 +20074 40810 11 +7464 9967 10 +47729 34837 8 +3400 55382 9 +12749 41827 13 +48032 45123 15 +45644 6356 4 +52442 30051 4 +22265 39657 9 +7568 994 14 +7767 28109 7 +27804 4495 8 +52133 4387 1 +37377 37323 0 +16872 26089 2 +20290 46751 9 +46153 55232 8 +23989 54863 16 +3054 16130 3 +50326 9742 1 +11071 28944 2 +47928 46278 8 +11854 17723 16 +59988 38400 13 +15168 45507 11 +15894 33059 17 +33087 49476 5 +49139 37132 9 +29432 37110 8 +30038 13706 11 +47195 48036 12 +1483 59368 6 +47094 14386 11 +11823 59728 9 +59962 55378 11 +19283 14892 8 +10448 47293 5 +59624 14363 5 +35224 57032 12 +26678 30405 8 +42481 2504 12 +38719 5250 7 +25290 39071 8 +27426 43211 4 +30016 54388 5 +17236 52315 9 +53592 57837 14 +51374 24627 4 +14698 28075 4 +28154 21122 8 +15732 5444 9 +35459 7632 4 +4658 53602 7 +7049 44709 9 +57784 26005 9 +5515 41197 7 +54003 7423 7 +51692 2594 6 +29397 58906 13 +31786 58611 11 +27019 14345 7 +46489 15394 9 +23835 44412 3 +22297 43935 2 +54364 10016 9 +57628 15498 11 +37061 32600 6 +9097 39726 9 +41741 45268 4 +33127 54977 7 +45835 497 14 +30006 9144 10 +32141 6671 10 +8228 1979 7 +43737 28666 9 +44232 50258 12 +2439 40863 6 +47307 47731 9 +57039 51355 15 +7327 50766 9 +43601 32375 12 +31500 735 12 +20261 26538 13 +3159 12634 4 +34677 53003 14 +34602 29212 12 +12035 26670 9 +35812 31068 11 +32263 40451 9 +8073 18526 11 +31961 57173 12 +29630 32298 6 +50035 35850 10 +11781 34408 5 +29293 41383 5 +33679 45803 5 +26548 41375 12 +22853 13358 17 +55004 10329 7 +24328 48803 10 +31668 36719 11 +14569 31291 9 +12014 56616 11 +26172 46435 13 +32733 46185 9 +37874 55871 10 +58566 33713 14 +5088 9372 12 +49251 23024 7 +55812 28187 5 +2944 39452 3 +25580 31852 7 +11109 19889 10 +59173 4221 14 +46785 26341 2 +35209 20206 7 +11093 3485 7 +12388 28140 3 +51426 50204 10 +12640 37981 6 +28428 30515 3 +53662 58875 5 +49624 26847 10 +37525 43036 14 +27672 52151 11 +51327 14966 4 +4430 15044 7 +50054 28793 9 +22568 13401 10 +58109 47146 12 +359 50593 9 +16812 36484 2 +14872 14436 17 +53571 49758 9 +2926 32555 12 +46477 43118 10 +38626 21776 9 +35505 10575 8 +29484 14069 9 +46467 14859 5 +50915 16381 4 +57392 23903 13 +12757 29041 8 +24969 18818 10 +3144 4916 17 +58129 26938 15 +715 55779 10 +11208 11801 14 +55083 7774 10 +11301 37561 10 +33622 38270 9 +34650 34286 10 +3921 33242 11 +25208 29684 3 +6438 21611 13 +50041 43511 17 +31745 47178 12 +15147 54861 11 +39122 8756 6 +45253 52580 5 +50460 11671 6 +57808 43289 7 +17142 49381 11 +12601 35831 8 +9115 29170 16 +21523 45486 2 +43659 14083 9 +20235 2150 17 +30098 13905 15 +20008 21129 7 +40962 51787 5 +35315 39826 9 +55120 11461 15 +25904 22993 14 +31361 21470 2 +45399 14792 11 +42310 44995 11 +50956 56658 6 +39079 14269 2 +29595 17546 13 +58774 39419 11 +57575 51032 9 +51577 37258 11 +58609 41686 4 +52434 1332 13 +28795 45599 5 +26826 45129 7 +37290 50165 7 +31999 41933 10 +37172 16486 10 +58534 363 10 +5092 59919 4 +10280 13093 8 +27137 23209 5 +52795 43889 12 +21463 19001 12 +29124 21934 15 +49462 55754 15 +38974 49185 1 +31059 28560 2 +45669 12159 6 +15221 2933 13 +40672 51641 10 +41992 49960 14 +55915 59614 11 +56343 11814 10 +52509 4911 5 +50856 44617 12 +36236 37823 10 +41133 11230 10 +16580 40832 11 +39024 16752 12 +57143 18555 12 +27639 4473 5 +45383 55490 4 +33011 5961 4 +21015 27607 14 +36444 36017 10 +2181 50366 9 +32044 33319 4 +36280 53497 9 +26043 43770 9 +27442 5669 15 +58642 14743 12 +18013 57104 3 +43843 35326 13 +27779 49970 15 +8200 29075 7 +46614 19438 11 +8076 40964 11 +35501 43908 14 +33507 12875 5 +6248 14145 14 +4363 32904 16 +12673 15940 7 +42704 45859 16 +45290 50870 7 +49598 25823 2 +55822 50542 0 +35110 1436 5 +49028 24914 11 +34352 52091 8 +17519 51524 5 +37994 50975 10 +41700 39923 12 +9909 9323 11 +45228 27415 12 +5915 11238 6 +47260 59626 13 +40020 36518 7 +10050 31502 7 +7445 23084 16 +20229 56009 9 +44595 31319 14 +43885 10484 8 +25354 29738 10 +26629 39677 9 +9211 23014 6 +59135 3754 12 +33943 8911 7 +25595 58951 9 +52030 47143 11 +32745 10603 11 +35936 41772 5 +874 55273 9 +8130 9005 7 +55282 33730 10 +36806 44095 9 +47966 58928 4 +38647 8455 12 +12745 52520 2 +46962 31330 7 +59780 9471 6 +13408 55154 6 +51838 2230 7 +10993 26263 4 +50786 22629 5 +36974 13888 16 +38422 12119 7 +28159 38401 6 +4687 12357 2 +55430 46291 9 +5017 48227 9 +402 30047 5 +33137 149 8 +53582 33997 12 +24750 20609 6 +15039 17437 11 +45179 19020 12 +50373 46016 16 +9828 56628 14 +2310 13128 4 +3456 29556 2 +30108 4300 5 +59873 8422 7 +53859 9897 7 +41469 10814 7 +34336 11460 10 +42227 25110 7 +40409 30115 8 +54633 39098 8 +37546 50784 13 +27673 5651 9 +11533 32964 4 +12040 15341 9 +4342 41345 10 +42887 47048 11 +28739 33349 13 +7129 55196 6 +1691 13204 13 +58744 24027 12 +45828 42407 14 +58764 41214 10 +27561 27823 14 +28010 46937 11 +1871 20306 12 +4877 54987 6 +31966 27943 7 +2192 53572 5 +15300 47561 7 +3556 50129 14 +7639 50579 12 +22326 6032 10 +42593 9389 13 +13868 56568 10 +5695 15035 12 +29284 29647 7 +14777 31074 16 +28011 44721 16 +24106 43712 5 +19017 59423 12 +40484 55865 11 +10089 52665 11 +52575 30089 2 +16504 22022 10 +2920 39408 3 +5837 11276 15 +23588 35573 14 +33354 37433 2 +22075 17846 7 +35712 16854 9 +48992 46290 12 +6506 58252 11 +28249 5968 13 +32678 50235 2 +50977 24934 8 +16771 39376 7 +45188 33795 10 +59420 4633 4 +54690 12181 5 +58059 54922 9 +42277 47626 0 +1107 6773 2 +5086 38062 3 +43842 25479 6 +8618 59349 13 +56129 53998 9 +45466 43503 12 +14702 47241 2 +55516 48394 12 +12158 11971 3 +35821 9976 4 +27745 5736 8 +2027 39033 8 +52924 35562 6 +26174 28626 4 +49187 4704 12 +30200 55491 9 +44915 37028 15 +5245 51962 5 +28161 18418 10 +2416 42668 13 +28317 53569 8 +24832 49415 13 +10792 14331 15 +47714 5382 5 +33823 39349 3 +19395 32617 17 +9708 47330 13 +32320 308 8 +43733 29827 9 +39794 36930 11 +38971 16319 15 +8893 58220 11 +35939 1479 8 +37217 16348 7 +21473 41588 6 +16552 46524 14 +36975 53088 2 +39441 22673 3 +48786 15208 4 +18677 4642 9 +15343 48511 8 +9304 44611 11 +35413 43802 11 +39072 17366 6 +39634 7906 11 +52491 43709 9 +34026 44192 6 +31006 53369 13 +32657 42923 12 +39768 46778 12 +37602 59477 7 +35050 13765 4 +5272 6119 6 +5303 5224 14 +5252 29364 3 +24278 43055 8 +15672 55015 14 +40434 15832 14 +19284 27981 13 +43825 41799 10 +47117 41581 9 +5480 56447 10 +14223 27361 12 +42420 57095 7 +11196 27238 12 +47959 7281 8 +42905 39683 6 +43969 27771 16 +48581 34079 11 +40059 569 9 +13771 17484 8 +42684 3107 4 +18271 7318 6 +12485 57474 7 +6411 33753 8 +27500 21743 9 +32900 728 10 +55051 20291 9 +46240 13762 3 +435 33151 8 +49419 54369 9 +6107 8296 5 +9167 45950 13 +46674 47018 9 +35513 5402 13 +54189 43538 13 +28978 24103 6 +28825 43730 6 +24128 35058 8 +59771 896 13 +1454 55868 8 +11028 26040 8 +39423 15531 16 +34496 6480 11 +28952 9020 13 +1721 59559 6 +50452 25855 5 +11076 34900 15 +13171 40027 10 +28164 8755 8 +28313 15731 11 +6925 24598 9 +59560 46013 13 +11340 31955 6 +30557 39089 13 +32859 50780 7 +6808 23743 17 +40437 56831 9 +36496 6389 11 +18292 45674 10 +40336 58480 7 +46090 40032 5 +34411 39211 10 +19424 43346 14 +26013 53983 9 +8708 14759 4 +49626 52236 15 +27888 43187 9 +51947 22027 5 +16310 19894 6 +12381 17491 8 +18098 59076 6 +53566 52707 10 +13779 12232 5 +52341 41740 12 +11002 9234 11 +20736 21341 9 +1090 37673 7 +45916 31410 4 +59459 14820 5 +55790 37156 12 +18294 23385 9 +53879 56347 2 +14059 40668 7 +23813 25515 9 +56070 29202 15 +17690 14501 8 +51051 10878 2 +21987 13202 12 +53350 48149 8 +16081 46758 8 +54478 72 8 +47653 17629 6 +46543 4481 10 +19107 29298 11 +23792 36459 14 +34609 58433 11 +33269 49581 11 +24596 25008 17 +4491 26586 12 +37989 8916 3 +59377 28381 13 +48479 50239 9 +53263 43548 12 +12306 21724 13 +27530 19208 12 +15049 34473 14 +15899 7287 16 +28346 21050 16 +26925 58737 2 +31177 7661 8 +59016 9626 7 +4584 22868 4 +50873 38723 15 +36498 25940 8 +50345 25344 7 +55761 58472 10 +2856 48030 13 +5352 49226 16 +1864 49174 12 +11243 18210 4 +35018 24411 14 +16810 2803 5 +160 20817 9 +23581 35921 11 +42347 1376 15 +8275 3553 3 +48765 21007 4 +23483 41443 3 +35060 18150 8 +5466 826 15 +4636 25815 10 +26640 323 15 +25764 19002 7 +54391 6976 8 +20834 29914 11 +992 48358 8 +19094 44282 4 +11585 31298 12 +54696 27677 8 +45296 39137 10 +50246 43145 16 +4657 47129 2 +1527 56217 1 +32569 30447 10 +11329 54821 15 +11571 12194 10 +536 59617 13 +59787 8238 10 +5087 17558 13 +17949 34503 11 +28961 16539 9 +12795 59656 13 +5443 2010 3 +30475 11921 4 +7688 33629 8 +13526 3962 10 +9869 21521 0 +43087 16804 16 +30907 142 7 +28716 25720 8 +11929 36799 10 +21065 58563 10 +8256 39420 9 +31051 4778 10 +754 15413 13 +25001 40350 9 +13833 48727 0 +49519 16008 10 +15392 40809 10 +232 7429 1 +48751 31519 6 +5121 38817 11 +34529 54037 13 +5641 23307 4 +39051 16209 3 +1236 4381 16 +38997 39010 11 +39358 43638 12 +49136 21803 14 +31862 57316 9 +4427 11995 3 +32134 16922 8 +8452 10657 3 +4956 4227 10 +10917 7371 6 +51395 59478 1 +41467 13076 1 +39135 54059 9 +45367 58919 6 +51680 11582 9 +20855 10389 9 +38643 43202 13 +39547 10035 13 +22352 47888 8 +27194 59555 9 +55930 43413 10 +36302 25389 7 +5646 25876 1 +36039 54651 3 +33854 43891 12 +51721 46432 8 +11816 57448 11 +59170 51106 8 +19609 12940 10 +33476 9929 14 +13081 24405 1 +49000 598 10 +37436 13074 8 +55418 58934 4 +1850 6474 15 +52080 54470 13 +25545 42352 15 +48374 36813 7 +15538 54809 6 +44482 39854 1 +56597 57083 16 +31953 46381 10 +37453 49231 7 +42182 10904 6 +42094 26633 13 +53622 23239 7 +32814 19890 3 +18750 4236 3 +1730 18690 15 +6137 50161 4 +11011 57424 6 +39775 39994 8 +53991 11385 15 +48856 58710 7 +47081 7443 10 +10842 57420 1 +4141 28366 9 +23561 48299 2 +15255 47122 9 +18489 22690 8 +18713 59845 9 +12131 33394 10 +39544 17980 13 +42638 37770 11 +20759 32960 10 +25447 41779 8 +55521 26000 4 +7359 32391 12 +12399 2945 6 +38910 48432 16 +50749 59392 4 +50343 9101 6 +18194 16075 9 +37370 9843 13 +52064 46819 7 +19305 1621 9 +50947 5530 10 +24768 20973 16 +57027 42832 8 +20907 32950 4 +10285 31923 10 +5820 51291 11 +1017 16276 6 +7378 54024 5 +9223 41191 7 +24665 3136 8 +24497 39827 17 +53786 2481 11 +55264 15944 17 +56218 57254 13 +7683 15027 13 +55206 16502 7 +4393 59796 8 +52814 2175 17 +57007 20437 5 +46010 32112 4 +38042 26491 8 +4619 20874 6 +26814 8064 11 +48825 46513 10 +3478 19960 7 +59089 51435 7 +38452 51055 14 +29004 29764 6 +15479 54994 10 +46675 21097 5 +40905 18687 8 +47280 27070 3 +43415 26484 9 +47670 5677 3 +46512 54765 3 +40512 11133 7 +335 7313 7 +51530 55576 2 +1885 18330 10 +33711 31769 4 +15991 56176 9 +44821 13595 11 +25711 3540 6 +6559 35806 6 +48702 39133 10 +46696 39858 6 +53148 5663 13 +46833 7342 8 +25255 59535 7 +11846 20298 1 +16050 45505 3 +2529 45026 11 +27491 857 9 +6120 53419 7 +57040 5113 5 +50359 15127 14 +57615 3343 9 +51854 41818 9 +19128 40240 8 +49057 38556 10 +37429 6230 7 +50061 26575 7 +27637 32453 8 +9496 56531 11 +50442 58861 8 +49723 31384 2 +25245 17823 7 +52452 2734 8 +30939 59445 6 +38760 19986 15 +16690 26369 2 +44838 17472 1 +20763 39756 17 +2176 52968 12 +58202 18582 11 +20695 20861 11 +51889 58613 4 +40584 14928 18 +49429 34812 6 +54858 2419 4 +56188 57502 9 +56171 28417 13 +3344 9989 3 +3380 37120 3 +40961 28429 13 +20017 48054 11 +28696 31499 12 +28734 18705 4 +19005 29616 13 +15141 7757 13 +4712 54080 6 +45923 35307 16 +36850 18212 5 +48886 24941 10 +40001 43053 8 +26418 52106 11 +15052 50349 12 +28821 10118 1 +40559 35155 10 +26838 2572 8 +59369 21966 9 +34565 15058 12 +37250 30675 6 +40849 23150 10 +59503 13491 8 +58299 12892 15 +47286 29349 16 +23921 24390 3 +8129 37824 9 +48086 32367 7 +59905 8795 7 +43886 25313 11 +13893 1389 15 +52887 31346 6 +35564 58623 12 +23925 57349 5 +29824 13343 17 +14546 24507 13 +42503 6067 11 +31067 32704 5 +53954 56948 12 +42035 19240 9 +54929 14127 15 +34502 38583 4 +27356 19273 3 +23688 42453 14 +25828 52184 18 +38840 2618 8 +20382 16179 10 +2343 27956 12 +38481 6820 14 +20768 59411 9 +19685 13721 9 +17518 52428 0 +59195 59604 12 +4614 10825 7 +57909 45288 3 +8196 14382 15 +48760 36672 14 +19919 56153 8 +19412 52818 12 +58149 54119 12 +56561 33067 10 +28431 39063 7 +26691 11472 8 +57034 34991 4 +31089 31506 11 +24642 45784 13 +6763 11100 6 +51646 7475 10 +57813 34160 8 +12244 55017 6 +33594 56036 8 +50821 14477 16 +46412 37181 5 +45424 14965 4 +12255 35755 17 +51146 53926 6 +50880 45905 6 +13003 37601 3 +58726 47184 7 +29050 57535 9 +22609 36710 14 +25466 47752 5 +51952 327 4 +46720 40462 8 +35677 14456 12 +50576 18589 6 +8032 29976 2 +42033 49937 8 +25838 48883 8 +58193 8959 12 +22419 8913 8 +3706 28526 14 +1354 22674 4 +5332 3513 11 +43189 28211 12 +30208 1918 15 +3207 59829 15 +21345 54638 13 +193 6328 8 +44155 11450 12 +3358 34564 10 +21811 5128 2 +10163 16019 9 +36658 34783 11 +26620 51820 4 +32917 56758 8 +58864 16599 6 +41171 9506 3 +24903 58555 9 +25005 53515 9 +10609 52860 15 +5597 41960 11 +36251 18231 2 +6503 31288 8 +36667 51591 7 +6358 47180 14 +42496 51227 7 +57510 59691 8 +32300 24602 10 +8682 14805 6 +1451 52645 13 +17133 8704 1 +32034 22657 1 +49305 36751 14 +6376 1521 10 +320 47115 5 +20182 38115 10 +6776 40966 10 +46263 21555 10 +29953 50033 12 +55080 55755 6 +6200 37101 6 +47947 42425 5 +1277 13319 8 +50606 46058 5 +25295 31991 12 +15571 26488 8 +11079 49806 8 +49182 27372 17 +19819 39772 6 +8002 12462 8 +12041 7557 1 +47299 57901 12 +10819 52308 13 +45770 12959 11 +27527 59413 7 +37187 7861 5 +4659 12253 14 +57029 23274 10 +53634 4547 14 +43666 17391 10 +20986 8824 11 +53361 31774 5 +56870 30361 5 +43151 50302 9 +35782 35404 1 +35948 29997 12 +50386 32256 18 +18111 59817 18 +18339 6786 6 +28590 41556 6 +54386 2548 13 +16813 437 7 +40694 47001 5 +8715 28819 7 +42356 1888 11 +51174 15792 14 +36934 35347 11 +23508 37614 6 +53971 27724 11 +17058 8021 1 +47021 27600 7 +27760 23993 6 +57258 43099 17 +10278 20069 7 +53209 50168 9 +25091 38184 10 +24466 25081 9 +41173 20022 6 +54843 6584 12 +47639 10116 3 +41066 31813 6 +8017 59063 7 +42025 43824 14 +13184 3113 1 +55667 42629 12 +55535 34886 16 +40533 1567 12 +29664 48741 1 +18482 24653 5 +36681 48263 8 +9196 15805 1 +26766 17929 9 +18967 16471 13 +43572 26055 6 +41755 36171 7 +29665 16956 13 +58506 14195 11 +10077 822 16 +22530 16498 7 +9809 1153 9 +32926 1995 8 +43130 5908 6 +1409 9324 6 +51890 56210 9 +48598 9977 5 +55372 53348 7 +51916 40175 12 +27336 31445 1 +4038 11182 9 +6869 7604 14 +54658 3201 6 +59982 53902 10 +3896 28765 9 +29219 38451 5 +57300 24549 12 +10666 52085 6 +50718 30046 10 +21699 32609 1 +43321 41551 15 +32690 18158 2 +20176 15629 12 +12993 15083 9 +54630 52275 12 +55341 34742 7 +37629 20536 2 +6574 8101 10 +25025 52963 6 +45097 7170 9 +49748 54202 8 +57627 42270 9 +17281 42812 8 +40058 51336 1 +44861 47046 8 +5481 47348 7 +47984 21588 7 +21831 13090 4 +17377 3905 13 +41592 19477 17 +19813 59849 5 +47407 13502 6 +53864 14271 9 +34816 30721 7 +36833 52266 7 +50130 34561 6 +45656 20906 11 +39774 15004 9 +39119 21176 3 +1584 5856 7 +43888 42284 12 +38134 9046 9 +59800 42762 10 +29114 13009 5 +51238 18205 10 +59474 22607 10 +27360 52300 10 +56947 59607 7 +814 58465 5 +29431 13414 17 +44776 13268 9 +50044 51713 17 +23250 11047 1 +47458 4148 14 +14918 40567 10 +9516 57822 3 +2356 58573 9 +17296 31315 13 +22440 29539 5 +43379 40215 15 +42981 14439 7 +16390 53693 6 +44380 20012 3 +38498 35214 9 +3258 21199 3 +3666 8203 5 +37599 26448 9 +22396 34445 12 +28389 50988 10 +24749 57414 7 +33845 17769 7 +55744 51939 11 +48965 46816 11 +9003 33007 2 +26117 46570 3 +45275 41521 8 +31518 54436 3 +35097 37755 13 +14115 34007 3 +34971 28574 13 +49228 15969 8 +47997 56543 7 +45535 53326 8 +22489 53208 9 +31013 3370 9 +35624 4438 8 +3775 31212 14 +5241 26213 4 +50316 37095 13 +6798 43608 8 +40127 34927 6 +36986 43125 14 +49085 40805 3 +2871 36566 6 +56825 16096 7 +15206 14302 7 +38557 54888 10 +29404 37301 14 +3558 44511 6 +48993 10427 8 +5525 19054 7 +50539 7721 3 +20287 2364 13 +28691 58659 4 +53935 50111 14 +7070 57220 10 +17731 27718 6 +28166 18161 8 +39932 59966 15 +52233 4273 13 +58997 3888 6 +6817 22562 10 +24169 49788 9 +51521 47641 14 +8474 12355 5 +58408 17011 6 +47170 22730 10 +48769 15585 7 +18532 56150 11 +57746 36896 5 +13861 21941 10 +55335 48617 9 +24751 59971 4 +47515 16353 7 +50732 43319 12 +46486 21040 11 +55507 43395 6 +38354 37846 6 +22765 48083 15 +42326 20781 7 +13484 26660 10 +31596 17738 3 +46622 25959 10 +2300 9138 11 +11651 2466 7 +48853 39046 8 +42943 33203 11 +30849 6162 9 +54465 57827 5 +15307 26201 10 +28770 16362 6 +4212 15401 8 +23598 16168 4 +26924 7496 2 +9019 23838 17 +378 45584 4 +48449 12890 13 +33012 36713 8 +36196 30032 11 +20225 56299 8 +36590 30816 8 +18225 9692 9 +34248 17739 14 +17177 2992 13 +37442 54053 11 +38000 32538 15 +18044 47063 4 +59387 4280 8 +36527 38030 15 +12350 2322 5 +40009 39271 11 +25639 40475 13 +49315 40084 9 +16337 45216 9 +30853 55334 8 +23918 501 14 +24753 52961 7 +19616 35468 15 +36979 33278 10 +6202 32146 9 +49359 21408 9 +52461 42173 15 +52643 15830 7 +11793 16808 10 +58831 39057 3 +26558 19420 10 +27997 23116 8 +21664 46945 11 +11064 34764 14 +5178 57064 7 +16576 17337 3 +30377 52357 12 +10251 11666 8 +42577 32605 9 +31081 19101 4 +4128 53810 10 +59261 32421 7 +43309 51508 6 +51614 18914 2 +6386 10165 11 +40348 32749 10 +6424 22050 13 +39577 3750 13 +9478 28209 5 +47749 9648 6 +21061 26842 11 +37304 46306 6 +51732 56072 12 +55140 21002 2 +22686 41292 7 +25955 17612 12 +10613 2904 9 +8857 6534 12 +57662 6069 8 +56071 31203 7 +37015 43363 10 +8398 24220 12 +34022 53140 5 +49301 26057 10 +40394 36625 7 +45329 19726 7 +39958 51159 11 +1490 49460 10 +9945 10771 13 +59061 26091 9 +25238 9289 5 +26165 2745 3 +10177 26446 11 +48668 1189 14 +44591 27572 3 +21001 13026 6 +49949 34192 4 +26806 7612 11 +54170 19936 13 +6417 26818 8 +36234 1066 16 +3860 51407 11 +22172 21942 4 +57136 47309 13 +4783 3496 7 +43721 51 4 +42499 47725 3 +11410 31143 11 +5100 57897 10 +33455 59585 7 +20381 57323 12 +57557 24004 11 +1005 40549 17 +29068 38938 5 +54772 12348 10 +9264 24455 8 +34125 32692 12 +22866 44579 13 +16354 19905 3 +48283 40453 14 +54656 30113 12 +29288 51287 2 +33246 13941 15 +55760 34728 9 +44682 21600 1 +53874 47661 8 +53815 42855 12 +44089 42546 2 +26502 5678 6 +18762 48952 15 +39938 25736 16 +2603 20692 12 +33739 36177 7 +12621 2788 5 +37428 58909 9 +30202 25109 11 +51809 58157 6 +58445 7854 12 +19147 44627 10 +55942 54901 7 +27322 52539 3 +8570 32030 12 +46539 8028 1 +48955 59685 15 +34055 37446 9 +51187 55739 6 +55845 54377 6 +5920 15346 9 +49652 59380 13 +45069 1775 4 +14576 4578 14 +57683 16042 17 +5795 27910 15 +10672 18963 7 +33681 2942 15 +37729 56701 10 +54971 32969 13 +55929 17425 9 +46073 40234 9 +16170 21793 11 +29227 54240 3 +16219 46176 14 +7102 20042 8 +44712 41880 4 +45657 10045 7 +16577 48492 13 +54052 24973 3 +56623 11185 15 +49700 31345 7 +58592 12356 7 +32759 17732 14 +3467 30948 5 +1328 47647 5 +23877 59663 11 +933 4980 13 +6225 21238 12 +29051 36689 7 +54959 26711 9 +48623 57435 5 +26531 57207 4 +29302 22425 15 +2820 29259 11 +47940 42166 11 +53731 38296 8 +7138 22214 2 +45518 57387 9 +40000 52800 16 +32265 18652 12 +9833 30803 15 +18934 37486 7 +31138 7188 1 +11766 28108 7 +15647 32459 3 +19288 9332 6 +57399 45712 6 +8972 6214 10 +9179 40938 11 +28300 25551 3 +51354 17087 8 +20545 32405 10 +56730 15514 7 +56280 41962 13 +48271 16015 14 +25576 21713 2 +43226 2936 10 +51066 45515 6 +40841 20801 2 +34647 58127 16 +59091 37232 6 +14604 50437 7 +16665 45608 1 +49654 50762 11 +51638 47609 5 +35843 24481 11 +4008 2846 2 +55848 19634 11 +22672 25264 10 +14942 44080 8 +7545 38722 9 +39416 51030 8 +45164 8688 9 +24054 26764 18 +16646 2397 11 +12439 44494 12 +40086 23649 4 +36427 49416 5 +8457 18288 12 +31674 46500 10 +20307 24216 10 +54374 28660 5 +39786 39992 3 +36861 2277 12 +1658 11449 3 +30081 22671 6 +11453 28643 8 +27837 6187 15 +53146 59017 9 +55002 2241 11 +3501 14625 16 +27676 7198 13 +47628 10952 7 +24909 48930 11 +48679 19146 10 +5322 36187 6 +20988 33842 2 +55032 6599 11 +21894 47160 14 +59002 14978 7 +49817 18446 6 +23824 52820 11 +31823 45886 7 +22046 21885 7 +9849 15648 5 +11351 49857 10 +14020 43812 12 +49803 23749 9 +34933 19050 15 +11653 10780 13 +46053 36927 15 +47692 15742 6 +9939 53616 11 +35857 13482 7 +23617 26205 10 +16011 21145 14 +59745 7304 14 +36203 9687 9 +50103 2802 3 +12975 20277 13 +47168 31408 14 +44034 11548 10 +14836 39264 10 +3898 57940 5 +958 53565 15 +52910 15724 17 +44789 19562 14 +13964 34328 14 +13951 27375 13 +51117 28651 4 +12080 20207 12 +37134 11945 3 +749 33171 11 +48885 36871 4 +25776 54559 11 +58589 13108 16 +47751 15893 6 +53008 1034 4 +9106 43955 7 +52449 31554 5 +39712 7069 16 +49425 13603 7 +45868 37321 11 +17660 23866 5 +15540 4151 11 +13869 2627 11 +31727 9715 3 +14772 22163 16 +10979 35570 11 +34172 30382 0 +21508 48181 2 +38017 25395 7 +50807 57680 12 +2095 35168 8 +7482 8845 12 +39075 11614 8 +21471 36823 16 +3534 52314 5 +42461 54283 12 +29491 42841 10 +22803 18864 8 +23742 40198 8 +49121 2493 9 +52136 16058 3 +22715 51246 9 +1835 32084 10 +35665 12454 11 +27525 59113 4 +15887 46676 8 +32132 5861 11 +32936 5371 17 +22662 48783 10 +28629 15589 8 +32400 55533 14 +8048 18356 5 +34704 14763 9 +29122 46788 10 +52310 23807 14 +3039 22307 7 +5190 22574 7 +2479 36536 7 +54297 44265 4 +5000 16112 12 +4709 20752 9 +19637 28687 12 +47240 33179 12 +40794 53438 16 +39873 2981 11 +35147 17388 12 +10471 55827 6 +7803 54631 4 +983 18059 6 +1419 26566 17 +7893 16641 16 +5453 44507 13 +10769 5024 16 +20822 16098 10 +7335 34113 11 +8815 19827 5 +9208 29860 8 +56142 449 10 +9139 24929 8 +17557 54587 10 +8674 50137 16 +14337 49627 10 +51465 30282 10 +51312 18847 7 +58932 55743 10 +9403 33262 7 +43423 7658 1 +57967 19191 5 +30132 4699 11 +3676 32006 8 +43904 12614 13 +43429 58394 9 +55000 29506 4 +43194 19747 17 +37891 26904 12 +32496 42730 7 +47339 33821 7 +58736 58701 15 +6008 41086 6 +1302 43316 8 +59014 18681 10 +56075 54424 11 +4960 39848 1 +47743 44390 7 +555 1777 10 +54910 22539 10 +22 40273 10 +46364 44143 4 +41147 27323 13 +41353 38638 8 +24870 3393 4 +7348 25292 11 +27264 29804 7 +42267 41950 9 +57986 20937 12 +52145 17422 7 +31879 58958 13 +32838 5140 12 +49346 30265 9 +794 17462 6 +9464 50532 13 +5431 26102 13 +45016 26929 11 +32360 5627 9 +37920 2899 1 +44267 7857 9 +30634 53345 5 +6296 19463 2 +51832 26577 9 +41951 38469 7 +38289 23257 9 +18259 41861 3 +33432 44522 6 +24215 31571 9 +38646 37549 15 +2817 46586 2 +29320 53563 2 +30180 35711 11 +12671 23952 3 +30066 45713 10 +36519 16229 3 +48076 8271 12 +57274 42404 10 +44927 7375 9 +22223 32618 18 +26576 19241 6 +39968 46943 6 +56576 12609 9 +45966 7951 1 +21051 10580 2 +26725 25604 8 +21362 20651 5 +14272 47859 11 +5771 14679 13 +40661 31321 6 +10445 25577 14 +31166 51681 7 +5936 34902 6 +7108 30125 14 +47460 32674 7 +20168 5126 12 +17386 44843 9 +57422 326 4 +47924 46488 2 +46908 2360 7 +9320 54400 10 +13148 15579 7 +15403 52733 10 +13322 6612 9 +9049 16912 10 +16038 46964 12 +25641 35095 7 +19615 34641 4 +39864 46105 10 +40091 56216 9 +2307 21651 8 +48997 21396 15 +54110 16784 7 +40774 22189 11 +16931 45627 6 +41854 54832 7 +20241 41339 16 +52613 17807 7 +22185 46355 12 +47534 52748 6 +50739 7493 11 +30950 32332 7 +46796 29837 2 +47107 32211 11 +34001 3410 8 +42053 13870 4 +39458 28572 10 +5001 42017 12 +41331 30814 9 +20278 26609 12 +39040 24537 3 +7513 53898 11 +54819 2887 6 +11259 31368 9 +9065 19708 6 +12929 7621 8 +19778 59925 16 +27225 29870 12 +354 56407 9 +18032 54397 9 +20932 32186 8 +23401 53447 3 +978 2211 6 +24173 52237 13 +832 3327 11 +104 36517 2 +48476 52281 5 +51891 44524 9 +26682 42883 11 +6916 4831 17 +17186 25362 17 +3216 57569 9 +36415 37298 13 +51001 2628 10 +28640 1256 11 +53885 45208 16 +5165 37155 10 +21534 42156 1 +15810 20558 12 +30552 38477 9 +58464 52935 12 +46204 12398 5 +34250 43867 7 +50324 39011 8 +28425 43004 14 +12187 9254 10 +18930 30875 6 +4380 19966 3 +3498 33906 9 +24975 32925 7 +45719 10704 13 +6109 34771 7 +35551 59928 10 +28438 38372 10 +58241 10566 7 +17015 436 11 +9262 59288 6 +33642 25318 11 +32068 51410 15 +42304 23648 8 +54840 4668 11 +26669 6297 5 +10992 18256 1 +51815 5707 2 +2067 1600 8 +31198 51474 4 +43686 37635 8 +1900 27721 4 +58101 26332 13 +36622 16197 15 +35760 45829 16 +26948 18753 15 +56258 40252 12 +30611 34544 12 +24862 49934 6 +17527 11868 9 +59426 43821 12 +26125 14811 5 +2743 315 8 +4225 54246 6 +18674 13167 14 +10308 4419 10 +43 5768 16 +32408 22615 8 +50057 2995 7 +47842 23917 4 +14111 35030 10 +18737 49754 4 +31517 6537 5 +55177 49327 7 +22453 17688 8 +43229 56708 9 +51914 12847 8 +8158 34961 2 +52911 25933 16 +4459 56042 16 +38346 30105 6 +51901 44296 2 +36686 31663 7 +5461 8405 3 +10079 54818 15 +37337 41438 3 +27051 28321 15 +10894 12986 10 +47835 51431 8 +7634 42625 4 +27734 22454 16 +46671 52178 14 +18236 43038 6 +38418 27138 10 +28531 33856 2 +38416 44864 7 +10851 13043 8 +22490 53949 7 +49773 572 3 +38601 58631 11 +38664 31562 4 +53390 42020 11 +52545 9743 8 +30545 1290 11 +11554 41495 11 +47254 33530 9 +9936 26277 8 +7641 56346 13 +17148 24321 5 +22646 12861 2 +23182 11177 10 +16785 7421 9 +56822 53577 8 +26177 6278 16 +35521 14927 12 +58814 5002 9 +25219 31010 11 +9185 12383 9 +17940 10965 11 +13304 28678 15 +2713 2622 10 +42906 54529 10 +4815 51561 4 +13241 20070 5 +35730 34946 16 +18037 59354 8 +4941 16733 10 +58397 6733 5 +21446 38158 9 +30456 27488 13 +38407 12582 6 +5159 13880 6 +23806 52147 12 +7549 48139 9 +42538 1073 6 +42527 23759 10 +14907 46895 10 +19681 25015 14 +16282 52157 16 +6414 47231 4 +31827 19689 2 +13751 42321 5 +6163 14492 14 +48455 17061 17 +56569 20386 9 +18992 51512 12 +9767 9588 14 +24164 30272 13 +6545 57496 5 +47361 41118 11 +47103 31840 18 +22015 16818 9 +58666 35535 13 +40074 47678 3 +52694 58484 11 +35875 29756 11 +16702 32923 14 +51872 48389 10 +47514 2722 12 +31841 14466 10 +35528 47988 10 +21801 21873 11 +56023 33793 9 +12552 18614 14 +2979 56234 5 +59819 47455 5 +14973 20722 14 +34925 33738 16 +26100 41825 12 +25379 13738 8 +59613 43549 0 +55694 34337 4 +38047 51091 7 +53411 28093 12 +44702 56755 8 +47549 47392 16 +22457 42181 6 +9459 50052 2 +5262 11155 4 +16820 50100 15 +25 30566 7 +44274 51552 11 +15376 51695 7 +59030 30856 8 +31118 57296 6 +22621 41220 5 +23046 36812 8 +42558 12908 13 +35500 45878 8 +21260 59850 5 +8967 13829 8 +9201 41018 12 +51179 54008 9 +36753 15082 3 +946 9792 10 +58290 19841 6 +34396 28950 5 +3704 39193 10 +48342 45508 7 +51933 27057 15 +46625 1912 9 +53038 7923 8 +25418 8335 8 +240 14956 17 +22282 58141 11 +47987 39382 17 +33754 48168 3 +6303 54726 9 +57695 57763 7 +27750 14728 14 +56232 6104 7 +5722 30990 9 +34578 9995 11 +37840 50798 13 +54563 36440 4 +28066 45426 6 +2857 1385 7 +25306 42498 12 +45896 52793 8 +20723 27132 10 +33896 9677 3 +24337 26739 12 +49300 38334 9 +5831 31860 7 +25691 22139 4 +39456 23043 11 +21108 52127 18 +6430 41627 3 +20252 3747 10 +418 4457 10 +8407 40570 9 +49983 34147 18 +6743 30386 5 +54343 55914 6 +29475 14179 9 +1988 32133 3 +11341 44428 9 +28026 37721 11 +5213 23610 8 +1177 45410 15 +29645 24916 13 +32676 10985 10 +44997 3897 9 +17751 1437 15 +18159 36942 2 +59622 54908 10 +59954 27891 9 +21376 48694 8 +28553 1342 16 +57678 13211 8 +41425 45029 1 +48594 48597 15 +32171 43681 5 +7196 31895 2 +17086 40372 16 +14028 10312 5 +30514 30857 11 +15398 9376 10 +15523 36105 8 +4371 36895 8 +23803 58750 14 +24344 39647 16 +40172 58360 5 +59699 44941 9 +23916 47187 3 +46980 7584 3 +43997 27526 8 +27268 1351 9 +33118 7594 2 +53940 8596 7 +2317 10535 6 +8262 28520 7 +31112 41472 7 +19348 52481 12 +22072 36959 17 +31386 16709 3 +58486 1488 7 +26756 5622 7 +12865 45963 12 +57469 14432 13 +40923 27188 6 +40511 37213 6 +23289 59267 0 +33964 5065 9 +19254 58437 18 +2921 55669 7 +50894 1766 9 +50433 30246 4 +47096 8420 6 +2001 27789 9 +39425 24682 16 +48397 36877 9 +5576 35654 7 +2268 42868 13 +23658 3101 8 +39495 33188 7 +14750 14738 10 +27531 54520 10 +26860 21698 10 +47362 57362 8 +27998 55365 15 +47505 37017 11 +53889 32721 4 +14931 43912 16 +49362 20944 3 +41115 12010 16 +39355 18303 14 +21802 656 2 +26955 14510 8 +16523 45138 6 +8602 37617 9 +28786 7363 3 +46566 45037 4 +52094 19500 12 +44319 5074 6 +5605 46239 5 +23352 44378 14 +6226 27636 9 +32782 8945 11 +36049 53035 9 +48806 30790 3 +31890 38918 3 +28838 57602 17 +14467 58074 3 +35805 41432 3 +36697 46704 11 +33884 1717 5 +3816 6863 11 +59148 3051 3 +44515 45283 6 +11987 22764 7 +31897 30243 7 +26075 56786 7 +15750 54849 10 +53305 59037 3 +27214 2394 11 +42662 5199 16 +9805 53304 9 +13097 58300 4 +8412 31456 13 +32903 30323 11 +50830 13832 7 +26071 15670 15 +35980 38026 14 +49091 47444 8 +1925 16107 10 +13113 8685 7 +25574 47980 8 +19885 36614 11 +52533 4031 10 +3867 50697 16 +49072 51635 7 +59210 16564 7 +57179 48028 16 +41952 28257 8 +40521 15236 8 +49850 25832 9 +31062 10218 13 +3592 4552 4 +16587 25890 11 +54855 43262 6 +3561 3766 9 +56815 48264 10 +54683 37413 6 +36211 35703 12 +36538 3669 14 +40054 10562 3 +4997 49424 7 +24618 6522 5 +3714 52980 9 +34603 37063 11 +23494 13233 2 +35057 56566 10 +27013 59698 3 +45910 31874 8 +45388 20503 15 +41207 42920 10 +45813 17218 12 +20283 26530 6 +55374 47537 14 +5133 21924 9 +45487 14257 13 +48855 47028 12 +3858 6265 12 +56235 6590 15 +14472 39954 4 +11262 29252 5 +3102 31715 9 +29101 31964 7 +30834 53895 18 +2517 47972 7 +16519 52896 5 +21291 8888 9 +44228 26555 12 +37713 32226 15 +46976 48913 5 +33367 21237 6 +27467 39533 4 +7461 35883 14 +30753 11330 9 +41980 53437 14 +6206 38250 6 +56942 22940 6 +8767 47585 7 +20865 53233 2 +43623 41366 7 +10135 37306 15 +3191 3966 9 +6950 9986 9 +16173 37362 12 +4551 21725 14 +45675 22169 5 +13489 59618 13 +3361 3907 10 +25229 9448 16 +19945 10101 11 +15519 43431 9 +42164 42635 15 +49393 38931 10 +30548 14684 7 +44663 35530 10 +7322 24451 6 +19046 44870 14 +30590 57577 10 +28642 8569 12 +35824 42256 15 +30230 33273 14 +34695 30368 4 +5456 30228 7 +49355 56417 5 +58344 13052 13 +1708 38777 10 +57338 57148 11 +47630 44010 14 +36329 5396 6 +11452 44575 12 +41875 51829 16 +7802 39467 7 +21505 4728 8 +34450 35734 8 +28746 44387 7 +51531 27257 9 +21149 24418 5 +1075 52294 7 +18825 2850 13 +33239 20815 8 +42419 927 3 +26940 67 7 +6362 31144 4 +34217 17417 6 +50790 59571 9 +54724 38283 6 +45587 42455 13 +57509 16118 8 +45257 43117 9 +11589 31770 14 +25733 7589 7 +37489 3042 4 +18282 266 6 +1272 24055 11 +27575 41758 12 +54350 55343 5 +34999 19551 5 +47596 27790 8 +57365 33122 8 +36992 31397 7 +33133 54333 8 +51897 7418 15 +7814 25427 8 +2368 3151 12 +22224 49255 6 +18909 15671 7 +18643 58596 11 +52709 34457 9 +38678 21236 12 +26282 17993 12 +9559 39223 15 +29842 9111 10 +43967 17424 10 +14511 29853 2 +22658 38142 8 +33643 54287 16 +24318 21819 10 +10103 18334 12 +2728 25407 14 +17041 14344 9 +31567 25869 4 +49164 3226 3 +44183 20725 9 +6836 35287 10 +7633 5196 2 +776 7713 9 +18999 37668 6 +31614 35340 9 +13458 55614 12 +50866 27594 11 +44638 45954 16 +15821 23904 8 +29365 4209 17 +16293 33206 9 +18464 22429 0 +10413 44295 4 +32304 43952 12 +57430 131 5 +40839 38506 16 +45773 43322 12 +44234 32929 15 +45895 14701 8 +49732 7317 4 +40825 30170 0 +3173 31714 15 +43646 29443 13 +10843 19815 8 +31094 31038 10 +36006 47398 11 +24493 20175 4 +39641 17928 4 +41450 21566 9 +53696 38054 5 +46132 55457 7 +38872 44514 12 +55454 26492 10 +56242 59297 10 +40527 6246 12 +20974 52478 10 +9806 26845 9 +33477 21744 7 +18625 13190 9 +28331 14259 5 +3866 8713 13 +28102 18217 8 +25977 1418 12 +10137 48267 3 +13992 41800 7 +10762 13539 7 +30079 23087 6 +3100 40724 5 +54624 11030 10 +10381 7336 12 +16839 26133 10 +45112 10670 4 +12556 45160 6 +31904 6353 8 +29206 37147 11 +45939 43846 8 +52965 49169 5 +56931 34540 15 +22025 25041 3 +11910 26729 9 +4190 39623 1 +54065 41264 9 +49793 19167 10 +11433 4143 7 +35675 26392 13 +8649 7062 5 +36552 19156 4 +9681 39651 14 +5897 46711 11 +25826 26060 13 +10489 45180 11 +50827 2905 4 +21918 2453 15 +40303 40283 9 +4669 25437 10 +57425 37883 8 +15153 48261 3 +59236 22602 13 +29270 55566 6 +44488 97 13 +20468 17004 10 +43512 21678 14 +6052 23935 9 +26704 7247 4 +16365 3931 16 +55655 22143 2 +50853 41064 13 +59723 52293 4 +19260 51099 16 +30162 12660 16 +36448 631 10 +54116 1980 10 +58815 53725 13 +28415 2292 13 +3954 32239 10 +59132 29081 8 +46234 1582 10 +59196 21094 6 +51823 25750 4 +13412 24438 14 +21405 7570 15 +33840 47040 12 +15500 26026 13 +47508 13700 3 +8700 3923 6 +41876 35380 15 +35056 15285 5 +15605 42438 11 +9217 47140 9 +23596 36867 8 +54245 29391 7 +44288 1174 2 +40467 55145 16 +32091 31271 10 +47105 22363 14 +12122 21131 8 +26937 10673 2 +45600 20513 16 +23273 38754 6 +35655 25009 7 +31659 6995 13 +4483 54719 10 +13360 28824 5 +13650 59330 9 +4725 35077 12 +45936 48497 6 +4759 38770 9 +30850 57956 11 +57371 47059 7 +37177 39817 6 +35602 35735 14 +43235 56602 4 +23445 32442 11 +49984 58658 9 +53908 13383 4 +10460 4848 12 +22558 54147 5 +42305 33876 11 +15426 32736 10 +5508 7897 4 +22119 19008 8 +7498 45280 9 +46469 1671 11 +32139 10235 13 +44218 4174 10 +13998 3774 15 +17195 17292 12 +54635 13800 8 +51102 5484 8 +19836 24433 5 +23413 32679 10 +12879 22899 4 +24020 30416 9 +15472 57328 7 +28994 28189 7 +47771 54125 6 +13129 8639 6 +50984 8008 15 +24619 35215 3 +7772 8341 6 +25967 38088 12 +18125 243 13 +59509 9062 11 +13499 42536 13 +16329 182 8 +37205 45839 9 +20848 28887 9 +35676 16976 15 +810 49457 14 +1345 56919 6 +14925 49006 12 +17315 17485 2 +53117 42679 8 +50887 36100 7 +16065 34702 13 +33967 5973 8 +16549 58373 5 +44664 12101 11 +22001 48840 12 +13975 49478 13 +39515 4019 15 +50428 1635 11 +28261 16786 11 +31646 55970 15 +47194 28206 9 +16527 27092 2 +47417 783 8 +13585 42313 10 +5206 23676 12 +54735 36546 10 +40802 11841 3 +33706 44371 3 +30899 28032 12 +32147 23129 16 +19509 30112 12 +59306 26443 15 +4830 21579 12 +44946 23910 10 +9512 26722 6 +32943 47506 13 +21494 58291 14 +14528 28931 12 +29222 55212 6 +51602 36361 11 +5344 15410 7 +31667 15148 4 +56716 29134 17 +39331 6668 18 +18921 37300 5 +7534 23334 14 +34807 45364 10 +31928 8502 9 +11649 30482 9 +17587 50236 1 +28088 9214 16 +42801 52813 0 +36237 48009 8 +36529 24510 8 +9130 41123 4 +58007 20341 15 +30667 57321 1 +41885 6852 9 +422 54139 16 +2152 15520 6 +46571 17861 9 +44641 38651 18 +965 38454 1 +1760 32725 14 +40167 13651 16 +56550 3203 8 +32053 57531 13 +112 1976 5 +53059 37579 10 +5465 3675 8 +24253 19599 8 +25196 4872 2 +48143 15219 13 +57142 5914 4 +54862 8435 10 +14695 51762 9 +51380 27248 8 +1375 42428 6 +1216 47485 14 +12770 3908 11 +18329 35242 13 +44196 42029 16 +65 2386 12 +22473 39157 1 +42614 3837 4 +22632 6525 8 +32711 24286 11 +42254 22156 8 +45263 37977 5 +53684 165 6 +13255 19529 12 +30316 56246 9 +33461 53176 10 +32096 2186 6 +31922 27339 15 +11600 48697 3 +31469 30900 8 +30453 8204 11 +34762 29366 4 +18860 15006 15 +7293 47975 6 +44449 25806 7 +18936 12910 9 +9946 45532 7 +38005 25460 9 +12860 21218 11 +30339 16355 7 +7829 1071 11 +21888 28945 4 +29678 48513 9 +3597 39132 11 +9409 49600 7 +11479 26248 14 +39749 12060 8 +16267 14434 10 +31962 43708 4 +59102 19924 1 +36991 24603 12 +41518 27110 10 +8811 23265 4 +13879 40614 5 +37536 24855 11 +7879 51488 10 +46315 5201 18 +57110 39087 13 +44337 50148 9 +15185 26139 2 +24366 22735 2 +34092 35593 12 +9975 11594 8 +38846 1867 7 +43277 5477 8 +33692 2450 8 +51923 46597 9 +24660 40140 6 +28221 24513 2 +37146 5784 14 +52084 14812 2 +556 11277 7 +33573 28595 13 +21391 8585 10 +26440 35386 10 +28134 44895 10 +47274 57047 17 +55679 47512 8 +19494 15902 11 +49254 27498 10 +15990 53689 5 +45531 51576 10 +4108 34924 6 +43168 35658 10 +671 4230 3 +40105 46455 12 +6722 20310 14 +49454 38415 3 +3448 26959 8 +9726 770 14 +4599 55380 3 +27071 5972 9 +15796 41258 7 +31779 35978 12 +59510 10803 1 +23644 32108 8 +42203 4240 7 +393 58321 13 +26021 54183 13 +47445 25456 9 +49192 17498 6 +32204 15896 7 +11654 50489 1 +5994 545 9 +38904 45983 6 +8054 51092 6 +18977 3362 5 +48792 35308 14 +2296 26355 8 +30615 36721 7 +12102 32688 7 +51915 31805 6 +14894 21435 7 +49406 33350 3 +863 21623 4 +43759 5867 5 +45788 9863 10 +13571 32842 8 +27066 47807 15 +36341 4766 6 +37392 58983 7 +24325 39583 13 +45623 31983 13 +4531 2570 12 +6539 48874 10 +1292 13923 10 +37741 20495 10 +58972 28463 16 +23556 31632 7 +37483 22599 13 +57291 59432 7 +45394 48391 6 +20630 22142 14 +48482 59640 9 +49746 55873 3 +13512 28965 7 +49999 25907 10 +9678 23882 1 +21580 59054 6 +43978 14498 4 +36068 38865 6 +24817 15156 9 +34934 35128 9 +53671 33013 8 +52061 25302 10 +40674 55903 10 +13042 14584 8 +2084 53472 6 +34617 58454 1 +10578 21202 6 +49555 20564 7 +31468 51899 16 +27346 16204 5 +6040 27847 6 +2426 37518 2 +47326 45994 5 +47261 42231 18 +8445 30240 1 +2616 50243 5 +14306 455 10 +32412 27065 9 +48128 8116 0 +24902 7351 14 +32857 4742 12 +8403 8340 10 +46064 42444 9 +56661 25060 12 +45127 44059 13 +57133 14310 9 +25816 48755 7 +12447 19866 5 +1683 16766 15 +36906 2406 13 +48555 51922 10 +58045 45571 10 +54595 59270 8 +2974 51229 7 +44661 22129 8 +1036 53555 15 +53514 39224 3 +51458 38425 3 +37788 59635 11 +16993 41284 7 +51522 24013 4 +57959 208 7 +3252 40471 7 +55988 2697 10 +25482 26921 17 +13959 6155 11 +23388 9739 10 +32368 11958 12 +53156 35012 3 +49456 37268 15 +47933 26999 9 +40052 28499 4 +47902 39852 3 +27471 1435 11 +23233 31883 10 +47901 51144 8 +32309 26034 13 +21012 51114 6 +32162 21683 2 +42296 59892 13 +6904 26394 9 +2975 33702 4 +11377 23953 4 +14246 21854 13 +36421 38259 9 +56008 46944 3 +5848 53328 6 +4651 58695 8 +27627 49645 8 +36194 26759 15 +642 45201 11 +9635 27492 6 +58502 51685 2 +9746 7838 5 +43927 54036 10 +37585 56396 12 +15104 56940 8 +36754 18609 15 +38633 34662 6 +59224 37336 14 +56364 1829 7 +20169 47730 12 +56541 40594 7 +12837 7686 7 +57211 55337 9 +2927 52529 13 +40036 8482 14 +16983 7399 5 +39126 24948 11 +793 14078 14 +33938 2446 3 +32123 6174 18 +9313 29993 9 +45620 23285 11 +43903 19023 16 +18791 46784 15 +54962 39191 17 +51786 11797 5 +57976 20152 8 +20718 20928 5 +12731 39164 10 +5786 51511 12 +33869 48300 4 +12386 56572 6 +31343 5590 10 +41139 18410 8 +50838 36918 6 +33527 36394 11 +49321 15217 5 +29872 23310 8 +3820 54437 14 +35250 55971 9 +46902 45470 8 +24378 32517 10 +20368 41011 5 +50367 59384 6 +15331 50703 16 +29249 46378 8 +41575 10503 14 +23971 14618 6 +10743 39924 12 +32228 7831 7 +16882 22013 7 +50028 21035 5 +45787 15508 9 +6022 28475 16 +54668 51539 5 +394 14964 2 +26311 53672 11 +7005 41399 4 +9935 58971 11 +14380 6176 15 +4580 55543 9 +44429 34294 13 +52381 46164 5 +40785 2526 6 +21878 32580 8 +46126 14552 6 +37646 35569 10 +59134 49911 7 +54906 22515 11 +59066 30854 8 +27935 4679 4 +52287 19518 6 +212 40870 9 +39030 5110 11 +36094 27232 3 +37082 13340 7 +4268 6797 10 +2458 53050 9 +53584 26417 10 +36592 20731 3 +57145 50482 10 +6319 32932 13 +36661 4454 9 +23720 39061 9 +4918 53863 3 +58118 56510 14 +27759 32253 9 +1475 55705 11 +23026 36508 7 +31781 6721 14 +19531 8871 10 +28326 7268 12 +29995 57958 15 +44464 12576 11 +53575 4731 1 +43977 5796 10 +9114 48868 12 +51936 37111 10 +47251 23727 6 +17898 42655 5 +29987 36309 11 +7825 23883 15 +53069 52703 9 +24866 23447 13 +18802 3088 7 +20818 7093 11 +54553 14099 5 +1604 15683 8 +30727 25504 7 +1657 18312 14 +37192 16970 7 +51394 13517 13 +23332 1273 9 +32218 28763 8 +2985 17709 7 +47680 11476 7 +30354 22128 9 +3037 46910 6 +24756 53525 16 +469 2299 17 +25851 48234 17 +51965 26665 6 +52742 20256 6 +50938 52853 9 +7768 53770 15 +28638 40385 10 +52992 10668 10 +19586 40755 18 +14109 26710 9 +9992 6379 16 +27108 44552 9 +23395 54461 5 +1768 15711 6 +15109 57601 3 +20778 9980 13 +23282 50980 12 +47910 53903 4 +14857 34247 6 +29417 35352 9 +26686 59172 7 +20990 33205 9 +54482 36807 16 +3290 21222 11 +23551 3688 15 +8664 42735 4 +55444 45139 12 +48780 58113 3 +57170 30828 1 +54372 6682 8 +14443 35193 17 +53579 35492 9 +28186 1030 4 +33142 59671 1 +47541 46579 4 +11007 47476 7 +30594 9248 4 +47424 36108 4 +2898 43240 8 +1989 10901 9 +54293 34417 9 +25533 22254 2 +33243 57750 4 +57554 20605 13 +11704 43163 15 +42198 29575 5 +6778 57154 4 +39536 18472 11 +14793 13144 7 +12957 44713 10 +9656 1300 14 +28942 42944 12 +56190 59497 4 +29698 56855 6 +18923 34088 7 +41930 33217 10 +37743 49222 5 +276 32454 1 +2860 59916 17 +3817 12825 13 +33168 38747 11 +31004 48444 10 +41713 50053 9 +49829 33371 10 +32040 36743 11 +27732 56002 14 +57440 51061 1 +3701 14905 8 +51982 57187 18 +10891 35682 11 +2528 59759 4 +55122 34530 12 +300 42993 17 +15489 13263 13 +52328 35645 16 +33862 12967 12 +31008 30077 8 +35838 44021 7 +27709 52186 9 +47803 45489 10 +27024 16139 8 +22944 50933 8 +33355 38069 12 +46644 49811 5 +57580 24933 9 +41664 35815 4 +28710 4673 14 +52722 48124 15 +35580 37999 14 +57995 19714 0 +36878 11748 10 +15102 57894 10 +29370 58825 12 +29537 42772 13 +39916 12442 13 +23170 26415 9 +14873 55820 14 +48829 6294 16 +43243 8662 8 +49946 34980 4 +50155 2476 13 +58682 8532 12 +33578 5967 3 +4251 34128 11 +30978 16787 1 +46085 41364 10 +51348 6567 6 +46932 28058 12 +19677 21292 5 +22699 16969 8 +13473 49861 8 +37435 40489 7 +13891 39083 13 +38267 41658 9 +24696 46645 9 +3412 27312 9 +5274 10497 8 +6560 46585 6 +40626 19899 8 +8990 21778 15 +54978 15267 10 +14980 10384 1 +59412 12096 16 +8782 36466 12 +17051 42249 4 +4972 20796 7 +9814 35491 8 +23349 47667 11 +4762 13955 3 +34466 12945 15 +22012 4966 7 +27249 10408 13 +13807 13188 7 +21433 38765 13 +40745 54273 6 +621 26324 12 +22496 55983 7 +56498 51459 10 +55030 42551 9 +4891 14808 11 +1143 29065 7 +51690 32155 8 +44607 32347 16 +34505 45321 8 +9018 31546 2 +56038 56528 8 +11441 19170 8 +50916 13701 5 +20949 16932 7 +19049 33280 7 +23518 15983 12 +10488 8495 4 +55968 27582 9 +53539 38240 6 +33521 18497 8 +54924 14152 9 +13541 29454 10 +31207 23344 10 +58065 3613 8 +54381 9735 6 +3008 11313 16 +36599 55603 12 +46825 55089 8 +33074 29746 9 +48250 9703 7 +34239 18128 9 +39578 765 10 +56558 24814 3 +11422 17166 9 +30043 10919 1 +12826 31585 9 +3033 56336 4 +52069 22965 14 +52678 22055 12 +19 44670 18 +51197 45553 9 +34399 44443 16 +11855 45431 10 +41561 3166 8 +16722 20242 3 +58069 43700 13 +28466 16520 0 +2711 45168 15 +20709 49114 16 +31450 41722 7 +12056 54256 8 +21739 6614 5 +6371 15424 9 +45274 31251 10 +46369 14581 12 +11795 25656 6 +31100 52560 13 +26202 29962 4 +55047 55161 6 +33397 3383 9 +47501 54680 3 +26612 41362 16 +3176 15811 9 +21311 33166 7 +4179 18599 3 +6867 53303 7 +31255 24592 12 +32792 18689 10 +27231 13177 10 +13201 8338 4 +51827 48585 16 +51383 3153 12 +9152 5111 12 +23006 399 5 +59157 22645 6 +7910 20383 7 +42148 57228 11 +31782 42343 8 +53583 36507 12 +52744 9640 15 +3851 24976 5 +22601 5981 0 +24540 38328 8 +1439 54536 15 +43251 53804 7 +8149 55798 12 +37389 6562 13 +19457 4450 14 +1958 5741 14 +55413 31324 11 +47490 44606 1 +13837 44896 9 +3568 18250 6 +39091 44904 12 +46963 31712 7 +39783 21022 8 +51894 24694 13 +45614 38006 15 +3002 52701 15 +56103 16467 8 +39937 29495 10 +28393 58771 6 +6499 29669 2 +42782 21838 13 +49942 46460 8 +16233 4447 10 +46548 47295 11 +58940 16426 11 +6845 32908 16 +10025 6981 5 +35925 6946 2 +8734 11773 14 +3512 46034 16 +13703 12188 10 +1206 57562 13 +37438 45853 5 +55836 39706 5 +47864 1240 12 +38853 52352 14 +43693 46246 12 +49113 48414 1 +17510 41968 9 +29129 11222 5 +29742 42942 10 +31218 44388 12 +42013 13997 18 +4259 36947 6 +57218 2549 11 +1750 13560 9 +54808 51233 6 +35768 31252 11 +16138 49371 16 +33756 36290 9 +3181 33668 11 +3186 12878 10 +22872 20455 10 +59399 41059 14 +19074 14742 5 +28846 3982 14 +38827 56389 3 +51735 23558 8 +42995 25192 9 +34460 37141 8 +56837 21103 13 +34320 44286 16 +21298 59176 7 +44362 24777 2 +55570 48163 12 +52677 34700 8 +23871 18081 8 +5776 52042 6 +38919 12068 10 +34718 52942 4 +53177 1550 3 +11863 36369 12 +19431 31358 15 +29346 27243 15 +27291 4825 13 +12882 17117 10 +57970 23107 13 +57596 39273 14 +9210 50404 7 +48686 50002 8 +51241 46448 7 +17979 9160 12 +2130 38104 9 +6031 58870 11 +16661 57704 11 +34329 25053 5 +30966 7544 7 +36617 22043 0 +4276 21286 11 +37364 42169 12 +33949 59824 3 +58959 58626 13 +6217 18165 6 +20534 46960 9 +9514 3770 14 +25981 10353 6 +931 606 6 +36200 59572 6 +46107 45173 5 +20742 48607 2 +33204 13681 9 +7631 34665 10 +44505 1806 12 +21422 55284 17 +11477 15140 10 +6351 58584 4 +34822 43689 11 +50042 57074 7 +32292 2840 4 +41126 10162 5 +57581 25080 12 +35430 14049 6 +22362 59274 8 +27864 29696 7 +20911 20282 9 +594 59921 11 +48195 51999 4 +19481 47560 13 +20315 5426 17 +3804 41067 10 +2932 21984 2 +53019 45366 3 +21576 56476 5 +1599 2415 4 +19956 29703 10 +6650 17108 7 +12656 51176 8 +3900 32870 6 +47474 19432 8 +42922 37557 11 +6199 58984 12 +22147 514 10 +8499 24499 16 +44526 27451 9 +2000 27020 6 +24363 25957 6 +27189 1211 3 +24129 54101 2 +22249 51189 10 +8593 46688 9 +25634 12291 13 +48134 32294 12 +8623 22679 10 +46983 11380 5 +47926 18754 15 +42241 2900 13 +59110 18835 10 +22318 14175 7 +36945 26786 17 +25956 16724 12 +6099 17736 6 +854 11723 8 +21323 3929 2 +34528 35230 15 +29207 56939 9 +52781 31024 8 +7274 11284 12 +57025 18422 11 +4875 27010 9 +11075 57631 4 +22236 38388 3 +35031 34138 6 +17455 20118 13 +40272 54486 2 +20186 46171 12 +11561 57541 8 +28993 15171 9 +58974 53807 11 +44381 43112 9 +40812 10872 5 +44732 41092 14 +57777 28328 6 +17852 18911 12 +50363 23994 10 +18665 51707 7 +53112 43276 8 +9736 581 5 +46965 19648 12 +34871 34638 3 +43949 17595 6 +46127 20916 7 +46150 37705 4 +25524 33848 1 +12287 59722 11 +37087 10432 9 +5117 6044 6 +49220 34393 6 +33300 48618 18 +39367 41141 3 +10604 38136 8 +7345 39701 1 +473 8991 7 +24875 15205 10 +35407 48463 5 +48038 21174 5 +34256 58057 6 +57835 27473 8 +52371 21763 9 +50257 25633 9 +30213 45971 11 +54729 50730 11 +28232 50747 10 +20507 25715 16 +51048 21887 5 +56382 52823 11 +55014 19181 4 +44773 7986 7 +4959 20586 9 +3705 49068 6 +25058 46316 16 +8554 51927 10 +42460 14975 9 +53180 35908 2 +53450 19402 6 +16052 48600 8 +35553 18229 8 +29321 45592 11 +27367 33221 7 +39023 47884 4 +42301 38675 7 +17696 55136 11 +2265 43462 14 +17 39663 10 +52025 19991 3 +2810 52477 15 +52553 5504 16 +20171 43135 8 +663 40838 11 +51461 18214 13 +58420 48053 11 +25943 12583 9 +8934 23680 9 +32161 32778 8 +42091 26016 6 +4674 1504 9 +48892 23185 6 +21658 22273 4 +45965 18994 14 +26458 1809 7 +48584 43839 15 +7740 38586 12 +7733 57710 14 +51814 58818 13 +59990 19965 12 +41035 41144 13 +39106 18671 8 +17373 46930 8 +55558 25888 13 +57390 25336 1 +24959 1577 13 +52089 3115 13 +58734 12009 16 +19059 57246 13 +35595 46211 4 +59379 17356 14 +34884 52165 5 +8869 53481 10 +49145 7820 6 +57888 19358 9 +29313 57975 12 +28700 17433 5 +25737 42345 8 +6720 59346 11 +33920 38706 7 +11151 5071 15 +15165 41359 17 +46042 54958 15 +1002 55449 2 +56377 57368 4 +33291 38996 15 +10593 17614 15 +58073 53948 9 +51997 40222 7 +1165 26335 3 +4458 45324 13 +39175 52773 3 +51931 817 10 +21193 312 17 +23157 39870 9 +16861 33772 6 +4737 41549 11 +44147 18682 12 +27883 53175 8 +33612 23999 11 +10182 49520 10 +21903 31309 9 +35855 51039 12 +16494 12097 10 +10824 20833 5 +29242 31101 8 +26454 48945 8 +19807 51730 17 +24857 8560 12 +23474 28599 8 +19594 32055 1 +30671 59952 6 +43361 43954 11 +23094 10352 7 +2447 43263 13 +35929 12569 8 +32376 50034 9 +27154 9996 13 +27007 15882 9 +42955 17066 11 +58228 24988 7 +26688 52514 8 +20177 50278 8 +59250 15211 13 +40997 39513 14 +24556 31523 16 +51069 56678 12 +50143 29239 8 +413 34761 13 +53108 43109 11 +4587 39519 14 +32097 11931 9 +56988 17999 14 +29691 9794 16 +52273 44786 7 +32236 33275 16 +5599 46637 6 +39190 6853 12 +42678 11252 12 +59409 17067 13 +2075 3132 3 +17647 5427 13 +54292 19045 11 +36036 31498 7 +33018 39649 15 +31245 10997 5 +55475 13295 8 +3365 1891 15 +39484 46048 8 +9492 44159 1 +43412 39201 13 +56938 2365 11 +44312 33339 4 +8866 32575 7 +18956 11140 12 +59844 35901 10 +18904 42755 9 +861 53449 7 +15368 47440 3 +25808 16012 8 +17951 32626 12 +52579 59501 11 +23708 33923 7 +17944 45960 4 +34159 43048 10 +32559 23478 5 +24956 46701 18 +37551 23292 12 +20749 3434 1 +3129 30143 6 +22251 3713 6 +28561 9790 3 +8353 35360 8 +30610 20330 16 +19592 27368 16 +4854 44842 4 +54787 7039 9 +54056 46787 13 +32803 24437 10 +33335 28879 9 +46160 31381 9 +49677 25642 15 +2843 38744 1 +17210 53297 0 +9004 47067 10 +2202 59290 3 +57172 28027 9 +36500 6402 15 +51313 41812 9 +226 47510 12 +58308 2031 11 +14601 21023 9 +14281 38870 17 +2968 39154 11 +50701 28279 18 +45471 17938 2 +32433 36199 17 +21167 41146 10 +21654 47331 11 +23646 32720 11 +58180 37881 1 +20879 32874 17 +23477 25763 9 +40639 53020 12 +42850 42833 13 +26784 42314 8 +24124 26099 3 +45792 8591 4 +3004 46294 1 +41918 9343 16 +35643 18627 6 +20704 13488 9 +57538 25804 6 +29221 48062 14 +41630 5821 14 +42829 12416 9 +20205 55919 2 +6396 289 10 +8942 28719 14 +14276 32463 0 +48169 39162 8 +27859 46157 3 +14129 25480 9 +14322 2399 10 +38100 46843 4 +38311 42748 8 +30770 57461 11 +3682 37052 5 +26349 31960 12 +15066 13445 8 +55572 20073 14 +26903 59324 6 +22559 59329 11 +35019 42474 11 +55635 9140 4 +36936 58066 6 +8109 27752 10 +23851 47913 6 +35959 56726 9 +34185 31886 9 +47750 18803 6 +11839 55395 13 +51078 34154 16 +34983 32021 18 +33789 41289 1 +367 38617 8 +2841 28219 15 +11274 39742 13 +21262 21466 7 +4191 33792 7 +51362 35617 13 +46236 14071 7 +57053 57084 8 +32882 54449 18 +21624 53415 14 +57050 51763 12 +20105 32731 13 +57308 28884 11 +13579 51264 7 +43917 50610 13 +47418 35429 10 +7690 16246 17 +21714 9511 6 +2139 54360 14 +34992 45757 7 +14988 3822 10 +35752 15187 6 +41200 47767 10 +12899 43932 8 +47192 21358 3 +9702 33353 15 +33000 28487 7 +5057 46451 11 +59403 11922 13 +34734 9751 17 +42934 56206 7 +18574 25117 16 +53158 9956 11 +31564 52837 9 +16103 43493 16 +4593 22771 8 +58664 17499 11 +6756 17729 9 +44472 27280 7 +20149 58730 8 +33511 56294 12 +997 52602 2 +8183 48626 8 +3279 41203 16 +13270 21513 9 +17804 34824 17 +28551 41891 11 +14128 25369 10 +21703 38075 7 +50453 38340 2 +34134 59760 5 +207 9810 12 +36904 49789 5 +12535 57013 14 +26321 7150 9 +5643 30818 7 +1669 8080 7 +39048 56529 12 +16313 46707 6 +31762 40917 7 +52548 6839 4 +19140 48860 10 +3164 8702 18 +27566 55052 16 +764 3049 9 +41436 23590 10 +18445 53165 7 +22481 23822 8 +36759 23130 7 +39074 30385 11 +49498 37717 7 +45006 47867 1 +11090 32919 17 +30960 44102 9 +10514 41473 6 +15098 7398 13 +41901 56610 9 +9178 28697 15 +39088 31845 15 +41817 38344 18 +17364 10629 10 +20964 32577 10 +29769 19235 13 +4373 47768 8 +2111 1449 8 +20563 7508 9 +53848 18113 8 +2699 3663 17 +52750 41077 7 +29979 47747 16 +11595 12468 13 +53955 16154 12 +59888 4909 17 +5992 37687 13 +2474 56424 6 +49854 5999 18 +30350 49536 11 +17344 56922 8 +3601 34451 8 +26594 3961 17 +50473 55782 11 +53198 37885 14 +10458 55921 10 +52900 40620 11 +44436 12919 14 +52501 42641 7 +9921 50981 10 +5948 19404 10 +32554 2421 9 +43093 52049 15 +46601 56418 8 +2422 54801 10 +12088 47642 7 +31259 5360 16 +15816 8354 3 +1810 224 6 +37552 5449 7 +10898 8370 6 +57661 7233 10 +46474 35174 18 +27454 12394 2 +1404 11213 16 +27591 37284 7 +1767 26540 6 +45802 51948 16 +18796 51288 7 +42903 45462 11 +54660 32087 6 +9842 5632 7 +19593 40887 13 +42865 2147 0 +44747 19225 2 +56768 26782 3 +38013 22257 12 +15669 3005 4 +17528 59351 11 +43095 52822 7 +52652 10707 6 +607 15601 15 +45996 847 5 +26960 55031 2 +10141 52386 6 +35333 24350 6 +45157 57484 14 +48624 19388 3 +19043 17224 13 +36573 16598 2 +30302 22427 8 +15311 10639 4 +38376 56571 11 +29988 14030 11 +39751 49896 11 +4672 34395 14 +5210 14414 6 +5875 33427 3 +57900 32200 10 +38589 40607 10 +47977 4881 4 +54361 58187 14 +6481 4308 15 +24495 13109 3 +22123 32458 12 +19327 12981 10 +39045 37460 14 +37157 21247 3 +14440 45098 12 +32636 32052 13 +37622 46069 3 +44151 12784 3 +30103 12792 18 +7453 34489 17 +17578 38767 14 +13209 59233 10 +17214 50840 2 +2724 41502 13 +11897 56344 14 +55437 3906 2 +24274 32582 9 +45002 12138 8 +26943 11806 8 +50914 11353 10 +44110 35143 7 +4196 53847 12 +18741 18419 9 +59747 44618 14 +44820 9245 9 +55094 34478 12 +6058 12293 11 +11812 54786 3 +22373 41263 5 +7022 30506 5 +56483 31363 13 +27927 29186 15 +10313 14677 10 +16638 34950 3 +53324 18658 9 +18261 8489 1 +13375 57829 7 +4339 15378 8 +12260 35072 11 +34305 28816 0 +44602 27202 7 +43995 46817 14 +2783 38619 6 +44613 16779 5 +7997 51315 14 +21039 36116 5 +176 43041 13 +3490 56436 0 +42111 12702 3 +42090 48349 11 +28479 13125 5 +46885 25944 15 +33469 41736 8 +31285 58114 6 +20511 50522 17 +14393 15431 6 +34091 6312 5 +50584 42158 8 +16073 19614 9 +15885 47851 9 +24919 28100 10 +47736 42935 5 +44936 55482 12 +16958 21769 2 +32625 57193 1 +24730 15399 5 +36894 53054 5 +33750 14238 8 +12963 3221 14 +46618 28044 7 +4667 58727 10 +29745 55447 14 +49515 2913 8 +24663 45450 10 +35740 46957 7 +2132 8964 7 +57688 52917 12 +59206 7092 13 +7473 12934 10 +25393 56331 10 +31595 11370 10 +30114 28284 8 +57849 23920 6 +15744 47757 11 +19547 4944 1 +24578 41683 8 +58897 41923 4 +17571 15697 10 +49554 9197 7 +42177 39266 15 +1304 20492 3 +40490 7362 10 +6369 32098 8 +52092 47475 12 +7921 34402 2 +57450 28683 12 +41163 36311 6 +18405 22316 18 +9871 4803 11 +52663 34133 8 +24224 37060 15 +56953 39750 10 +50772 40003 1 +18740 1337 14 +25233 44516 16 +26424 23724 12 +48568 41785 12 +32277 54209 5 +16415 46083 2 +56432 741 16 +11067 22464 9 +33762 40102 18 +7224 26197 6 +34840 36603 13 +1061 37316 17 +37163 51166 10 +357 14072 7 +52343 22575 9 +9701 59033 5 +695 57182 8 +32293 9891 2 +46578 39517 16 +37353 18561 16 +10487 38784 9 +56255 26308 8 +35349 11563 5 +34439 48665 11 +40877 17914 5 +20427 38033 3 +43502 24661 9 +27469 43249 9 +41303 16113 13 +995 4533 13 +16615 34182 9 +15517 44690 15 +13727 22833 5 +58272 2363 12 +12114 30734 6 +41165 39857 4 +6593 51881 8 +48140 1515 14 +14654 23192 11 +52739 47181 12 +54469 52201 16 +41695 58492 6 +45667 38164 6 +24734 32014 8 +2738 37921 2 +46946 48527 11 +34865 45722 9 +58081 45174 18 +23615 58032 6 +13290 40087 9 +34719 35291 15 +56906 30998 16 +54943 9038 0 +49023 32440 6 +1142 28897 16 +57679 15434 5 +44162 49832 4 +5570 43915 1 +11747 53737 11 +3506 51724 10 +50377 28915 12 +41830 4390 3 +39295 27054 7 +34152 47526 5 +14350 34635 11 +35395 46757 5 +41570 1391 2 +8049 11780 7 +33108 2890 1 +16921 41553 0 +49878 17597 10 +51945 38325 14 +58382 25638 12 +13519 26887 4 +7864 22712 6 +27912 58995 8 +48706 1562 12 +23085 7123 7 +49887 2597 8 +33110 3376 1 +24149 31217 8 +36782 8034 15 +26059 59012 1 +24035 8483 9 +28600 18392 11 +22008 3228 7 +51505 6266 6 +24063 43101 5 +11219 18175 12 +1430 15889 8 +36412 51058 8 +29984 29387 15 +21382 12382 3 +37234 11740 6 +21723 33058 14 +56122 37383 7 +46739 53461 13 +44724 17413 2 +29203 52429 0 +10519 28528 4 +301 49003 16 +4137 32065 3 +37303 18729 7 +11576 13901 6 +34890 23815 14 +35993 13546 12 +15511 44260 5 +16763 1917 9 +27286 18573 13 +19304 46668 5 +36168 24581 6 +39144 13530 17 +48776 51693 10 +2509 7338 9 +25201 36700 14 +47111 49924 8 +15222 58025 14 +25147 11271 4 +8812 11395 1 +4907 59839 14 +6573 51314 8 +34263 18933 9 +2083 25133 8 +45574 50876 4 +38317 14691 4 +20989 24238 5 +27517 4996 7 +20250 13897 13 +4641 7466 4 +25030 32105 14 +48506 57266 3 +51571 24181 10 +56456 5572 10 +46337 29572 13 +44746 43010 13 +30848 53329 6 +53027 33079 0 +21084 8839 7 +16424 20101 10 +36353 26932 3 +21855 2191 8 +52710 20474 2 +55794 26803 10 +35445 24475 8 +3739 20903 7 +4376 4607 5 +40889 2072 10 +36014 37365 7 +6989 40121 4 +39644 23099 4 +27074 4507 10 +8099 9137 7 +2219 40206 11 +18029 44083 9 +46958 29453 11 +46532 6848 8 +18151 20051 8 +31891 18171 5 +39407 2813 2 +50687 19051 9 +47813 54100 16 +25729 16347 5 +34555 8880 3 +22040 573 1 +25628 47557 3 +52047 9258 2 +31625 45796 3 +24960 28758 11 +17487 11307 7 +51665 42147 16 +9263 44666 13 +48897 17567 6 +15709 43106 13 +24904 45385 2 +36935 51800 14 +37281 6072 16 +37610 6551 8 +2633 55344 16 +40741 38751 7 +22261 47815 8 +59860 5452 8 +53798 3865 14 +36892 26550 4 +6793 52476 4 +5204 24864 6 +42224 2971 8 +13606 4077 7 +11746 22647 5 +27965 18331 12 +42371 46300 6 +16477 41017 17 +15787 59574 9 +50585 16688 9 +13707 29500 5 +40014 15897 13 +17943 6618 9 +46032 36516 14 +26923 49014 0 +23240 3091 5 +9601 10936 16 +56449 32217 8 +37876 5176 8 +19558 12948 13 +50492 52082 1 +9504 35629 17 +9856 9770 6 +32189 41866 9 +43688 34527 9 +5812 37078 11 +42487 39405 5 +11008 26336 13 +57196 16832 12 +19115 28545 3 +9987 36734 1 +14656 37107 8 +38274 14937 4 +43001 13326 18 +56778 46656 2 +5739 687 15 +1091 9098 8 +7875 55126 9 +6952 11640 15 +42505 43033 4 +52978 30264 7 +5951 33430 13 +32114 36733 7 +37756 48293 5 +8411 36120 4 +39197 40827 9 +6178 26257 14 +18026 47819 6 +40194 26560 12 +56648 55233 9 +40665 9325 14 +3324 26298 7 +26574 3759 13 +12923 26225 9 +36273 41959 2 +55910 13822 5 +23244 18644 14 +30972 56309 7 +54794 22827 7 +13312 29628 6 +32976 20676 13 +12642 22867 6 +52892 36741 8 +37510 35807 9 +51836 13787 4 +3296 55160 13 +41283 51434 3 +10009 4790 3 +43862 10029 11 +33977 5325 14 +47875 48412 1 +8547 64 13 +31992 17930 7 +37609 54380 6 +58791 36376 9 +32381 5242 13 +30673 19033 7 +56029 9445 10 +26883 53703 11 +59781 25380 5 +45190 16859 9 +53049 28126 3 +41477 9773 6 +56674 16937 8 +19011 47734 2 +59687 25438 10 +19448 6136 15 +29587 11959 7 +9011 34735 5 +30882 48277 8 +17542 45970 9 +9873 8819 13 +24872 41273 6 +54211 58515 11 +43664 56987 10 +57590 46766 13 +33866 19464 11 +24991 11591 16 +18379 30149 5 +17586 46479 1 +33091 51087 9 +17306 7507 10 +36722 24610 8 +45483 12623 15 +55159 15988 17 +52938 35390 7 +25584 1916 0 +33167 9782 14 +28605 31867 12 +13176 55623 16 +20388 38648 5 +15994 35011 4 +10669 21766 1 +33332 2696 10 +53851 18237 5 +24527 37947 11 +8671 51207 17 +36123 42651 11 +53586 13849 11 +11286 47155 4 +45798 45702 14 +44688 24629 4 +6762 32488 8 +2524 11338 8 +57109 46936 2 +54262 22064 6 +21719 11316 13 +7403 30837 12 +35951 51961 6 +23189 12587 7 +10470 52802 5 +37573 9780 9 +51992 49032 5 +126 31159 10 +4136 14036 7 +42513 37591 9 +42554 40601 9 +2005 34951 13 +47478 2467 12 +48659 12164 13 +50495 9645 8 +34403 50261 2 +12955 58313 4 +15207 24788 11 +48073 34180 10 +48609 20089 14 +57438 8825 10 +55763 31146 11 +32757 4537 13 +55671 633 1 +41668 51843 11 +58422 31475 15 +45661 33514 10 +46302 6640 2 +46832 53113 14 +12177 47132 8 +49031 19626 7 +1591 17812 10 +35762 32612 9 +16699 48877 3 +3850 10338 5 +55528 58240 6 +2190 59578 10 +42953 40270 13 +25203 8706 9 +3539 10156 6 +53337 50890 15 +298 44115 11 +15405 54135 11 +29363 51621 14 +25648 44476 14 +57545 56304 18 +24121 43002 2 +44958 14562 5 +258 27450 16 +31326 936 16 +9764 58342 15 +27333 15015 3 +40368 16568 16 +1938 24476 3 +26519 52053 14 +42272 40476 11 +143 9600 4 +5797 3588 18 +32998 8246 5 +37750 54734 12 +25035 27094 6 +18985 47964 17 +24516 20703 12 +28841 15539 4 +54687 17785 18 +31223 45783 7 +46552 23308 2 +38625 369 13 +15752 47735 5 +24066 50285 12 +33621 9855 10 +30295 43947 10 +12460 15808 11 +33080 28488 4 +59044 58417 5 +15682 18780 6 +44137 45460 10 +32313 22179 1 +11579 1681 2 +6139 28864 13 +17250 21479 5 +31020 25918 1 +32975 9597 7 +58250 30916 9 +53854 59972 8 +46953 12872 3 +35325 17227 12 +26610 1310 1 +13419 35123 12 +41023 25284 8 +21404 56212 15 +21144 35228 8 +19856 21900 6 +13658 38076 10 +48130 33656 16 +5823 10147 9 +6815 56898 7 +16637 4013 9 +53188 20642 7 +46699 54654 16 +27103 2861 12 +30190 17602 15 +7712 28433 16 +34548 50765 11 +8818 42046 8 +12025 27107 16 +10420 45554 4 +14744 27558 10 +21609 18928 14 +18834 46103 8 +35104 10507 11 +46233 42815 17 +40444 16791 6 +50409 20239 7 +47152 55215 9 +30134 53947 9 +25069 30562 11 +12881 59581 11 +30137 24034 1 +23138 16823 9 +13316 46312 13 +1053 58346 9 +57520 17503 2 +532 23236 14 +59257 21321 7 +22542 8439 12 +42466 12226 8 +22883 30267 11 +47853 40548 13 +57085 38913 16 +55373 22051 6 +38850 8397 8 +5239 36048 4 +33624 44783 8 +28587 56694 15 +8851 13152 17 +31977 452 7 +57249 41340 6 +10186 44616 17 +11694 41376 0 +53125 53092 12 +2600 14653 11 +33030 3127 12 +26184 7006 9 +10552 7518 15 +5346 59540 14 +58332 58622 14 +23901 53137 16 +30088 8162 1 +47599 34011 8 +15218 40986 10 +49717 6651 4 +58524 47520 10 +22321 57619 11 +18865 41462 7 +6546 43893 11 +9904 37109 11 +23178 18743 7 +56733 11609 14 +13236 26907 6 +2609 37041 14 +40985 18668 11 +42884 43670 12 +6405 37212 10 +17275 38516 9 +11146 15740 15 +19922 23619 13 +37558 16085 5 +25630 59584 8 +1324 39897 5 +32806 543 13 +5234 16099 8 \ No newline at end of file diff --git a/examples/mnist_add/main.py b/examples/mnist_add/main.py new file mode 100644 index 0000000..025f9d4 --- /dev/null +++ b/examples/mnist_add/main.py @@ -0,0 +1,160 @@ +import argparse +import os.path as osp + +import torch +from torch import nn +from torch.optim import RMSprop, lr_scheduler + +from abl.bridge import SimpleBridge +from abl.data.evaluation import ReasoningMetric, SymbolAccuracy +from abl.learning import ABLModel, BasicNN +from abl.reasoning import GroundKB, KBBase, PrologKB, Reasoner +from abl.utils import ABLLogger, print_log + +from datasets import get_dataset +from models.nn import LeNet5 + + +class AddKB(KBBase): + def __init__(self, pseudo_label_list=list(range(10))): + super().__init__(pseudo_label_list) + + def logic_forward(self, nums): + return sum(nums) + + +class AddGroundKB(GroundKB): + def __init__(self, pseudo_label_list=list(range(10)), GKB_len_list=[2]): + super().__init__(pseudo_label_list, GKB_len_list) + + def logic_forward(self, nums): + return sum(nums) + + +def main(): + parser = argparse.ArgumentParser(description="MNIST Addition example") + parser.add_argument( + "--no-cuda", action="store_true", default=False, help="disables CUDA training" + ) + parser.add_argument( + "--epochs", + type=int, + default=1, + help="number of epochs in each learning loop iteration (default : 1)", + ) + parser.add_argument( + "--lr", type=float, default=3e-4, help="base model learning rate (default : 0.0003)" + ) + parser.add_argument("--alpha", type=float, default=0.9, help="alpha in RMSprop (default : 0.9)") + parser.add_argument( + "--batch-size", type=int, default=32, help="base model batch size (default : 32)" + ) + parser.add_argument( + "--loops", type=int, default=2, help="number of loop iterations (default : 2)" + ) + parser.add_argument( + "--segment_size", type=int, default=0.01, help="segment size (default : 0.01)" + ) + parser.add_argument("--save_interval", type=int, default=1, help="save interval (default : 1)") + parser.add_argument( + "--max-revision", + type=int, + default=-1, + help="maximum revision in reasoner (default : -1)", + ) + parser.add_argument( + "--require-more-revision", + type=int, + default=0, + help="require more revision in reasoner (default : 0)", + ) + kb_type = parser.add_mutually_exclusive_group() + kb_type.add_argument( + "--prolog", action="store_true", default=False, help="use PrologKB (default: False)" + ) + kb_type.add_argument( + "--ground", action="store_true", default=False, help="use GroundKB (default: False)" + ) + + args = parser.parse_args() + + # Build logger + print_log("Abductive Learning on the MNIST Addition example.", logger="current") + + ### Working with Data + print_log("Working with Data.", logger="current") + train_data = get_dataset(train=True, get_pseudo_label=True) + test_data = get_dataset(train=False, get_pseudo_label=True) + + ### Building the Learning Part + print_log("Building the Learning Part.", logger="current") + + # Build necessary components for BasicNN + cls = LeNet5(num_classes=10) + loss_fn = nn.CrossEntropyLoss(label_smoothing=0.2) + optimizer = RMSprop(cls.parameters(), lr=args.lr, alpha=args.alpha) + use_cuda = not args.no_cuda and torch.cuda.is_available() + device = torch.device("cuda" if use_cuda else "cpu") + scheduler = lr_scheduler.OneCycleLR( + optimizer, + max_lr=args.lr, + pct_start=0.15, + epochs=args.loops, + steps_per_epoch=int(1 / args.segment_size), + ) + + # Build BasicNN + base_model = BasicNN( + cls, + loss_fn, + optimizer, + scheduler=scheduler, + device=device, + batch_size=args.batch_size, + num_epochs=args.epochs, + ) + + # Build ABLModel + model = ABLModel(base_model) + + ### Building the Reasoning Part + print_log("Building the Reasoning Part.", logger="current") + + # Build knowledge base + if args.prolog: + kb = PrologKB(pseudo_label_list=list(range(10)), pl_file="add.pl") + elif args.ground: + kb = AddGroundKB() + else: + kb = AddKB() + + # Create reasoner + reasoner = Reasoner( + kb, max_revision=args.max_revision, require_more_revision=args.require_more_revision + ) + + ### Building Evaluation Metrics + print_log("Building Evaluation Metrics.", logger="current") + metric_list = [SymbolAccuracy(prefix="mnist_add"), ReasoningMetric(kb=kb, prefix="mnist_add")] + + ### Bridge Learning and Reasoning + print_log("Bridge Learning and Reasoning.", logger="current") + bridge = SimpleBridge(model, reasoner, metric_list) + + # Retrieve the directory of the Log file and define the directory for saving the model weights. + log_dir = ABLLogger.get_current_instance().log_dir + weights_dir = osp.join(log_dir, "weights") + + # Train and Test + bridge.train( + train_data, + loops=args.loops, + segment_size=args.segment_size, + save_interval=args.save_interval, + save_dir=weights_dir, + ) + bridge.test(test_data) + + +if __name__ == "__main__": + main() diff --git a/examples/mnist_add/main_with_model_converter.py b/examples/mnist_add/main_with_model_converter.py new file mode 100644 index 0000000..8f3cde4 --- /dev/null +++ b/examples/mnist_add/main_with_model_converter.py @@ -0,0 +1,160 @@ +import argparse +import os.path as osp + +from torch import nn +from torch.optim import RMSprop, lr_scheduler + +from lambdaLearn.Algorithm.AbductiveLearning.bridge import SimpleBridge +from lambdaLearn.Algorithm.AbductiveLearning.data.evaluation import ReasoningMetric, SymbolAccuracy +from lambdaLearn.Algorithm.AbductiveLearning.learning import ABLModel +from lambdaLearn.Algorithm.AbductiveLearning.learning.model_converter import ModelConverter +from lambdaLearn.Algorithm.AbductiveLearning.reasoning import GroundKB, KBBase, PrologKB, Reasoner +from lambdaLearn.Algorithm.AbductiveLearning.utils import ABLLogger, print_log +from lambdaLearn.Algorithm.SemiSupervised.Classification.FixMatch import FixMatch + +from datasets import get_dataset +from models.nn import LeNet5 + + +class AddKB(KBBase): + def __init__(self, pseudo_label_list=list(range(10))): + super().__init__(pseudo_label_list) + + def logic_forward(self, nums): + return sum(nums) + + +class AddGroundKB(GroundKB): + def __init__(self, pseudo_label_list=list(range(10)), GKB_len_list=[2]): + super().__init__(pseudo_label_list, GKB_len_list) + + def logic_forward(self, nums): + return sum(nums) + + +def main(): + parser = argparse.ArgumentParser(description="MNIST Addition example") + parser.add_argument( + "--no-cuda", action="store_true", default=False, help="disables CUDA training" + ) + parser.add_argument( + "--epochs", + type=int, + default=1, + help="number of epochs in each learning loop iteration (default : 1)", + ) + parser.add_argument( + "--lr", type=float, default=3e-4, help="base model learning rate (default : 0.0003)" + ) + parser.add_argument("--alpha", type=float, default=0.9, help="alpha in RMSprop (default : 0.9)") + parser.add_argument( + "--batch-size", type=int, default=32, help="base model batch size (default : 32)" + ) + parser.add_argument( + "--loops", type=int, default=2, help="number of loop iterations (default : 2)" + ) + parser.add_argument( + "--segment_size", type=int, default=0.01, help="segment size (default : 0.01)" + ) + parser.add_argument("--save_interval", type=int, default=1, help="save interval (default : 1)") + parser.add_argument( + "--max-revision", + type=int, + default=-1, + help="maximum revision in reasoner (default : -1)", + ) + parser.add_argument( + "--require-more-revision", + type=int, + default=0, + help="require more revision in reasoner (default : 0)", + ) + kb_type = parser.add_mutually_exclusive_group() + kb_type.add_argument( + "--prolog", action="store_true", default=False, help="use PrologKB (default: False)" + ) + kb_type.add_argument( + "--ground", action="store_true", default=False, help="use GroundKB (default: False)" + ) + + args = parser.parse_args() + + # Build logger + print_log("Abductive Learning on the MNIST Addition example.", logger="current") + + ### Working with Data + print_log("Working with Data.", logger="current") + train_data = get_dataset(train=True, get_pseudo_label=True) + test_data = get_dataset(train=False, get_pseudo_label=True) + + ### Building the Learning Part + print_log("Building the Learning Part.", logger="current") + + # Build necessary components for BasicNN + model = FixMatch( + network=LeNet5(), + threshold=0.95, + lambda_u=1.0, + mu=7, + T=0.5, + epoch=1, + num_it_epoch=2**20, + num_it_total=2**20, + device="cuda", + ) + + loss_fn = nn.CrossEntropyLoss(label_smoothing=0.2) + optimizer_dict = dict(optimizer=RMSprop, lr=0.0003, alpha=0.9) + scheduler_dict = dict( + scheduler=lr_scheduler.OneCycleLR, max_lr=0.0003, pct_start=0.15, total_steps=200 + ) + + converter = ModelConverter() + base_model = converter.convert_lambdalearn_to_basicnn( + model, loss_fn=loss_fn, optimizer_dict=optimizer_dict, scheduler_dict=scheduler_dict + ) + + # Build ABLModel + model = ABLModel(base_model) + + ### Building the Reasoning Part + print_log("Building the Reasoning Part.", logger="current") + + # Build knowledge base + if args.prolog: + kb = PrologKB(pseudo_label_list=list(range(10)), pl_file="add.pl") + elif args.ground: + kb = AddGroundKB() + else: + kb = AddKB() + + # Create reasoner + reasoner = Reasoner( + kb, max_revision=args.max_revision, require_more_revision=args.require_more_revision + ) + + ### Building Evaluation Metrics + print_log("Building Evaluation Metrics.", logger="current") + metric_list = [SymbolAccuracy(prefix="mnist_add"), ReasoningMetric(kb=kb, prefix="mnist_add")] + + ### Bridge Learning and Reasoning + print_log("Bridge Learning and Reasoning.", logger="current") + bridge = SimpleBridge(model, reasoner, metric_list) + + # Retrieve the directory of the Log file and define the directory for saving the model weights. + log_dir = ABLLogger.get_current_instance().log_dir + weights_dir = osp.join(log_dir, "weights") + + # Train and Test + bridge.train( + train_data, + loops=args.loops, + segment_size=args.segment_size, + save_interval=args.save_interval, + save_dir=weights_dir, + ) + bridge.test(test_data) + + +if __name__ == "__main__": + main() diff --git a/examples/mnist_add/mnist_add.ipynb b/examples/mnist_add/mnist_add.ipynb new file mode 100644 index 0000000..b5e323a --- /dev/null +++ b/examples/mnist_add/mnist_add.ipynb @@ -0,0 +1,489 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# MNIST Addition\n", + "\n", + "This notebook shows an implementation of [MNIST Addition](https://arxiv.org/abs/1805.10872). In this task, pairs of MNIST handwritten images and their sums are given, alongwith a domain knowledge base containing information on how to perform addition operations. The task is to recognize the digits of handwritten images and accurately determine their sum.\n", + "\n", + "Intuitively, we first use a machine learning model (learning part) to convert the input images to digits (we call them pseudo-labels), and then use the knowledge base (reasoning part) to calculate the sum of these digits. Since we do not have ground-truth of the digits, in Abductive Learning, the reasoning part will leverage domain knowledge and revise the initial digits yielded by the learning part through abductive reasoning. This process enables us to further update the machine learning model." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "# Import necessary libraries and modules\n", + "import os.path as osp\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import torch\n", + "import torch.nn as nn\n", + "from torch.optim import RMSprop, lr_scheduler\n", + "\n", + "from abl.bridge import SimpleBridge\n", + "from abl.data.evaluation import ReasoningMetric, SymbolAccuracy\n", + "from abl.learning import ABLModel, BasicNN\n", + "from abl.reasoning import KBBase, Reasoner\n", + "from abl.utils import ABLLogger, print_log\n", + "\n", + "from datasets import get_dataset\n", + "from models.nn import LeNet5" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Working with Data\n", + "\n", + "First, we get the training and testing datasets:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "train_data = get_dataset(train=True, get_pseudo_label=True)\n", + "test_data = get_dataset(train=False, get_pseudo_label=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`train_data` and `test_data` share identical structures: tuples with three components: X (list where each element is a list of two images), gt_pseudo_label (list where each element is a list of two digits, i.e., pseudo-labels) and Y (list where each element is the sum of the two digits). The length and structures of datasets are illustrated as follows.\n", + "\n", + "Note: ``gt_pseudo_label`` is only used to evaluate the performance of the learning part but not to train the model." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Both train_data and test_data consist of 3 components: X, gt_pseudo_label, Y\n", + "\n", + "Length of X, gt_pseudo_label, Y in train_data: 30000, 30000, 30000\n", + "Length of X, gt_pseudo_label, Y in test_data: 5000, 5000, 5000\n", + "\n", + "X is a list, with each element being a list of 2 Tensor.\n", + "gt_pseudo_label is a list, with each element being a list of 2 int.\n", + "Y is a list, with each element being a int.\n" + ] + } + ], + "source": [ + "print(f\"Both train_data and test_data consist of 3 components: X, gt_pseudo_label, Y\")\n", + "print()\n", + "train_X, train_gt_pseudo_label, train_Y = train_data\n", + "print(\n", + " f\"Length of X, gt_pseudo_label, Y in train_data: \"\n", + " + f\"{len(train_X)}, {len(train_gt_pseudo_label)}, {len(train_Y)}\"\n", + ")\n", + "test_X, test_gt_pseudo_label, test_Y = test_data\n", + "print(\n", + " f\"Length of X, gt_pseudo_label, Y in test_data: \"\n", + " + f\"{len(test_X)}, {len(test_gt_pseudo_label)}, {len(test_Y)}\"\n", + ")\n", + "print()\n", + "\n", + "X_0, gt_pseudo_label_0, Y_0 = train_X[0], train_gt_pseudo_label[0], train_Y[0]\n", + "print(\n", + " f\"X is a {type(train_X).__name__}, \"\n", + " + f\"with each element being a {type(X_0).__name__} \"\n", + " + f\"of {len(X_0)} {type(X_0[0]).__name__}.\"\n", + ")\n", + "print(\n", + " f\"gt_pseudo_label is a {type(train_gt_pseudo_label).__name__}, \"\n", + " + f\"with each element being a {type(gt_pseudo_label_0).__name__} \"\n", + " + f\"of {len(gt_pseudo_label_0)} {type(gt_pseudo_label_0[0]).__name__}.\"\n", + ")\n", + "print(f\"Y is a {type(train_Y).__name__}, \" + f\"with each element being a {type(Y_0).__name__}.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The ith element of X, gt_pseudo_label, and Y together constitute the ith data example. As an illustration, in the first data example of the training set, we have:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "X in the first data example (a list of two images):\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgMAAAD1CAYAAADNj/Z6AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAKHklEQVR4nO3dT4hV9f/H8bk22ZRDVJgbI4MKK1rZoqg2kbTIUHAXNURUK6loUSQkVNiuooJw0cboD9EfIiISCiaoRQNTWhhFiyGyFiVkY0oJ6vlufvzgy8/f+1w798694+vx2L6u957v1+7p2YH5TK9pmmYCAIi1YtQXAACMlhgAgHBiAADCiQEACCcGACCcGACAcGIAAMKJAQAIJwYAINxkvy/s9XrDvA6gD8vxwFD3Dhi9tnuHJwMAEE4MAEA4MQAA4cQAAIQTAwAQTgwAQDgxAADhxAAAhBMDABBODABAODEAAOHEAACEEwMAEE4MAEA4MQAA4cQAAIQTAwAQTgwAQDgxAADhxAAAhBMDABBODABAODEAAOHEAACEEwMAEE4MAEA4MQAA4cQAAIQTAwAQTgwAQDgxAADhxAAAhBMDABBODABAODEAAOHEAACEEwMAEE4MAEC4yVFfAADZzj333HJfvXr1El3J/2/Tpk3lvmbNmnL/66+/yv3dd98t9wMHDpR7V54MAEA4MQAA4cQAAIQTAwAQTgwAQDgxAADhxAAAhOs1TdP09cJeb9jXsuytWrWq3F966aVyv/fee8v9xIkT5f7OO++Ue9vf4dtvv13uJ0+eLPeuZmdny/3w4cND/fzloM+v61hx76DNI488Uu7PP//8El3J6CwsLJT75Zdf3un92+4dngwAQDgxAADhxAAAhBMDABBODABAODEAAOHEAACEc87AAK1YUbfV3NxcuW/YsGGQl/N/tP0djvpn2D///PNy37JlS7kvLi4O8nLG0qj/jv4N944z35o1a8r9999/L/e2M1bWrl1b7rfffnu5D8JHH31U7m3/G/fu3dtpbztnpo1zBgCAkhgAgHBiAADCiQEACCcGACCcGACAcGIAAMKJAQAI59ChJfTkk0+W+44dO4b6+eN+6FCb66+/vtzn5+eX6EpGZ9z/jk7FvePMt2fPnnJvO1DsmWeeGeTlcAoOHQIASmIAAMKJAQAIJwYAIJwYAIBwYgAAwokBAAg3OeoLSPLss8+W++LiYqf3v/nmm8v9kksuKfe2n0P94osvyn3btm3lvnLlynIHxtPMzEy5b9y4sdx/+OGHQV4OQ+DJAACEEwMAEE4MAEA4MQAA4cQAAIQTAwAQTgwAQLhe0+cvSPc7ybnsssvK/Ztvvin36enpcv/222/L/YYbbij3Y8eOlfuZoM+v61hx7xh/bd/N/fv3l/vFF19c7tddd125O4dg+NruHZ4MAEA4MQAA4cQAAIQTAwAQTgwAQDgxAADhxAAAhJsc9QWwfLzwwgvl3vazym22bdtW7gnnCMAo7Ny5s9zXrVtX7o8//ni5O0dg/HkyAADhxAAAhBMDABBODABAODEAAOHEAACEEwMAEM45A/yvDRs2lPutt97a6f0PHDhQ7gsLC53eHzi1+++/v9zvu+++Tu+/a9euTn+e0fNkAADCiQEACCcGACCcGACAcGIAAMKJAQAIJwYAIFyvaZqmrxf2esO+FoZs5cqV5X7o0KFyn5qaKvfjx4+X+0033VTu8/Pz5c7ERJ9f17Hi3jF8GzduLPc9e/aU+1lnndXp848cOVLu3333Xblv37693GdnZ0/7mvhvbfcOTwYAIJwYAIBwYgAAwokBAAgnBgAgnBgAgHBiAADCOWcgyPvvv1/umzdv7vT+b775ZrnPzMx0en+cM5Bq9erV5b6wsFDu5513Xrnv27ev3K+88spyX7Gi/u/K6enpct+/f3+5t51Rcvjw4XLHOQMAQAsxAADhxAAAhBMDABBODABAODEAAOHEAACEmxz1BTA4d911V7lv2bKl3Nt+DvXgwYPl/tBDD5U78O+cOHGi3D/77LNy//jjj8t9165dp3tJ/+WCCy4o971795b7tddeW+4PPPBAuT/33HPlTjtPBgAgnBgAgHBiAADCiQEACCcGACCcGACAcGIAAMI5Z2AZueKKK8r9tddeK/euv1d+586d5X7o0KFO7w/j6I033ij3888/v/U97rzzznI/cuRIubd9tzZv3tx6DcN07NixTnubP//8s9Ofp50nAwAQTgwAQDgxAADhxAAAhBMDABBODABAODEAAOGcMzBGrrnmmnL/+uuvy71pmk6f33aOwCuvvNLp/WE5ajtH4I477mh9j3vuuafcX3755dO6pqU2OVn/q2L79u3lvn79+nL/6aefyv2tt94qd7rzZAAAwokBAAgnBgAgnBgAgHBiAADCiQEACCcGACCccwaW0DnnnFPuTz31VLmfffbZnT7/008/Lfenn3663I8fP97p82E52rdvX7n3c87AY489Vu5tP8f/448/lvuvv/5a7mvXri33tnvTo48+Wu433nhjubfdO5544olyP3r0aLnTnScDABBODABAODEAAOHEAACEEwMAEE4MAEA4MQAA4XpN0zR9vbDXG/a1nPFmZmbKfffu3Z3e/48//ij3Sy+9tNz//vvvTp/P8PX5dR0ry/3e0fa9aTuHYGJiYuLCCy8c0NWc2j///FPuU1NTQ/38gwcPlnvbOQWvvvrqIC+HU2i7d3gyAADhxAAAhBMDABBODABAODEAAOHEAACEEwMAEM45AwO0bt26cv/+++/Lve13ip88ebLct27dWu4ffvhhuTP+nDMwfl588cXW1zz44IPlPu7/Hy0sLJT7LbfcUu4///zzIC+Hf8E5AwBASQwAQDgxAADhxAAAhBMDABBODABAODEAAOEmR30By0nbOQA7duzo9OfbvP766+XuHAFYeg8//HDray666KJyv/vuuwd1Oac0Oztb7u+99165f/DBB+X+yy+/nPY1MV48GQCAcGIAAMKJAQAIJwYAIJwYAIBwYgAAwokBAAgnBgAgXK9pmqavF/Z6w76WsTczM1Puu3fv7vT+v/32W7lfffXV5b64uNjp8xl/fX5dx4p7x8TEqlWryn39+vXlvmnTpnL/5JNPyn1ubq7cl+M/V5yetr9jTwYAIJwYAIBwYgAAwokBAAgnBgAgnBgAgHBiAADCOWfgf1x11VWtr/nqq6/KfWpqqtyPHj1a7rfddlu5f/nll+XOmW85/jz4mX7vgOXAOQMAQEkMAEA4MQAA4cQAAIQTAwAQTgwAQDgxAADhJkd9AeNienq69TVt5wi0mZ+fL3fnCAAwCp4MAEA4MQAA4cQAAIQTAwAQTgwAQDgxAADhxAAAhHPOwADNzc2V+9atW5foSgCgf54MAEA4MQAA4cQAAIQTAwAQTgwAQDgxAADhxAAAhOs1TdP09cJeb9jXArTo8+s6Vtw7YPTa7h2eDABAODEAAOHEAACEEwMAEE4MAEA4MQAA4cQAAIQTAwAQTgwAQDgxAADhxAAAhBMDABBODABAODEAAOHEAACE6zXL8RekAwAD48kAAIQTAwAQTgwAQDgxAADhxAAAhBMDABBODABAODEAAOHEAACE+w8bbeVfVjXu7QAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "gt_pseudo_label in the first data example (a list of two ground truth pseudo-labels): [7, 5]\n", + "Y in the first data example (their sum result): 12\n" + ] + } + ], + "source": [ + "X_0, gt_pseudo_label_0, Y_0 = train_X[0], train_gt_pseudo_label[0], train_Y[0]\n", + "print(f\"X in the first data example (a list of two images):\")\n", + "plt.subplot(1, 2, 1)\n", + "plt.axis(\"off\")\n", + "plt.imshow(X_0[0].squeeze(), cmap=\"gray\")\n", + "plt.subplot(1, 2, 2)\n", + "plt.axis(\"off\")\n", + "plt.imshow(X_0[1].squeeze(), cmap=\"gray\")\n", + "plt.show()\n", + "print(\n", + " f\"gt_pseudo_label in the first data example (a list of two ground truth pseudo-labels): {gt_pseudo_label_0}\"\n", + ")\n", + "print(f\"Y in the first data example (their sum result): {Y_0}\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Building the Learning Part" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To build the learning part, we need to first build a machine learning base model. We use a simple [LeNet-5 neural network](https://en.wikipedia.org/wiki/LeNet), and encapsulate it within a `BasicNN` object to create the base model. `BasicNN` is a class that encapsulates a PyTorch model, transforming it into a base model with an sklearn-style interface. " + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "cls = LeNet5(num_classes=10)\n", + "loss_fn = nn.CrossEntropyLoss(label_smoothing=0.2)\n", + "optimizer = RMSprop(cls.parameters(), lr=0.0003, alpha=0.9)\n", + "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n", + "scheduler = lr_scheduler.OneCycleLR(optimizer, max_lr=0.0003, pct_start=0.15, total_steps=200)\n", + "\n", + "base_model = BasicNN(\n", + " cls,\n", + " loss_fn,\n", + " optimizer,\n", + " scheduler=scheduler,\n", + " device=device,\n", + " batch_size=32,\n", + " num_epochs=1,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`BasicNN` offers methods like `predict` and `predict_prob`, which are used to predict the class index and the probabilities of each class for images. As shown below:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Predicted class index for a batch of 32 instances: ndarray with shape (32,)\n", + "Predicted class probabilities for a batch of 32 instances: ndarray with shape (32, 10)\n" + ] + } + ], + "source": [ + "data_instances = [torch.randn(1, 28, 28).to(device) for _ in range(32)]\n", + "pred_idx = base_model.predict(X=data_instances)\n", + "print(\n", + " f\"Predicted class index for a batch of 32 instances: \"\n", + " + f\"{type(pred_idx).__name__} with shape {pred_idx.shape}\"\n", + ")\n", + "pred_prob = base_model.predict_proba(X=data_instances)\n", + "print(\n", + " f\"Predicted class probabilities for a batch of 32 instances: \"\n", + " + f\"{type(pred_prob).__name__} with shape {pred_prob.shape}\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "However, the base model built above deals with instance-level data (i.e., individual images), and can not directly deal with example-level data (i.e., a pair of images). Therefore, we wrap the base model into `ABLModel`, which enables the learning part to train, test, and predict on example-level data." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "model = ABLModel(base_model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As an illustration, consider this example of training on example-level data using the `predict` method in `ABLModel`. In this process, the method accepts data examples as input and outputs the class labels and the probabilities of each class for all instances within these data examples." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Predicted class labels for the 100 data examples: \n", + "a list of length 100, and each element is a ndarray of shape (2,).\n", + "\n", + "Predicted class probabilities for the 100 data examples: \n", + "a list of length 100, and each element is a ndarray of shape (2, 10).\n" + ] + } + ], + "source": [ + "from abl.data.structures import ListData\n", + "\n", + "# ListData is a data structure provided by ABL-Package that can be used to organize data examples\n", + "data_examples = ListData()\n", + "# We use the first 100 data examples in the training set as an illustration\n", + "data_examples.X = train_X[:100]\n", + "data_examples.gt_pseudo_label = train_gt_pseudo_label[:100]\n", + "data_examples.Y = train_Y[:100]\n", + "\n", + "# Perform prediction on the 100 data examples\n", + "pred_label, pred_prob = model.predict(data_examples)[\"label\"], model.predict(data_examples)[\"prob\"]\n", + "print(\n", + " f\"Predicted class labels for the 100 data examples: \\n\"\n", + " + f\"a list of length {len(pred_label)}, and each element is \"\n", + " + f\"a {type(pred_label[0]).__name__} of shape {pred_label[0].shape}.\\n\"\n", + ")\n", + "print(\n", + " f\"Predicted class probabilities for the 100 data examples: \\n\"\n", + " + f\"a list of length {len(pred_prob)}, and each element is \"\n", + " + f\"a {type(pred_prob[0]).__name__} of shape {pred_prob[0].shape}.\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Building the Reasoning Part" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the reasoning part, we first build a knowledge base which contain information on how to perform addition operations. We build it by creating a subclass of `KBBase`. In the derived subclass, we initialize the `pseudo_label_list` parameter specifying list of possible pseudo-labels, and override the `logic_forward` function defining how to perform (deductive) reasoning." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "class AddKB(KBBase):\n", + " def __init__(self, pseudo_label_list=list(range(10))):\n", + " super().__init__(pseudo_label_list)\n", + "\n", + " # Implement the deduction function\n", + " def logic_forward(self, nums):\n", + " return sum(nums)\n", + "\n", + "\n", + "kb = AddKB()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The knowledge base can perform logical reasoning (both deductive reasoning and abductive reasoning). Below is an example of performing (deductive) reasoning, and users can refer to [Documentation]() for details of abductive reasoning." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Reasoning result of pseudo-labels [1, 2] is 3.\n" + ] + } + ], + "source": [ + "pseudo_labels = [1, 2]\n", + "reasoning_result = kb.logic_forward(pseudo_labels)\n", + "print(f\"Reasoning result of pseudo-labels {pseudo_labels} is {reasoning_result}.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note: In addition to building a knowledge base based on `KBBase`, we can also establish a knowledge base with a ground KB using `GroundKB`, or a knowledge base implemented based on Prolog files using `PrologKB`. The corresponding code for these implementations can be found in the `main.py` file. Those interested are encouraged to examine it for further insights." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then, we create a reasoner by instantiating the class ``Reasoner``. Due to the indeterminism of abductive reasoning, there could be multiple candidates compatible to the knowledge base. When this happens, reasoner can minimize inconsistencies between the knowledge base and pseudo-labels predicted by the learning part, and then return only one candidate that has the highest consistency." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "reasoner = Reasoner(kb)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note: During creating reasoner, the definition of \"consistency\" can be customized within the `dist_func` parameter. In the code above, we employ a consistency measurement based on confidence, which calculates the consistency between the data example and candidates based on the confidence derived from the predicted probability. In `main.py`, we provide options for utilizing other forms of consistency measurement.\n", + "\n", + "Note: Also, during process of inconsistency minimization, one can leverage [ZOOpt library](https://github.com/polixir/ZOOpt) for acceleration. Options for this are also available in `main.py`. Those interested are encouraged to explore these features." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Building Evaluation Metrics" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, we set up evaluation metrics. These metrics will be used to evaluate the model performance during training and testing. Specifically, we use `SymbolAccuracy` and `ReasoningMetric`, which are used to evaluate the accuracy of the machine learning model’s predictions and the accuracy of the final reasoning results, respectively." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "metric_list = [SymbolAccuracy(prefix=\"mnist_add\"), ReasoningMetric(kb=kb, prefix=\"mnist_add\")]" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Bridging Learning and Reasoning\n", + "\n", + "Now, the last step is to bridge the learning and reasoning part. We proceed this step by creating an instance of `SimpleBridge`." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "bridge = SimpleBridge(model, reasoner, metric_list)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Perform training and testing by invoking the `train` and `test` methods of `SimpleBridge`." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "# Build logger\n", + "print_log(\"Abductive Learning on the MNIST Addition example.\", logger=\"current\")\n", + "log_dir = ABLLogger.get_current_instance().log_dir\n", + "weights_dir = osp.join(log_dir, \"weights\")\n", + "\n", + "bridge.train(train_data, loops=2, segment_size=0.01, save_interval=1, save_dir=weights_dir)\n", + "bridge.test(test_data)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "abl", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.18" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "9c8d454494e49869a4ee4046edcac9a39ff683f7d38abf0769f648402670238e" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/mnist_add/models/nn.py b/examples/mnist_add/models/nn.py new file mode 100644 index 0000000..93f5cda --- /dev/null +++ b/examples/mnist_add/models/nn.py @@ -0,0 +1,28 @@ +from torch import nn + + +class LeNet5(nn.Module): + def __init__(self, num_classes=10, image_size=(28, 28, 1)): + super(LeNet5, self).__init__() + self.size = 16 * ((image_size[0] // 2 - 6) // 2) * ((image_size[1] // 2 - 6) // 2) + self.encoder = nn.Sequential( + nn.Conv2d(1, 6, 5), + nn.MaxPool2d(2, 2), # 6 24 24 -> 6 12 12 + nn.ReLU(True), + nn.Conv2d(6, 16, 5), # 6 12 12 -> 16 8 8 + nn.MaxPool2d(2, 2), # 16 8 8 -> 16 4 4 + nn.ReLU(True), + ) + self.classifier = nn.Sequential( + nn.Linear(self.size, 120), + nn.ReLU(), + nn.Linear(120, 84), + nn.ReLU(), + nn.Linear(84, num_classes), + ) + + def forward(self, x): + x = self.encoder(x) + x = x.view(-1, self.size) + x = self.classifier(x) + return x diff --git a/examples/mnist_add/requirements.txt b/examples/mnist_add/requirements.txt new file mode 100644 index 0000000..24fb7af --- /dev/null +++ b/examples/mnist_add/requirements.txt @@ -0,0 +1,2 @@ +abl +matplotlib \ No newline at end of file diff --git a/examples/mnist_add/weights/all_weights_here.txt b/examples/mnist_add/weights/all_weights_here.txt new file mode 100644 index 0000000..e69de29 diff --git a/examples/zoo/README.md b/examples/zoo/README.md new file mode 100644 index 0000000..3b8c4e0 --- /dev/null +++ b/examples/zoo/README.md @@ -0,0 +1,23 @@ +# Zoo Example + +This example shows a simple implementation of [Zoo](https://archive.ics.uci.edu/dataset/111/zoo). In this task, attributes of animals (such as presence of hair, eggs, etc.) and their targets (the animal class they belong to) are given, along with a knowledge base which contain information about the relations between attributes and targets, e.g., Implies(milk == 1, mammal == 1). The goal of this task is to develop a learning model that can predict the targets of animals based on their attributes. + +## Run + +```bash +pip install -r requirements.txt +python main.py +``` + +## Usage + +```bash +usage: main.py [-h] [--loops LOOPS] + +Zoo example + +optional arguments: + -h, --help show this help message and exit + --loops LOOPS number of loop iterations (default : 3) + +``` diff --git a/examples/zoo/get_dataset.py b/examples/zoo/get_dataset.py new file mode 100644 index 0000000..18c10fd --- /dev/null +++ b/examples/zoo/get_dataset.py @@ -0,0 +1,33 @@ +import numpy as np +import openml + + +# Function to load and preprocess the dataset +def load_and_preprocess_dataset(dataset_id): + dataset = openml.datasets.get_dataset( + dataset_id, download_data=True, download_qualities=False, download_features_meta_data=False + ) + X, y, _, attribute_names = dataset.get_data(target=dataset.default_target_attribute) + # Convert data types + for col in X.select_dtypes(include="bool").columns: + X[col] = X[col].astype(int) + y = y.cat.codes.astype(int) + X, y = X.to_numpy(), y.to_numpy() + return X, y + + +# Function to split data (one shot) +def split_dataset(X, y, test_size=0.3): + # For every class: 1 : (1-test_size)*(len-1) : test_size*(len-1) + label_indices, unlabel_indices, test_indices = [], [], [] + for class_label in np.unique(y): + idxs = np.where(y == class_label)[0] + np.random.shuffle(idxs) + n_train_unlabel = int((1 - test_size) * (len(idxs) - 1)) + label_indices.append(idxs[0]) + unlabel_indices.extend(idxs[1 : 1 + n_train_unlabel]) + test_indices.extend(idxs[1 + n_train_unlabel :]) + X_label, y_label = X[label_indices], y[label_indices] + X_unlabel, y_unlabel = X[unlabel_indices], y[unlabel_indices] + X_test, y_test = X[test_indices], y[test_indices] + return X_label, y_label, X_unlabel, y_unlabel, X_test, y_test diff --git a/examples/zoo/kb.py b/examples/zoo/kb.py new file mode 100644 index 0000000..86b1886 --- /dev/null +++ b/examples/zoo/kb.py @@ -0,0 +1,93 @@ +import openml +from z3 import If, Implies, Int, Not, Solver, Sum, sat # noqa: F401 + +from abl.reasoning import KBBase + + +class ZooKB(KBBase): + def __init__(self): + super().__init__(pseudo_label_list=list(range(7)), use_cache=False) + + self.solver = Solver() + + # Load information of Zoo dataset + dataset = openml.datasets.get_dataset( + dataset_id=62, + download_data=False, + download_qualities=False, + download_features_meta_data=False, + ) + X, y, categorical_indicator, attribute_names = dataset.get_data( + target=dataset.default_target_attribute + ) + self.attribute_names = attribute_names + self.target_names = y.cat.categories.tolist() + # print("Attribute names are: ", self.attribute_names) + # print("Target names are: ", self.target_names) + # self.attribute_names = ["hair", "feathers", "eggs", "milk", "airborne", "aquatic", "predator", "toothed", "backbone", "breathes", "venomous", "fins", "legs", "tail", "domestic", "catsize"] # noqa: E501 + # self.target_names = ["mammal", "bird", "reptile", "fish", "amphibian", "insect", "invertebrate"] # noqa: E501 + + # Define variables + for name in self.attribute_names + self.target_names: + exec( + f"globals()['{name}'] = Int('{name}')" + ) # or use dict to create var and modify rules + # Define rules + rules = [ + Implies(milk == 1, mammal == 1), + Implies(mammal == 1, milk == 1), + Implies(mammal == 1, backbone == 1), + Implies(mammal == 1, breathes == 1), + Implies(feathers == 1, bird == 1), + Implies(bird == 1, feathers == 1), + Implies(bird == 1, eggs == 1), + Implies(bird == 1, backbone == 1), + Implies(bird == 1, breathes == 1), + Implies(bird == 1, legs == 2), + Implies(bird == 1, tail == 1), + Implies(reptile == 1, backbone == 1), + Implies(reptile == 1, breathes == 1), + Implies(reptile == 1, tail == 1), + Implies(fish == 1, aquatic == 1), + Implies(fish == 1, toothed == 1), + Implies(fish == 1, backbone == 1), + Implies(fish == 1, Not(breathes == 1)), + Implies(fish == 1, fins == 1), + Implies(fish == 1, legs == 0), + Implies(fish == 1, tail == 1), + Implies(amphibian == 1, eggs == 1), + Implies(amphibian == 1, aquatic == 1), + Implies(amphibian == 1, backbone == 1), + Implies(amphibian == 1, breathes == 1), + Implies(amphibian == 1, legs == 4), + Implies(insect == 1, eggs == 1), + Implies(insect == 1, Not(backbone == 1)), + Implies(insect == 1, legs == 6), + Implies(invertebrate == 1, Not(backbone == 1)), + ] + # Define weights and sum of violated weights + self.weights = {rule: 1 for rule in rules} + self.total_violation_weight = Sum( + [If(Not(rule), self.weights[rule], 0) for rule in self.weights] + ) + + def logic_forward(self, pseudo_label, data_point): + attribute_names, target_names = self.attribute_names, self.target_names + solver = self.solver + total_violation_weight = self.total_violation_weight + pseudo_label, data_point = pseudo_label[0], data_point[0] + + self.solver.reset() + for name, value in zip(attribute_names, data_point): + solver.add(eval(f"{name} == {value}")) + for cate, name in zip(self.pseudo_label_list, target_names): + value = 1 if (cate == pseudo_label) else 0 + solver.add(eval(f"{name} == {value}")) + + if solver.check() == sat: + model = solver.model() + total_weight = model.evaluate(total_violation_weight) + return total_weight.as_long() + else: + # No solution found + return 1e10 diff --git a/examples/zoo/main.py b/examples/zoo/main.py new file mode 100644 index 0000000..093976d --- /dev/null +++ b/examples/zoo/main.py @@ -0,0 +1,92 @@ +import argparse +import os.path as osp + +import numpy as np +from sklearn.ensemble import RandomForestClassifier + +from abl.bridge import SimpleBridge +from abl.data.evaluation import ReasoningMetric, SymbolAccuracy +from abl.learning import ABLModel +from abl.reasoning import Reasoner +from abl.utils import ABLLogger, confidence_dist, print_log, tab_data_to_tuple + +from get_dataset import load_and_preprocess_dataset, split_dataset +from kb import ZooKB + + +def consitency(data_example, candidates, candidate_idxs, reasoning_results): + pred_prob = data_example.pred_prob + model_scores = confidence_dist(pred_prob, candidate_idxs) + rule_scores = np.array(reasoning_results) + scores = model_scores + rule_scores + return scores + + +def main(): + parser = argparse.ArgumentParser(description="Zoo example") + parser.add_argument( + "--loops", type=int, default=3, help="number of loop iterations (default : 3)" + ) + args = parser.parse_args() + + # Build logger + print_log("Abductive Learning on the ZOO example.", logger="current") + + ### Working with Data + print_log("Working with Data.", logger="current") + + X, y = load_and_preprocess_dataset(dataset_id=62) + X_label, y_label, X_unlabel, y_unlabel, X_test, y_test = split_dataset(X, y, test_size=0.3) + label_data = tab_data_to_tuple(X_label, y_label) + test_data = tab_data_to_tuple(X_test, y_test) + train_data = tab_data_to_tuple(X_unlabel, y_unlabel) + + ### Building the Learning Part + print_log("Building the Learning Part.", logger="current") + + # Build base model + base_model = RandomForestClassifier() + + # Build ABLModel + model = ABLModel(base_model) + + ### Building the Reasoning Part + print_log("Building the Reasoning Part.", logger="current") + + # Build knowledge base + kb = ZooKB() + + # Create reasoner + reasoner = Reasoner(kb, dist_func=consitency) + + ### Building Evaluation Metrics + print_log("Building Evaluation Metrics.", logger="current") + metric_list = [SymbolAccuracy(prefix="zoo"), ReasoningMetric(kb=kb, prefix="zoo")] + + ### Bridging learning and reasoning + print_log("Bridge Learning and Reasoning.", logger="current") + bridge = SimpleBridge(model, reasoner, metric_list) + + # Retrieve the directory of the Log file and define the directory for saving the model weights. + log_dir = ABLLogger.get_current_instance().log_dir + weights_dir = osp.join(log_dir, "weights") + + # Performing training and testing + print_log("------- Use labeled data to pretrain the model -----------", logger="current") + base_model.fit(X_label, y_label) + print_log("------- Test the initial model -----------", logger="current") + bridge.test(test_data) + print_log("------- Use ABL to train the model -----------", logger="current") + bridge.train( + train_data=train_data, + label_data=label_data, + loops=args.loops, + segment_size=len(X_unlabel), + save_dir=weights_dir, + ) + print_log("------- Test the final model -----------", logger="current") + bridge.test(test_data) + + +if __name__ == "__main__": + main() diff --git a/examples/zoo/requirements.txt b/examples/zoo/requirements.txt new file mode 100644 index 0000000..2f73c5b --- /dev/null +++ b/examples/zoo/requirements.txt @@ -0,0 +1,4 @@ +abl +z3-solver +openml +scikit-learn \ No newline at end of file diff --git a/examples/zoo/zoo.ipynb b/examples/zoo/zoo.ipynb new file mode 100644 index 0000000..b7effea --- /dev/null +++ b/examples/zoo/zoo.ipynb @@ -0,0 +1,388 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Zoo\n", + "\n", + "This notebook shows an implementation of [Zoo](https://archive.ics.uci.edu/dataset/111/zoo). In this task, attributes of animals (such as presence of hair, eggs, etc.) and their targets (the animal class they belong to) are given, along with a knowledge base which contain information about the relations between attributes and targets, e.g., Implies(milk == 1, mammal == 1). \n", + "\n", + "The goal of this task is to develop a learning model that can predict the targets of animals based on their attributes. In the initial stages, when the model is under-trained, it may produce incorrect predictions that conflict with the relations contained in the knowledge base. When this happens, abductive reasoning can be employed to adjust these results and retrain the model accordingly. This process enables us to further update the learning model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Import necessary libraries and modules\n", + "import os.path as osp\n", + "\n", + "import numpy as np\n", + "from sklearn.ensemble import RandomForestClassifier\n", + "\n", + "from abl.bridge import SimpleBridge\n", + "from abl.data.evaluation import ReasoningMetric, SymbolAccuracy\n", + "from abl.learning import ABLModel\n", + "from abl.reasoning import Reasoner\n", + "from abl.utils import ABLLogger, confidence_dist, print_log, tab_data_to_tuple\n", + "\n", + "from get_dataset import load_and_preprocess_dataset, split_dataset\n", + "from kb import ZooKB" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Working with Data\n", + "\n", + "First, we load and preprocess the [Zoo dataset](https://archive.ics.uci.edu/dataset/111/zoo), and split it into labeled/unlabeled/test data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "X, y = load_and_preprocess_dataset(dataset_id=62)\n", + "X_label, y_label, X_unlabel, y_unlabel, X_test, y_test = split_dataset(X, y, test_size=0.3)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Zoo dataset consist of tabular data. The attributes contains 17 boolean values (e.g., hair, feathers, eggs, milk, airborne, aquatic, etc.) and the target is a integer value in range [0,6] representing 7 classes (e.g., mammal, bird, reptile, fish, amphibian, insect, and other). Below is an illustration:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Shape of X and y: (101, 16) (101,)\n", + "First five elements of X:\n", + "[[True False False True False False True True True True False False 4\n", + " False False True]\n", + " [True False False True False False False True True True False False 4\n", + " True False True]\n", + " [False False True False False True True True True False False True 0\n", + " True False False]\n", + " [True False False True False False True True True True False False 4\n", + " False False True]\n", + " [True False False True False False True True True True False False 4\n", + " True False True]]\n", + "First five elements of y:\n", + "[0 0 3 0 0]\n" + ] + } + ], + "source": [ + "print(\"Shape of X and y:\", X.shape, y.shape)\n", + "print(\"First five elements of X:\")\n", + "print(X[:5])\n", + "print(\"First five elements of y:\")\n", + "print(y[:5])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, we transform the tabular data to the format required by ABL-Package, which is a tuple of (X, gt_pseudo_label, Y). In this task, we treat the attributes as X and the targets as gt_pseudo_label (ground truth pseudo-labels). Y (reasoning results) are expected to be 0, indicating no rules are violated." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "label_data = tab_data_to_tuple(X_label, y_label, reasoning_result=0)\n", + "test_data = tab_data_to_tuple(X_test, y_test, reasoning_result=0)\n", + "train_data = tab_data_to_tuple(X_unlabel, y_unlabel, reasoning_result=0)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Building the Learning Part" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To build the learning part, we need to first build a machine learning base model. We use a [Random Forest](https://en.wikipedia.org/wiki/Random_forest) as the base model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "base_model = RandomForestClassifier()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "However, the base model built above deals with instance-level data, and can not directly deal with example-level data. Therefore, we wrap the base model into `ABLModel`, which enables the learning part to train, test, and predict on example-level data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model = ABLModel(base_model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Building the Reasoning Part" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the reasoning part, we first build a knowledge base which contains information about the relations between attributes (X) and targets (pseudo-labels), e.g., Implies(milk == 1, mammal == 1). The knowledge base is built in the `ZooKB` class within file `kb.py`, and is derived from the `KBBase` class." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "kb = ZooKB()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As mentioned, for all attributes and targets in the dataset, the reasoning results are expected to be 0 since there should be no violations of the established knowledge in real data. As shown below:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Example 0: the attributes are: [True False False True False False True True True True False False 4 False\n", + " False True], and the target is 0.\n", + "Reasoning result is 0.\n", + "\n", + "Example 1: the attributes are: [True False False True False False False True True True False False 4 True\n", + " False True], and the target is 0.\n", + "Reasoning result is 0.\n", + "\n", + "Example 2: the attributes are: [False False True False False True True True True False False True 0 True\n", + " False False], and the target is 3.\n", + "Reasoning result is 0.\n", + "\n", + "Example 3: the attributes are: [True False False True False False True True True True False False 4 False\n", + " False True], and the target is 0.\n", + "Reasoning result is 0.\n", + "\n", + "Example 4: the attributes are: [True False False True False False True True True True False False 4 True\n", + " False True], and the target is 0.\n", + "Reasoning result is 0.\n", + "\n" + ] + } + ], + "source": [ + "for idx, (x, y_item) in enumerate(zip(X[:5], y[:5])):\n", + " print(f\"Example {idx}: the attributes are: {x}, and the target is {y_item}.\")\n", + " print(f\"Reasoning result is {kb.logic_forward([y_item], [x])}.\")\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then, we create a reasoner by instantiating the class ``Reasoner``. Due to the indeterminism of abductive reasoning, there could be multiple candidates compatible to the knowledge base. When this happens, reasoner can minimize inconsistencies between the knowledge base and pseudo-labels predicted by the learning part, and then return only one candidate that has the highest consistency." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def consitency(data_example, candidates, candidate_idxs, reasoning_results):\n", + " pred_prob = data_example.pred_prob\n", + " model_scores = confidence_dist(pred_prob, candidate_idxs)\n", + " rule_scores = np.array(reasoning_results)\n", + " scores = model_scores + rule_scores\n", + " return scores\n", + "\n", + "\n", + "reasoner = Reasoner(kb, dist_func=consitency)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Building Evaluation Metrics" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, we set up evaluation metrics. These metrics will be used to evaluate the model performance during training and testing. Specifically, we use `SymbolAccuracy` and `ReasoningMetric`, which are used to evaluate the accuracy of the machine learning model’s predictions and the accuracy of the final reasoning results, respectively." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metric_list = [SymbolAccuracy(prefix=\"zoo\"), ReasoningMetric(kb=kb, prefix=\"zoo\")]" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Bridging Learning and Reasoning\n", + "\n", + "Now, the last step is to bridge the learning and reasoning part. We proceed this step by creating an instance of `SimpleBridge`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "bridge = SimpleBridge(model, reasoner, metric_list)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Perform training and testing by invoking the `train` and `test` methods of `SimpleBridge`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "12/22 11:48:01 - abl - INFO - Abductive Learning on the ZOO example.\n", + "12/22 11:48:01 - abl - INFO - ------- Use labeled data to pretrain the model -----------\n", + "12/22 11:48:01 - abl - INFO - ------- Test the initial model -----------\n", + "12/22 11:48:01 - abl - INFO - Evaluation ended, zoo/character_accuracy: 0.903 zoo/reasoning_accuracy: 0.903 \n", + "12/22 11:48:01 - abl - INFO - ------- Use ABL to train the model -----------\n", + "12/22 11:48:01 - abl - INFO - loop(train) [1/3] segment(train) [1/1] \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "12/22 11:48:02 - abl - INFO - Evaluation start: loop(val) [1]\n", + "12/22 11:48:03 - abl - INFO - Evaluation ended, zoo/character_accuracy: 1.000 zoo/reasoning_accuracy: 1.000 \n", + "12/22 11:48:03 - abl - INFO - loop(train) [2/3] segment(train) [1/1] \n", + "12/22 11:48:04 - abl - INFO - Evaluation start: loop(val) [2]\n", + "12/22 11:48:05 - abl - INFO - Evaluation ended, zoo/character_accuracy: 1.000 zoo/reasoning_accuracy: 1.000 \n", + "12/22 11:48:05 - abl - INFO - loop(train) [3/3] segment(train) [1/1] \n", + "12/22 11:48:05 - abl - INFO - Evaluation start: loop(val) [3]\n", + "12/22 11:48:06 - abl - INFO - Evaluation ended, zoo/character_accuracy: 1.000 zoo/reasoning_accuracy: 1.000 \n", + "12/22 11:48:06 - abl - INFO - ------- Test the final model -----------\n", + "12/22 11:48:06 - abl - INFO - Evaluation ended, zoo/character_accuracy: 0.968 zoo/reasoning_accuracy: 0.968 \n" + ] + } + ], + "source": [ + "# Build logger\n", + "print_log(\"Abductive Learning on the Zoo example.\", logger=\"current\")\n", + "log_dir = ABLLogger.get_current_instance().log_dir\n", + "weights_dir = osp.join(log_dir, \"weights\")\n", + "\n", + "print_log(\"------- Use labeled data to pretrain the model -----------\", logger=\"current\")\n", + "base_model.fit(X_label, y_label)\n", + "print_log(\"------- Test the initial model -----------\", logger=\"current\")\n", + "bridge.test(test_data)\n", + "print_log(\"------- Use ABL to train the model -----------\", logger=\"current\")\n", + "bridge.train(\n", + " train_data=train_data,\n", + " label_data=label_data,\n", + " loops=3,\n", + " segment_size=len(X_unlabel),\n", + " save_dir=weights_dir,\n", + ")\n", + "print_log(\"------- Test the final model -----------\", logger=\"current\")\n", + "bridge.test(test_data)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We may see from the results, after undergoing training with ABL, the model's accuracy has improved." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "abl", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.18" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "9c8d454494e49869a4ee4046edcac9a39ff683f7d38abf0769f648402670238e" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/framework.py b/framework.py deleted file mode 100644 index b41a334..0000000 --- a/framework.py +++ /dev/null @@ -1,155 +0,0 @@ -# coding: utf-8 -#================================================================# -# Copyright (C) 2021 Freecss All rights reserved. -# -# File Name ：framework.py -# Author ：freecss -# Email ：karlfreecss@gmail.com -# Created Date ：2021/06/07 -# Description ： -# -#================================================================# - -import pickle as pk - -import numpy as np - -from utils.plog import INFO, DEBUG, clocker - -@clocker -def block_sample(X_bak, Y_bak, C_bak, sample_num, epoch_idx): - part_num = (len(X_bak) // sample_num) - if part_num == 0: - part_num = 1 - seg_idx = epoch_idx % part_num - INFO("seg_idx:", seg_idx, ", part num:", part_num, ", data num:", len(X_bak)) - X = X_bak[sample_num * seg_idx: sample_num * (seg_idx + 1)] - Y = Y_bak[sample_num * seg_idx: sample_num * (seg_idx + 1)] - C = C_bak[sample_num * seg_idx: sample_num * (seg_idx + 1)] - - return X, Y, C - -def get_taglist(self, Y): - tmp = [[str(x) for x in label] for label in Y] - tmp = sorted(list(set(tmp))) - return tmp - -@clocker -def result_statistics(pseudo_Y, Y, abduced_Y): - - abd_err_num = 0 - abd_char_num = 0 - abd_char_acc = 0 - abd_failed = 0 - word_err_num = 0 - - ori_char_num = 0 - ori_char_acc = 0 - - for tidx, (pseudo_y, y, abduced_y) in enumerate(zip(pseudo_Y, Y, abduced_Y)): - pseudo_y = pseudo_y - if sum(abduced_y != y) != 0: - abd_err_num += 1 - if abduced_y is not None: - abd_char_num += len(y) - abd_char_acc += sum(abduced_y == y) - else: - abd_failed += 1 - - ori_char_num += len(pseudo_y) - ori_char_acc += sum(pseudo_y == y) - - if abduced_y is not None and sum(y != pseudo_y) == 0 and sum(pseudo_y != abduced_y) > 0: - INFO(pseudo_y, y, abduced_y) - pk.dump((pseudo_y, y, abduced_y), open("bug.pk", "wb")) - - if sum(pseudo_y != y) != 0: - word_err_num += 1 - - INFO("") - INFO("Abd word level accuracy:", 1 - word_err_num / len(pseudo_Y)) - INFO("Abd char level accuracy:", abd_char_acc / abd_char_num) - INFO("Ori char level accuracy:", ori_char_acc / ori_char_num) - INFO("") - - result = {"total_word" : len(pseudo_Y), "accuracy_word" : len(pseudo_Y) - word_err_num, - "total_abd_char": abd_char_num, "accuracy_abd_char" : abd_char_acc, - "total_ori_char": ori_char_num, "accuracy_ori_char" : ori_char_acc, - "total_abd_failed": abd_failed} - - return result - -@clocker -def filter_data(X, abduced_Y): - finetune_Y = [] - finetune_X = [] - for abduced_x, abduced_y in zip(X, abduced_Y): - if abduced_y is not None: - finetune_X.append(abduced_x) - finetune_Y.append(abduced_y) - return finetune_X, finetune_Y - -@clocker -def is_all_sublabel_exist(labels, std_label_list): - if not labels: - return False - - labels = np.array(labels).T - for idx, (std_label, label) in enumerate(zip(std_label_list, labels)): - std_num = len(set(std_label)) - sublabel_num = len(set(label)) - if std_num != sublabel_num: - INFO(f"sublabel {idx} should have {std_num} class, but data only have {sublabel_num} class", screen=True) - return False - return True - -def pretrain(model, X, Y): - pass - -def train(model, abducer, X, Y, C = None, epochs = 10, sample_num = -1, verbose = -1, check_sublabel = True): - # Set default parameters - if sample_num == -1: - sample_num = len(X) - - if verbose < 1: - verbose = epochs - - if C is None: - C = [None] * len(X) - - # Set function running time recorder - valid_func = clocker(model.valid) - predict_func = clocker(model.predict) - train_func = clocker(model.train) - - abduce_func = clocker(abducer.batch_abduce) - - X_bak = X - Y_bak = Y - C_bak = C - - # Abductive learning train process - res = {} - for epoch_idx in range(epochs): - X, Y, C = block_sample(X_bak, Y_bak, C_bak, sample_num, epoch_idx) - preds_res = predict_func(X) - abduced_Y = abduce_func(preds_res, C) - finetune_X, finetune_Y = filter_data(X, abduced_Y) - score, score_list = valid_func(X, Y) - if ((epoch_idx + 1) % verbose == 0) or (epoch_idx == epochs - 1): - res = result_statistics(preds_res["cls"], Y, abduced_Y) - INFO(res) - - if check_sublabel and (not is_all_sublabel_exist(finetune_Y, model.label_lists)): - INFO("There is some sub label missing", len(finetune_Y)) - break - - if len(finetune_X) > 0: - train_func(finetune_X, finetune_Y)#, n_epoch = 10) - else: - INFO("lack of data, all abduced failed", len(finetune_X)) - return res - #return ret - -if __name__ == "__main__": - pass diff --git a/models/basic_model.py b/models/basic_model.py deleted file mode 100644 index c54bb22..0000000 --- a/models/basic_model.py +++ /dev/null @@ -1,362 +0,0 @@ -# coding: utf-8 -#================================================================# -# Copyright (C) 2020 Freecss All rights reserved. -# -# File Name ：basic_model.py -# Author ：freecss -# Email ：karlfreecss@gmail.com -# Created Date ：2020/11/21 -# Description ： -# -#================================================================# - -import sys -sys.path.append("..") - -import torch -from torch.autograd import Variable -from torch.utils.data import Dataset -import torchvision - -import utils.utils as mutils - -import os -from multiprocessing import Pool - -import random -import torch -from torch.utils.data import Dataset -from torch.utils.data import sampler -import torchvision.transforms as transforms -import six -import sys -from PIL import Image -import numpy as np -import collections - -class resizeNormalize(object): - - def __init__(self, size, interpolation=Image.BILINEAR): - self.size = size - self.interpolation = interpolation - self.toTensor = transforms.ToTensor() - self.transform = transforms.Compose([ - #transforms.ToPILImage(), - #transforms.RandomHorizontalFlip(), - #transforms.RandomVerticalFlip(), - #transforms.RandomRotation(30), - #transforms.RandomAffine(30), - transforms.ToTensor(), - - ]) - - def __call__(self, img): - #img = img.resize(self.size, self.interpolation) - #img = self.toTensor(img) - img = self.transform(img) - img.sub_(0.5).div_(0.5) - return img - -class XYDataset(Dataset): - def __init__(self, X, Y, transform=None, target_transform=None): - self.X = X - self.Y = Y - - self.n_sample = len(X) - self.transform = transform - self.target_transform = target_transform - - def __len__(self): - return len(self.X) - - def __getitem__(self, index): - assert index < len(self), 'index range error' - - img = self.X[index] - if self.transform is not None: - img = self.transform(img) - - label = self.Y[index] - if self.target_transform is not None: - label = self.target_transform(label) - - return (img, label, index) - -class alignCollate(object): - - def __init__(self, imgH=32, imgW=100, keep_ratio=False, min_ratio=1): - self.imgH = imgH - self.imgW = imgW - self.keep_ratio = keep_ratio - self.min_ratio = min_ratio - - def __call__(self, batch): - images, labels, img_keys = zip(*batch) - - imgH = self.imgH - imgW = self.imgW - if self.keep_ratio: - ratios = [] - for image in images: - w, h = image.shape[:2] - ratios.append(w / float(h)) - ratios.sort() - max_ratio = ratios[-1] - imgW = int(np.floor(max_ratio * imgH)) - imgW = max(imgH * self.min_ratio, imgW) # assure imgH >= imgW - - transform = resizeNormalize((imgW, imgH)) - images = [transform(image) for image in images] - images = torch.cat([t.unsqueeze(0) for t in images], 0) - labels = torch.LongTensor(labels) - - return images, labels, img_keys - -class FakeRecorder(): - def __init__(self): - pass - - def print(self, *x): - pass - -from torch.nn import init -from torch import nn -def weigth_init(m): - if isinstance(m, nn.Conv2d): - init.xavier_uniform_(m.weight.data) - init.constant_(m.bias.data,0.1) - elif isinstance(m, nn.BatchNorm2d): - m.weight.data.fill_(1) - m.bias.data.zero_() - elif isinstance(m, nn.Linear): - m.weight.data.normal_(0,0.01) - m.bias.data.zero_() - -class BasicModel(): - def __init__(self, - model, - criterion, - optimizer, - converter, - device, - params, - sign_list, - recorder = None): - - self.model = model.to(device) - self.model.apply(weigth_init) - self.criterion = criterion - self.optimizer = optimizer - self.converter = converter - self.device = device - sign_list = sorted(list(set(sign_list))) - self.mapping = dict(zip(sign_list, list(range(len(sign_list))))) - self.remapping = dict(zip(list(range(len(sign_list))), sign_list)) - - if recorder is None: - recorder = FakeRecorder() - self.recorder = recorder - - self.save_interval = params.saveInterval - self.params = params - pass - - def _fit(self, data_loader, n_epoch, stop_loss): - recorder = self.recorder - recorder.print("model fitting") - - min_loss = 999999999 - for epoch in range(n_epoch): - loss_value = self.train_epoch(data_loader) - recorder.print(f"{epoch}/{n_epoch} model training loss is {loss_value}") - if loss_value < min_loss: - min_loss = loss_value - if loss_value < stop_loss: - break - recorder.print("Model fitted, minimal loss is ", min_loss) - return loss_value - - def str2ints(self, Y): - return [self.mapping[y] for y in Y] - - def fit(self, data_loader = None, - X = None, - y = None, - n_epoch = 100, - stop_loss = 0.001): - if data_loader is None: - params = self.params - Y = self.str2ints(y) - train_dataset = XYDataset(X, Y) - sampler = None - data_loader = torch.utils.data.DataLoader(train_dataset, batch_size=params.batchSize, \ - shuffle=True, sampler=sampler, num_workers=int(params.workers), \ - collate_fn=alignCollate(imgH=params.imgH, imgW=params.imgW, keep_ratio=params.keep_ratio)) - return self._fit(data_loader, n_epoch, stop_loss) - - def train_epoch(self, data_loader): - loss_avg = mutils.averager() - - for i, data in enumerate(data_loader): - X = data[0] - Y = data[1] - cost = self.train_batch(X, Y) - loss_avg.add(cost) - - loss_value = float(loss_avg.val()) - loss_avg.reset() - return loss_value - - def train_batch(self, X, Y): - #cpu_images, cpu_texts, _ = data - model = self.model - criterion = self.criterion - optimizer = self.optimizer - converter = self.converter - device = self.device - - # set training mode - for p in model.parameters(): - p.requires_grad = True - model.train() - - # init training status - torch.autograd.set_detect_anomaly(True) - optimizer.zero_grad() - - # model predict - X = X.to(device) - Y = Y.to(device) - pred_Y = model(X) - - # calculate loss - loss = criterion(pred_Y, Y) - - # back propagation and optimize - loss.backward() - optimizer.step() - return loss - - def _predict(self, data_loader): - model = self.model - criterion = self.criterion - converter = self.converter - params = self.params - device = self.device - - for p in model.parameters(): - p.requires_grad = False - - model.eval() - - n_correct = 0 - - results = [] - for i, data in enumerate(data_loader): - X = data[0].to(device) - pred_Y = model(X) - results.append(pred_Y) - - return torch.cat(results, axis=0) - - def predict(self, data_loader = None, X = None, print_prefix = ""): - params = self.params - if data_loader is None: - Y = [0] * len(X) - val_dataset = XYDataset(X, Y) - sampler = None - data_loader = torch.utils.data.DataLoader(val_dataset, batch_size=params.batchSize, \ - shuffle=False, sampler=sampler, num_workers=int(params.workers), \ - collate_fn=alignCollate(imgH=params.imgH, imgW=params.imgW, keep_ratio=params.keep_ratio)) - - recorder = self.recorder - recorder.print('Start Predict ', print_prefix) - Y = self._predict(data_loader).argmax(axis=1) - return [self.remapping[int(y)] for y in Y] - - def predict_proba(self, data_loader = None, X = None, print_prefix = ""): - params = self.params - if data_loader is None: - Y = [0] * len(X) - val_dataset = XYDataset(X, Y) - sampler = None - data_loader = torch.utils.data.DataLoader(val_dataset, batch_size=params.batchSize, \ - shuffle=False, sampler=sampler, num_workers=int(params.workers), \ - collate_fn=alignCollate(imgH=params.imgH, imgW=params.imgW, keep_ratio=params.keep_ratio)) - - recorder = self.recorder - recorder.print('Start Predict ', print_prefix) - return torch.softmax(self._predict(data_loader), axis=1) - - def _val(self, data_loader, print_prefix): - model = self.model - criterion = self.criterion - recorder = self.recorder - converter = self.converter - params = self.params - device = self.device - recorder.print('Start val ', print_prefix) - - for p in model.parameters(): - p.requires_grad = False - - model.eval() - - n_correct = 0 - pred_num = 0 - loss_avg = mutils.averager() - for i, data in enumerate(data_loader): - X = data[0].to(device) - Y = data[1].to(device) - - pred_Y = model(X) - - correct_num = sum(Y == pred_Y.argmax(axis=1)) - loss = criterion(pred_Y, Y) - loss_avg.add(loss) - - n_correct += correct_num - pred_num += len(X) - - accuracy = float(n_correct) / float(pred_num) - recorder.print('[%s] Val loss: %f, accuray: %f' % (print_prefix, loss_avg.val(), accuracy)) - return accuracy - - def val(self, data_loader = None, X = None, y = None, print_prefix = ""): - params = self.params - if data_loader is None: - y = self.str2ints(y) - val_dataset = XYDataset(X, y) - sampler = None - data_loader = torch.utils.data.DataLoader(val_dataset, batch_size=params.batchSize, \ - shuffle=True, sampler=sampler, num_workers=int(params.workers), \ - collate_fn=alignCollate(imgH=params.imgH, imgW=params.imgW, keep_ratio=params.keep_ratio)) - return self._val(data_loader, print_prefix) - - def score(self, data_loader = None, X = None, y = None, print_prefix = ""): - return self.val(data_loader, X, y, print_prefix) - - def save(self, save_dir): - recorder = self.recorder - if not os.path.exists(save_dir): - os.mkdir(save_dir) - recorder.print("Saving model and opter") - save_path = os.path.join(save_dir, "net.pth") - torch.save(self.model.state_dict(), save_path) - - save_path = os.path.join(save_dir, "opt.pth") - torch.save(self.optimizer.state_dict(), save_path) - - def load(self, load_dir): - recorder = self.recorder - recorder.print("Loading model and opter") - load_path = os.path.join(load_dir, "net.pth") - self.model.load_state_dict(torch.load(load_path)) - - load_path = os.path.join(load_dir, "opt.pth") - self.optimizer.load_state_dict(torch.load(load_path)) - -if __name__ == "__main__": - pass - - diff --git a/models/lenet5.py b/models/lenet5.py deleted file mode 100644 index 676f553..0000000 --- a/models/lenet5.py +++ /dev/null @@ -1,96 +0,0 @@ -# coding: utf-8 -#================================================================# -# Copyright (C) 2021 Freecss All rights reserved. -# -# File Name ：lenet5.py -# Author ：freecss -# Email ：karlfreecss@gmail.com -# Created Date ：2021/03/03 -# Description ： -# -#================================================================# - -import sys -sys.path.append("..") - -import torchvision - -import torch -from torch import nn -from torch.nn import functional as F -from torch.autograd import Variable -import torchvision.transforms as transforms - -from models.basic_model import BasicModel - -import utils.plog as plog - -class LeNet5(nn.Module): - def __init__(self): - super().__init__() - self.conv1 = nn.Conv2d(1, 6, 3, padding=1) - self.conv2 = nn.Conv2d(6, 16, 3) - self.conv3 = nn.Conv2d(16, 16, 3) - - self.fc1 = nn.Linear(256, 120) - self.fc2 = nn.Linear(120, 84) - self.fc3 = nn.Linear(84, 13) - - def forward(self, x): - '''前向传播函数''' - x = F.max_pool2d(F.relu(self.conv1(x)), (2, 2)) - x = F.max_pool2d(F.relu(self.conv2(x)), (2, 2)) - x = F.relu(self.conv3(x)) - x = x.view(-1, self.num_flat_features(x)) - #print(x.size()) - x = F.relu(self.fc1(x)) - x = F.relu(self.fc2(x)) - x = self.fc3(x) - return x - - def num_flat_features(self, x): - #x.size()返回值为(256, 16, 5, 5)，size的值为(16, 5, 5)，256是batch_size - size = x.size()[1:] #x.size返回的是一个元组，size表示截取元组中第二个开始的数字 - num_features = 1 - for s in size: - num_features *= s - return num_features - -class Params: - imgH = 28 - imgW = 28 - keep_ratio = True - saveInterval = 10 - batchSize = 16 - num_workers = 16 - -def get_data(): #数据预处理 - transform = transforms.Compose([transforms.ToTensor(), - transforms.Normalize((0.5), (0.5))]) - #transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) - #训练集 - train_set = torchvision.datasets.MNIST(root='data/', train=True, transform=transform, download=True) - train_loader = torch.utils.data.DataLoader(train_set, batch_size=1024, shuffle=True, num_workers = 16) - #测试集 - test_set = torchvision.datasets.MNIST(root='data/', train=False, transform=transform, download=True) - test_loader = torch.utils.data.DataLoader(test_set, batch_size = 1024, shuffle = False, num_workers = 16) - classes = ('plane','car','bird','cat','deer','dog','frog','horse','ship','truck') - - return train_loader, test_loader, classes - -if __name__ == "__main__": - recorder = plog.ResultRecorder() - cls = LeNet5() - criterion = nn.CrossEntropyLoss(size_average=True) - optimizer = torch.optim.Adam(cls.parameters(), lr=0.001, betas=(0.9, 0.99)) - device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") - model = BasicModel(cls, criterion, optimizer, None, device, Params(), recorder) - - train_loader, test_loader, classes = get_data() - - #model.val(test_loader, print_prefix = "before training") - model.fit(train_loader, n_epoch = 100) - model.val(test_loader, print_prefix = "after trained") - res = model.predict(test_loader, print_prefix = "predict") - print(res.argmax(axis=1)[:10]) - diff --git a/models/wabl_models.py b/models/wabl_models.py deleted file mode 100644 index 4b27418..0000000 --- a/models/wabl_models.py +++ /dev/null @@ -1,189 +0,0 @@ -# coding: utf-8 -#================================================================# -# Copyright (C) 2020 Freecss All rights reserved. -# -# File Name ：models.py -# Author ：freecss -# Email ：karlfreecss@gmail.com -# Created Date ：2020/04/02 -# Description ： -# -#================================================================# -from itertools import chain - -from sklearn.tree import DecisionTreeClassifier -from sklearn.model_selection import cross_val_score - -from sklearn.svm import LinearSVC - -from sklearn.pipeline import make_pipeline -from sklearn.preprocessing import StandardScaler -from sklearn.svm import SVC -from sklearn.gaussian_process import GaussianProcessClassifier -from sklearn.gaussian_process.kernels import RBF - -import pickle as pk -import random - -from sklearn.neighbors import KNeighborsClassifier -import numpy as np - -def get_part_data(X, i): - return list(map(lambda x : x[i], X)) - -def merge_data(X): - ret_mark = list(map(lambda x : len(x), X)) - ret_X = list(chain(*X)) - return ret_X, ret_mark - - -def reshape_data(Y, marks): - begin_mark = 0 - ret_Y = [] - for mark in marks: - end_mark = begin_mark + mark - ret_Y.append(Y[begin_mark:end_mark]) - begin_mark = end_mark - return ret_Y - - -class WABLBasicModel: - """ - label_lists 的目标在于为各个符号设置编号，无论方法是给出字典形式的概率还是给出list形式的，都可以通过这种方式解决. - 后续可能会考虑更加完善的措施，降低这部分的复杂度 - 当模型共享的时候，label_lists 之间的元素也是共享的 - """ - - def __init__(self): - pass - - def predict(self, X): - if self.share: - data_X, marks = merge_data(X) - prob = self.cls_list[0].predict_proba(X = data_X) - cls = np.array(prob).argmax(axis = 1) - - prob = reshape_data(prob, marks) - cls = reshape_data(cls, marks) - else: - cls_result = [] - prob_result = [] - for i in range(self.code_len): - data_X = get_part_data(X, i) - tmp_prob = self.cls_list[i].predict_proba(X = data_X) - cls_result.append(np.array(tmp_prob).argmax(axis = 1)) - prob_result.append(tmp_prob) - - cls = list(zip(*cls_result)) - prob = list(zip(*prob_result)) - - return {"cls" : cls, "prob" : prob} - - def valid(self, X, Y): - if self.share: - data_X, _ = merge_data(X) - data_Y, _ = merge_data(Y) - score = self.cls_list[0].score(X = data_X, y = data_Y) - return score, [score] - else: - score_list = [] - for i in range(self.code_len): - data_X = get_part_data(X, i) - data_Y = get_part_data(Y, i) - score_list.append(self.cls_list[i].score(data_X, data_Y)) - - return sum(score_list) / len(score_list), score_list - - def train(self, X, Y): - #self.label_lists = [] - if self.share: - data_X, _ = merge_data(X) - data_Y, _ = merge_data(Y) - self.cls_list[0].fit(X = data_X, y = data_Y) - else: - for i in range(self.code_len): - data_X = get_part_data(X, i) - data_Y = get_part_data(Y, i) - self.cls_list[i].fit(data_X, data_Y) - - def _set_label_lists(self, label_lists): - label_lists = [sorted(list(set(label_list))) for label_list in label_lists] - self.label_lists = label_lists - -class DecisionTree(WABLBasicModel): - def __init__(self, code_len, label_lists, share = False): - self.code_len = code_len - self._set_label_lists(label_lists) - - self.cls_list = [] - self.share = share - if share: - # 本质上是同一个分类器 - self.cls_list.append(DecisionTreeClassifier(random_state = 0, min_samples_leaf = 3)) - self.cls_list = self.cls_list * self.code_len - else: - for _ in range(code_len): - self.cls_list.append(DecisionTreeClassifier(random_state = 0, min_samples_leaf = 3)) - -class KNN(WABLBasicModel): - def __init__(self, code_len, label_lists, share = False, k = 3): - self.code_len = code_len - self._set_label_lists(label_lists) - - self.cls_list = [] - self.share = share - if share: - # 本质上是同一个分类器 - self.cls_list.append(KNeighborsClassifier(n_neighbors = k)) - self.cls_list = self.cls_list * self.code_len - else: - for _ in range(code_len): - self.cls_list.append(KNeighborsClassifier(n_neighbors = k)) - -class CNN(WABLBasicModel): - def __init__(self, base_model, code_len, label_lists, share = True): - assert share == True, "Not implemented" - - label_lists = [sorted(list(set(label_list))) for label_list in label_lists] - self.label_lists = label_lists - - self.code_len = code_len - - self.cls_list = [] - self.share = share - if share: - self.cls_list.append(base_model) - - def train(self, X, Y, n_epoch = 100): - #self.label_lists = [] - if self.share: - # 因为是同一个分类器，所以只需要把数据放在一起，然后训练其中任意一个即可 - data_X, _ = merge_data(X) - data_Y, _ = merge_data(Y) - self.cls_list[0].fit(X = data_X, y = data_Y, n_epoch = n_epoch) - #self.label_lists = [sorted(list(set(data_Y)))] * self.code_len - else: - for i in range(self.code_len): - data_X = get_part_data(X, i) - data_Y = get_part_data(Y, i) - self.cls_list[i].fit(data_X, data_Y) - #self.label_lists.append(sorted(list(set(data_Y)))) - - -if __name__ == "__main__": - #data_path = "utils/hamming_data/generated_data/hamming_7_3_0.20.pk" - data_path = "datasets/generated_data/0_code_7_2_0.00.pk" - codes, data, labels = pk.load(open(data_path, "rb")) - - cls = KNN(7, False, k = 3) - cls.train(data, labels) - print(cls.valid(data, labels)) - for res in cls.predict_proba(data): - print(res) - break - - for res in cls.predict(data): - print(res) - break - print("Trained") - diff --git a/nonshare_example.py b/nonshare_example.py deleted file mode 100644 index dbc2b44..0000000 --- a/nonshare_example.py +++ /dev/null @@ -1,97 +0,0 @@ -# coding: utf-8 -#================================================================# -# Copyright (C) 2021 Freecss All rights reserved. -# -# File Name ：nonshare_example.py -# Author ：freecss -# Email ：karlfreecss@gmail.com -# Created Date ：2021/06/07 -# Description ： -# -#================================================================# - -from utils.plog import logger -from models.wabl_models import DecisionTree, KNN -import pickle as pk -import numpy as np -import time -import framework - -from multiprocessing import Pool -import os -from datasets.data_generator import generate_data_via_codes, code_generator -from collections import defaultdict -from abducer.abducer_base import AbducerBase -from abducer.kb import ClsKB, RegKB - -def run_test(params): - code_len, times, code_num, share, model_type, need_prob, letter_num = params - - if share: - result_dir = "share_result" - else: - result_dir = "non_share_result" - - recoder_file_path = f"{result_dir}/random_{times}_{code_len}_{code_num}_{model_type}_{need_prob}.pk" - - words = code_generator(code_len, code_num, letter_num) - kb = ClsKB(words) - abducer = AbducerBase(kb, dist_func = "confidence", pred_res_parse = lambda x : x["prob"]) - - label_lists = [[] for _ in range(code_len)] - for widx, word in enumerate(words): - for cidx, c in enumerate(word): - label_lists[cidx].append(c) - - if share: - label_lists = [sum(label_lists, [])] - - recoder = logger() - recoder.set_savefile("test.log") - for idx, err in enumerate(range(15, 41)): - start = time.process_time() - err = err / 40. - if 1 - err < (1. / letter_num): - break - - print("Start expriment", idx) - if model_type == "KNN": - model = KNN(code_len, label_lists = label_lists, share=share) - elif model_type == "DT": - model = DecisionTree(code_len, label_lists = label_lists, share=share) - - pre_X, pre_Y = generate_data_via_codes(words, err, letter_num) - X, Y = generate_data_via_codes(words, 0, letter_num) - - str_words = ["".join(str(c) for c in word) for word in words] - - recoder.print(str_words) - - model.train(pre_X, pre_Y) - abl_epoch = 30 - res = framework.train(model, abducer, X, Y, sample_num = 10000, verbose = 1) - print("Initial data accuracy:", 1 - err) - print("Abd word accuracy: ", res["accuracy_word"] * 1.0 / res["total_word"]) - print("Abd char accuracy: ", res["accuracy_abd_char"] * 1.0 / res["total_abd_char"]) - print("Ori char accuracy: ", res["accuracy_ori_char"] * 1.0 / res["total_ori_char"]) - print("End expriment", idx) - print() - - recoder.dump(open(recoder_file_path, "wb")) - return True - -if __name__ == "__main__": - os.system("mkdir share_result") - os.system("mkdir non_share_result") - - for times in range(5): - for code_num in [32, 64, 128]: - params = [11, times, code_num, False, "KNN", True, 2] - run_test(params) - - params = [11, times, code_num, False, "KNN", False, 2] - run_test(params) - - #params = [11, 0, 32, False, "DT", False, 2] - #run_test(params) - diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..534a288 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,44 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "abl" +version = "0.1.5" +authors = [ + { name="LAMDA 2024" }, +] +description = "Abductive learning package project" +readme = "README.md" +requires-python = ">=3.6.0" +license = {text = "MIT LICENSE"} +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Science/Research", + "Intended Audience :: Developers", + "Programming Language :: Python", + "Topic :: Software Development", + "Topic :: Scientific/Engineering", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", +] +dependencies = [ + "numpy>=1.15.0", + "pyswip==0.2.9", + "torch>=1.11.0", + "torchvision>=0.12.0", + "zoopt>=0.3.0", + "termcolor>=2.3.0" +] + +[project.urls] +Homepage = "https://github.com/AbductiveLearning/ABL-Package" +Issues = "https://github.com/AbductiveLearning/ABL-Package/issues" + +[project.optional-dependencies] +test = [ + "pytest-cov", + "black==22.10.0", +] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..8fd3a3f --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +numpy>=1.15.0, +pyswip==0.2.9, +torch>=1.11.0, +torchvision>=0.12.0, +zoopt>=0.3.0, +termcolor>=2.3.0 \ No newline at end of file diff --git a/share_example.py b/share_example.py deleted file mode 100644 index 1b2e233..0000000 --- a/share_example.py +++ /dev/null @@ -1,96 +0,0 @@ -# coding: utf-8 -#================================================================# -# Copyright (C) 2021 Freecss All rights reserved. -# -# File Name ：share_example.py -# Author ：freecss -# Email ：karlfreecss@gmail.com -# Created Date ：2021/06/07 -# Description ： -# -#================================================================# - -from utils.plog import logger -from models.wabl_models import DecisionTree, KNN -import pickle as pk -import numpy as np -import time -import framework - -from multiprocessing import Pool -import os -from datasets.data_generator import generate_data_via_codes, code_generator -from collections import defaultdict -from abducer.abducer_base import AbducerBase -from abducer.kb import ClsKB, RegKB - -def run_test(params): - code_len, times, code_num, share, model_type, need_prob, letter_num = params - - if share: - result_dir = "share_result" - else: - result_dir = "non_share_result" - - recoder_file_path = f"{result_dir}/random_{times}_{code_len}_{code_num}_{model_type}_{need_prob}.pk"# - - words = code_generator(code_len, code_num, letter_num) - kb = ClsKB(words) - abducer = AbducerBase(kb) - - label_lists = [[] for _ in range(code_len)] - for widx, word in enumerate(words): - for cidx, c in enumerate(word): - label_lists[cidx].append(c) - - if share: - label_lists = [sum(label_lists, [])] - - recoder = logger() - recoder.set_savefile("test.log") - for idx, err in enumerate(range(0, 41)): - print("Start expriment", idx) - start = time.process_time() - err = err / 40. - if 1 - err < (1. / letter_num): - break - if model_type == "KNN": - model = KNN(code_len, label_lists = label_lists, share=share) - elif model_type == "DT": - model = DecisionTree(code_len, label_lists = label_lists, share=share) - - pre_X, pre_Y = generate_data_via_codes(words, err, letter_num) - X, Y = generate_data_via_codes(words, 0, letter_num) - - str_words = ["".join(str(c) for c in word) for word in words] - - recoder.print(str_words) - - model.train(pre_X, pre_Y) - abl_epoch = 30 - res = framework.train(model, abducer, X, Y, sample_num = 10000, verbose = 1) - print("Initial data accuracy:", 1 - err) - print("Abd word accuracy: ", res["accuracy_word"] * 1.0 / res["total_word"]) - print("Abd char accuracy: ", res["accuracy_abd_char"] * 1.0 / res["total_abd_char"]) - print("Ori char accuracy: ", res["accuracy_ori_char"] * 1.0 / res["total_ori_char"]) - print("End expriment", idx) - print() - - recoder.dump(open(recoder_file_path, "wb")) - return True - -if __name__ == "__main__": - os.system("mkdir share_result") - os.system("mkdir non_share_result") - - for times in range(5): - for code_num in [32, 64, 128]: - params = [11, times, code_num, True, "KNN", True, 2] - run_test(params) - - params = [11, times, code_num, True, "KNN", False, 2] - run_test(params) - - #params = [11, 0, 32, True, "DT", True, 2] - #run_test(params) - diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..2b71466 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,232 @@ +import numpy as np +import platform +import pytest +import torch +import torch.nn as nn +import torch.optim as optim + +from abl.data.structures import ListData +from abl.learning import BasicNN +from abl.reasoning import GroundKB, KBBase, PrologKB, Reasoner + + +class LeNet5(nn.Module): + def __init__(self, num_classes=10, image_size=(28, 28)): + super(LeNet5, self).__init__() + self.conv1 = nn.Sequential( + nn.Conv2d(1, 6, 3, padding=1), + nn.ReLU(), + nn.MaxPool2d(kernel_size=2, stride=2), + ) + self.conv2 = nn.Sequential( + nn.Conv2d(6, 16, 3), nn.ReLU(), nn.MaxPool2d(kernel_size=2, stride=2) + ) + self.conv3 = nn.Sequential(nn.Conv2d(16, 16, 3), nn.ReLU()) + + feature_map_size = (np.array(image_size) // 2 - 2) // 2 - 2 + num_features = 16 * feature_map_size[0] * feature_map_size[1] + + self.fc1 = nn.Sequential(nn.Linear(num_features, 120), nn.ReLU()) + self.fc2 = nn.Sequential(nn.Linear(120, 84), nn.ReLU()) + self.fc3 = nn.Linear(84, num_classes) + + def forward(self, x): + x = self.conv1(x) + x = self.conv2(x) + x = self.conv3(x) + x = torch.flatten(x, 1) + x = self.fc1(x) + x = self.fc2(x) + x = self.fc3(x) + return x + + +# Fixture for BasicNN instance +@pytest.fixture +def basic_nn_instance(): + model = LeNet5() + loss_fn = nn.CrossEntropyLoss() + optimizer = optim.Adam(model.parameters()) + return BasicNN(model, loss_fn, optimizer) + + +# Fixture for base_model instance +@pytest.fixture +def base_model_instance(): + model = LeNet5() + loss_fn = nn.CrossEntropyLoss() + optimizer = optim.Adam(model.parameters()) + return BasicNN(model, loss_fn, optimizer) + + +# Fixture for ListData instance +@pytest.fixture +def list_data_instance(): + data_examples = ListData() + data_examples.X = [list(torch.randn(2, 1, 28, 28)) for _ in range(3)] + data_examples.Y = [1, 2, 3] + data_examples.gt_pseudo_label = [[1, 2], [3, 4], [5, 6]] + return data_examples + + +@pytest.fixture +def data_examples_add(): + # favor 1 in first one + prob1 = [ + [0, 0.99, 0, 0, 0, 0, 0, 0.01, 0, 0], + [0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1], + ] + # favor 7 in first one + prob2 = [ + [0, 0.01, 0, 0, 0, 0, 0, 0.99, 0, 0], + [0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1], + ] + + data_examples_add = ListData() + data_examples_add.X = None + data_examples_add.pred_pseudo_label = [[1, 1], [1, 1], [1, 1], [1, 1]] + data_examples_add.pred_prob = [prob1, prob2, prob1, prob2] + data_examples_add.Y = [8, 8, 17, 10] + return data_examples_add + + +@pytest.fixture +def data_examples_hwf(): + data_examples_hwf = ListData() + data_examples_hwf.X = None + data_examples_hwf.pred_pseudo_label = [ + ["5", "+", "2"], + ["5", "+", "9"], + ["5", "+", "9"], + ["5", "-", "8", "8", "8"], + ] + data_examples_hwf.pred_prob = [None, None, None, None] + data_examples_hwf.Y = [3, 64, 65, 3.17] + return data_examples_hwf + + +class AddKB(KBBase): + def __init__(self, pseudo_label_list=list(range(10)), use_cache=False): + super().__init__(pseudo_label_list, use_cache=use_cache) + + def logic_forward(self, nums): + return sum(nums) + + +class AddGroundKB(GroundKB): + def __init__(self, pseudo_label_list=list(range(10)), GKB_len_list=[2]): + super().__init__(pseudo_label_list, GKB_len_list) + + def logic_forward(self, nums): + return sum(nums) + + +class HwfKB(KBBase): + def __init__( + self, + pseudo_label_list=[ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "+", + "-", + "times", + "div", + ], + max_err=1e-3, + use_cache=False, + ): + super().__init__(pseudo_label_list, max_err, use_cache) + + def _valid_candidate(self, formula): + if len(formula) % 2 == 0: + return False + for i in range(len(formula)): + if i % 2 == 0 and formula[i] not in [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + ]: + return False + if i % 2 != 0 and formula[i] not in ["+", "-", "times", "div"]: + return False + return True + + def logic_forward(self, formula): + if not self._valid_candidate(formula): + return None + mapping = {str(i): str(i) for i in range(1, 10)} + mapping.update({"+": "+", "-": "-", "times": "*", "div": "/"}) + formula = [mapping[f] for f in formula] + return eval("".join(formula)) + + +class HedKB(PrologKB): + def __init__(self, pseudo_label_list, pl_file): + super().__init__(pseudo_label_list, pl_file) + + def consist_rule(self, exs, rules): + rules = str(rules).replace("'", "") + pl_query = "eval_inst_feature(%s, %s)." % (exs, rules) + return len(list(self.prolog.query(pl_query))) != 0 + + +@pytest.fixture +def kb_add(): + return AddKB() + + +@pytest.fixture +def kb_add_cache(): + return AddKB(use_cache=True) + + +@pytest.fixture +def kb_add_ground(): + return AddGroundKB() + + +@pytest.fixture +def kb_add_prolog(): + if platform.system() == "Darwin": + return + kb = PrologKB(pseudo_label_list=list(range(10)), pl_file="examples/mnist_add/add.pl") + return kb + + +@pytest.fixture +def kb_hwf1(): + return HwfKB(max_err=0.1) + + +@pytest.fixture +def kb_hwf2(): + return HwfKB(max_err=1) + + +@pytest.fixture +def kb_hed(): + if platform.system() == "Darwin": + return + kb = HedKB( + pseudo_label_list=[1, 0, "+", "="], + pl_file="examples/hed/reasoning/learn_add.pl", + ) + return kb + + +@pytest.fixture +def reasoner_instance(kb_add): + return Reasoner(kb_add, "confidence") diff --git a/tests/test_abl_model.py b/tests/test_abl_model.py new file mode 100644 index 0000000..c9e256e --- /dev/null +++ b/tests/test_abl_model.py @@ -0,0 +1,75 @@ +from unittest.mock import Mock, create_autospec + +import numpy as np +import pytest + +from abl.learning import ABLModel + + +class TestABLModel(object): + def test_ablmodel_initialization(self): + """Test the initialization method of the ABLModel class.""" + + invalid_base_model = Mock(spec=[]) + with pytest.raises(NotImplementedError): + ABLModel(invalid_base_model) + + invalid_base_model = Mock(spec=["fit"]) + invalid_base_model.fit.return_value = 1.0 + with pytest.raises(NotImplementedError): + ABLModel(invalid_base_model) + + invalid_base_model = Mock(spec=["predict"]) + invalid_base_model.predict.return_value = np.array(1.0) + with pytest.raises(NotImplementedError): + ABLModel(invalid_base_model) + + base_model = Mock(spec=["fit", "predict"]) + base_model.fit.return_value = 1.0 + base_model.predict.return_value = np.array(1.0) + model = ABLModel(base_model) + assert hasattr(model, "base_model"), "The model should have a 'base_model' attribute." + + def test_ablmodel_predict(self, base_model_instance, list_data_instance): + """Test the predict method of the ABLModel class.""" + model = ABLModel(base_model_instance) + predictions = model.predict(list_data_instance) + assert isinstance(predictions, dict), "Predictions should be returned as a dictionary." + assert isinstance(predictions["label"], list) + assert isinstance(predictions["prob"], list) + + basic_nn_mock = create_autospec(base_model_instance, spec_set=True) + delattr(basic_nn_mock, "predict_proba") + model = ABLModel(basic_nn_mock) + predictions = model.predict(list_data_instance) + assert isinstance(predictions, dict), "Predictions should be returned as a dictionary." + assert isinstance(predictions["label"], list) + assert predictions["prob"] is None + + def test_ablmodel_train(self, base_model_instance, list_data_instance): + """Test the train method of the ABLModel class.""" + model = ABLModel(base_model_instance) + list_data_instance.abduced_idx = [[1, 2], [3, 4], [5, 6]] + ins = model.train(list_data_instance) + assert ins == model.base_model, "Training should return the base model instance." + + def test_ablmodel_save_load(self, base_model_instance, tmp_path): + """Test the save method of the ABLModel class.""" + model = ABLModel(base_model_instance) + model_path = tmp_path / "model.pth" + model.save(save_path=str(model_path)) + assert model_path.exists() + model.load(load_path=str(model_path)) + assert isinstance(model.base_model, type(base_model_instance)) + + def test_ablmodel_invalid_operation(self, base_model_instance): + """Test invalid operation handling in the ABLModel class.""" + model = ABLModel(base_model_instance) + with pytest.raises(ValueError): + model._model_operation("invalid_operation", save_path=None) + + def test_ablmodel_operation_without_path(self, base_model_instance): + """Test operation without providing a path in the ABLModel class.""" + model = ABLModel(base_model_instance) + with pytest.raises(ValueError): + model.save() # No path provided diff --git a/tests/test_basic_nn.py b/tests/test_basic_nn.py new file mode 100644 index 0000000..deaa760 --- /dev/null +++ b/tests/test_basic_nn.py @@ -0,0 +1,117 @@ +import numpy +import pytest +import torch +import torch.nn as nn +import torch.optim as optim +from torch.utils.data import DataLoader, TensorDataset + + +class TestBasicNN(object): + @pytest.fixture + def sample_data(self): + return torch.randn(32, 1, 28, 28), torch.randint(0, 10, (32,)) + + @pytest.fixture + def sample_data_loader_with_label(self, sample_data): + X, y = sample_data + return DataLoader(TensorDataset(X, y), batch_size=4) + + @pytest.fixture + def sample_data_loader_without_label(self, sample_data): + X, _ = sample_data + return DataLoader( + TensorDataset(X), + batch_size=4, + collate_fn=lambda batch: torch.stack([item[0] for item in batch]), + ) + + def test_initialization(self, basic_nn_instance): + """Test initialization of the BasicNN class""" + assert basic_nn_instance.model is not None + assert isinstance(basic_nn_instance.loss_fn, nn.Module) + assert isinstance(basic_nn_instance.optimizer, optim.Optimizer) + + def test_training_methods(self, basic_nn_instance, sample_data, sample_data_loader_with_label): + """Test train_epoch, fit, and score methods of the BasicNN class""" + + # Test train_epoch + loss = basic_nn_instance.train_epoch(sample_data_loader_with_label) + assert isinstance(loss, float) + + # Test fit with direct data + X, y = sample_data + ins = basic_nn_instance.fit(X=list(X), y=list(y)) + assert ins == basic_nn_instance + + # Test fit with DataLoader + ins = basic_nn_instance.fit(data_loader=sample_data_loader_with_label) + assert ins == basic_nn_instance + + # Test invalid fit method input + with pytest.raises(ValueError): + basic_nn_instance.fit(X=None, y=None, data_loader=None) + + # Test score with direct data + accuracy = basic_nn_instance.score(X=list(X), y=list(y)) + assert 0 <= accuracy <= 1 + + # Test score with DataLoader + accuracy = basic_nn_instance.score(data_loader=sample_data_loader_with_label) + assert 0 <= accuracy <= 1 + + def test_prediction_methods( + self, basic_nn_instance, sample_data, sample_data_loader_without_label + ): + """Test predict and predict_proba methods of the BasicNN class""" + X, _ = sample_data + + # Test predict with direct data + predictions = basic_nn_instance.predict(X=list(X)) + assert len(predictions) == len(X) + assert numpy.isin(predictions, list(range(10))).all() + + # Test predict_proba with direct data + predict_proba = basic_nn_instance.predict_proba(X=list(X)) + assert len(predict_proba) == len(X) + assert ((0 <= predict_proba) & (predict_proba <= 1)).all() + + # Test predict and predict_proba with DataLoader + for method in [basic_nn_instance.predict, basic_nn_instance.predict_proba]: + result = method(data_loader=sample_data_loader_without_label) + assert len(result) == len(X) + if method == basic_nn_instance.predict: + assert numpy.isin(result, list(range(10))).all() + else: + assert ((0 <= result) & (result <= 1)).all() + + def test_save_load(self, basic_nn_instance, tmp_path): + """Test save and load methods of the BasicNN class""" + + # Test save with explicit save_path + explicit_save_path = tmp_path / "model_explicit.pth" + basic_nn_instance.save(epoch_id=1, save_path=str(explicit_save_path)) + assert explicit_save_path.exists(), "Model should be saved to the explicit path" + + # Test save without providing save_path (using save_dir) + basic_nn_instance.save_dir = str(tmp_path) + implicit_save_path = tmp_path / "model_checkpoint_epoch_1.pth" + basic_nn_instance.save(epoch_id=1) + assert implicit_save_path.exists(), "Model should be saved to the implicit path in save_dir" + + # Test error when save_path and save_dir are both None + basic_nn_instance.save_dir = None + with pytest.raises(ValueError): + basic_nn_instance.save(epoch_id=1) + + # Test error on loading from a None path + with pytest.raises(ValueError): + basic_nn_instance.load(load_path=None) + + # Test loading model state + original_state = basic_nn_instance.model.state_dict() + basic_nn_instance.load(load_path=str(explicit_save_path)) + loaded_state = basic_nn_instance.model.state_dict() + for key in original_state: + assert torch.allclose( + original_state[key], loaded_state[key] + ), "Model state should be restored after loading" diff --git a/tests/test_reasoning.py b/tests/test_reasoning.py new file mode 100644 index 0000000..d3a08b6 --- /dev/null +++ b/tests/test_reasoning.py @@ -0,0 +1,272 @@ +import numpy as np +import platform +import pytest + +from abl.reasoning import PrologKB, Reasoner + + +class TestKBBase(object): + def test_init(self, kb_add): + assert kb_add.pseudo_label_list == list(range(10)) + + def test_init_cache(self, kb_add_cache): + assert kb_add_cache.pseudo_label_list == list(range(10)) + assert kb_add_cache.use_cache is True + + def test_logic_forward(self, kb_add): + result = kb_add.logic_forward([1, 2]) + assert result == 3 + with pytest.raises(TypeError): + kb_add.logic_forward([1, 2], [0.1, -0.2, 0.2, -0.3]) + + def test_revise_at_idx(self, kb_add): + result = kb_add.revise_at_idx([0, 2], 2, [0.1, -0.2, 0.2, -0.3], []) + assert result == ([[0, 2]], [2]) + result = kb_add.revise_at_idx([1, 2], 2, [0.1, -0.2, 0.2, -0.3], []) + assert result == ([], []) + result = kb_add.revise_at_idx([1, 2], 2, [0.1, -0.2, 0.2, -0.3], [0, 1]) + assert result == ([[0, 2], [1, 1], [2, 0]], [2, 2, 2]) + + def test_abduce_candidates(self, kb_add): + result = kb_add.abduce_candidates( + [0, 1], 1, [0.1, -0.2, 0.2, -0.3], max_revision_num=2, require_more_revision=0 + ) + assert result == ([[0, 1]], [1]) + result = kb_add.abduce_candidates( + [1, 2], 1, [0.1, -0.2, 0.2, -0.3], max_revision_num=2, require_more_revision=0 + ) + assert result == ([[1, 0]], [1]) + + +class TestGroundKB(object): + def test_init(self, kb_add_ground): + assert kb_add_ground.pseudo_label_list == list(range(10)) + assert kb_add_ground.GKB_len_list == [2] + assert kb_add_ground.GKB + + def test_logic_forward_ground(self, kb_add_ground): + result = kb_add_ground.logic_forward([1, 2]) + assert result == 3 + + def test_abduce_candidates_ground(self, kb_add_ground): + result = kb_add_ground.abduce_candidates( + [1, 2], 1, [0.1, -0.2, 0.2, -0.3], max_revision_num=2, require_more_revision=0 + ) + assert result == ([(1, 0)], [1]) + + +class TestPrologKB(object): + def test_init_pl1(self, kb_add_prolog): + if platform.system() == "Darwin": + return + assert kb_add_prolog.pseudo_label_list == list(range(10)) + assert kb_add_prolog.pl_file == "examples/mnist_add/add.pl" + + def test_init_pl2(self, kb_hed): + if platform.system() == "Darwin": + return + assert kb_hed.pseudo_label_list == [1, 0, "+", "="] + assert kb_hed.pl_file == "examples/hed/reasoning/learn_add.pl" + + def test_prolog_file_not_exist(self): + if platform.system() == "Darwin": + return + pseudo_label_list = [1, 2] + non_existing_file = "path/to/non_existing_file.pl" + with pytest.raises(FileNotFoundError) as excinfo: + PrologKB(pseudo_label_list=pseudo_label_list, pl_file=non_existing_file) + assert non_existing_file in str(excinfo.value) + + def test_logic_forward_pl1(self, kb_add_prolog): + if platform.system() == "Darwin": + return + result = kb_add_prolog.logic_forward([1, 2]) + assert result == 3 + + def test_logic_forward_pl2(self, kb_hed): + if platform.system() == "Darwin": + return + consist_exs = [ + [1, 1, "+", 0, "=", 1, 1], + [1, "+", 1, "=", 1, 0], + [0, "+", 0, "=", 0], + ] + inconsist_exs = [ + [1, 1, "+", 0, "=", 1, 1], + [1, "+", 1, "=", 1, 0], + [0, "+", 0, "=", 0], + [0, "+", 0, "=", 1], + ] + assert kb_hed.logic_forward(consist_exs) is True + assert kb_hed.logic_forward(inconsist_exs) is False + + def test_revise_at_idx(self, kb_add_prolog): + if platform.system() == "Darwin": + return + result = kb_add_prolog.revise_at_idx([1, 2], 2, [0.1, -0.2, 0.2, -0.3], [0]) + assert result == ([[0, 2]], [2]) + + +class TestReaonser(object): + def test_reasoner_init(self, reasoner_instance): + assert reasoner_instance.dist_func == "confidence" + + def test_invalid_predefined_dist_func(self, kb_add): + with pytest.raises(NotImplementedError) as excinfo: + Reasoner(kb_add, "invalid_dist_func") + assert 'Valid options for predefined dist_func include "hamming" and "confidence"' in str( + excinfo.value + ) + + def random_dist(self, data_example, candidates, candidate_idxs, reasoning_results): + cost_list = [np.random.rand() for _ in candidates] + return cost_list + + def test_user_defined_dist_func(self, kb_add): + reasoner = Reasoner(kb_add, self.random_dist) + assert reasoner.dist_func == self.random_dist + + def invalid_dist1(self, candidates): + cost_list = np.array([np.random.rand() for _ in candidates]) + return cost_list + + def invalid_dist2(self, data_example, candidates, candidate_idxs, reasoning_results): + cost_list = np.array([np.random.rand() for _ in candidates]) + return np.append(cost_list, np.random.rand()) + + def test_invalid_user_defined_dist_func(self, kb_add, data_examples_add): + with pytest.raises(ValueError) as excinfo: + Reasoner(kb_add, self.invalid_dist1) + assert "User-defined dist_func must have exactly four parameters" in str(excinfo.value) + with pytest.raises(ValueError) as excinfo: + reasoner = Reasoner(kb_add, self.invalid_dist2) + reasoner.batch_abduce(data_examples_add) + assert ( + "The length of the array returned by dist_func must be " + + "equal to the number of candidates" + in str(excinfo.value) + ) + + +class TestBatchAbduce(object): + def test_batch_abduce_add(self, kb_add, data_examples_add): + reasoner1 = Reasoner(kb_add, "confidence", max_revision=1, require_more_revision=0) + reasoner2 = Reasoner(kb_add, "confidence", max_revision=1, require_more_revision=1) + reasoner3 = Reasoner(kb_add, "confidence", max_revision=2, require_more_revision=0) + reasoner4 = Reasoner(kb_add, "confidence", max_revision=2, require_more_revision=1) + assert reasoner1.batch_abduce(data_examples_add) == [[1, 7], [7, 1], [], [1, 9]] + assert reasoner2.batch_abduce(data_examples_add) == [[1, 7], [7, 1], [], [1, 9]] + assert reasoner3.batch_abduce(data_examples_add) == [ + [1, 7], + [7, 1], + [8, 9], + [1, 9], + ] + assert reasoner4.batch_abduce(data_examples_add) == [ + [1, 7], + [7, 1], + [8, 9], + [7, 3], + ] + + def test_batch_abduce_ground(self, kb_add_ground, data_examples_add): + reasoner1 = Reasoner(kb_add_ground, "confidence", max_revision=1, require_more_revision=0) + reasoner2 = Reasoner(kb_add_ground, "confidence", max_revision=1, require_more_revision=1) + reasoner3 = Reasoner(kb_add_ground, "confidence", max_revision=2, require_more_revision=0) + reasoner4 = Reasoner(kb_add_ground, "confidence", max_revision=2, require_more_revision=1) + assert reasoner1.batch_abduce(data_examples_add) == [(1, 7), (7, 1), [], (1, 9)] + assert reasoner2.batch_abduce(data_examples_add) == [(1, 7), (7, 1), [], (1, 9)] + assert reasoner3.batch_abduce(data_examples_add) == [ + (1, 7), + (7, 1), + (8, 9), + (1, 9), + ] + assert reasoner4.batch_abduce(data_examples_add) == [ + (1, 7), + (7, 1), + (8, 9), + (7, 3), + ] + + def test_batch_abduce_prolog(self, kb_add_prolog, data_examples_add): + if platform.system() == "Darwin": + return + reasoner1 = Reasoner(kb_add_prolog, "confidence", max_revision=1, require_more_revision=0) + reasoner2 = Reasoner(kb_add_prolog, "confidence", max_revision=1, require_more_revision=1) + reasoner3 = Reasoner(kb_add_prolog, "confidence", max_revision=2, require_more_revision=0) + reasoner4 = Reasoner(kb_add_prolog, "confidence", max_revision=2, require_more_revision=1) + assert reasoner1.batch_abduce(data_examples_add) == [[1, 7], [7, 1], [], [1, 9]] + assert reasoner2.batch_abduce(data_examples_add) == [[1, 7], [7, 1], [], [1, 9]] + assert reasoner3.batch_abduce(data_examples_add) == [ + [1, 7], + [7, 1], + [8, 9], + [1, 9], + ] + assert reasoner4.batch_abduce(data_examples_add) == [ + [1, 7], + [7, 1], + [8, 9], + [7, 3], + ] + + def test_batch_abduce_zoopt(self, kb_add_prolog, data_examples_add): + if platform.system() == "Darwin": + return + reasoner1 = Reasoner(kb_add_prolog, "confidence", use_zoopt=True, max_revision=1) + reasoner2 = Reasoner(kb_add_prolog, "confidence", use_zoopt=True, max_revision=2) + assert reasoner1.batch_abduce(data_examples_add) == [[1, 7], [7, 1], [], [1, 9]] + assert reasoner2.batch_abduce(data_examples_add) == [ + [1, 7], + [7, 1], + [8, 9], + [7, 3], + ] + + def test_batch_abduce_hwf1(self, kb_hwf1, data_examples_hwf): + reasoner1 = Reasoner(kb_hwf1, "hamming", max_revision=3, require_more_revision=0) + reasoner2 = Reasoner(kb_hwf1, "hamming", max_revision=0.5, require_more_revision=0) + reasoner3 = Reasoner(kb_hwf1, "hamming", max_revision=0.9, require_more_revision=0) + res = reasoner1.batch_abduce(data_examples_hwf) + assert res == [ + ["1", "+", "2"], + ["8", "times", "8"], + [], + ["4", "-", "6", "div", "8"], + ] + res = reasoner2.batch_abduce(data_examples_hwf) + assert res == [["1", "+", "2"], [], [], []] + res = reasoner3.batch_abduce(data_examples_hwf) + assert res == [ + ["1", "+", "2"], + ["8", "times", "8"], + [], + ["4", "-", "6", "div", "8"], + ] + + def test_batch_abduce_hwf2(self, kb_hwf2, data_examples_hwf): + reasoner1 = Reasoner(kb_hwf2, "hamming", max_revision=3, require_more_revision=0) + reasoner2 = Reasoner(kb_hwf2, "hamming", max_revision=0.5, require_more_revision=0) + reasoner3 = Reasoner(kb_hwf2, "hamming", max_revision=0.9, require_more_revision=0) + res = reasoner1.batch_abduce(data_examples_hwf) + assert res == [ + ["1", "+", "2"], + ["7", "times", "9"], + ["8", "times", "8"], + ["5", "-", "8", "div", "8"], + ] + res = reasoner2.batch_abduce(data_examples_hwf) + assert res == [ + ["1", "+", "2"], + ["7", "times", "9"], + [], + ["5", "-", "8", "div", "8"], + ] + res = reasoner3.batch_abduce(data_examples_hwf) + assert res == [ + ["1", "+", "2"], + ["7", "times", "9"], + ["8", "times", "8"], + ["5", "-", "8", "div", "8"], + ] diff --git a/utils/plog.py b/utils/plog.py deleted file mode 100644 index 941dc11..0000000 --- a/utils/plog.py +++ /dev/null @@ -1,152 +0,0 @@ -# coding: utf-8 -#================================================================# -# Copyright (C) 2020 Freecss All rights reserved. -# -# File Name ：plog.py -# Author ：freecss -# Email ：karlfreecss@gmail.com -# Created Date ：2020/10/23 -# Description ： -# -#================================================================# - -import time -import logging -import pickle as pk -import os -import functools - -log_name = "default_log.txt" -logging.basicConfig(level=logging.INFO, - filename=log_name, - filemode='a', - format='%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s') - -global recorder -recorder = None - -def mkdir(dirpath): - if not os.path.exists(dirpath): - os.makedirs(dirpath) - -class ResultRecorder: - def __init__(self, pk_dir = None, pk_filepath = None): - self.set_savefile(pk_dir, pk_filepath) - - self.result = {} - logging.info("===========================================================") - logging.info("============= Result Recorder Version: 0.02 ===============") - logging.info("===========================================================\n") - - pass - - def set_savefile(self, pk_dir = None, pk_filepath = None): - if pk_dir is None: - pk_dir = "result" - mkdir(pk_dir) - - if pk_filepath is None: - local_time = time.strftime("%Y%m%d_%H_%M_%S", time.localtime()) - pk_filepath = os.path.join(pk_dir, local_time + ".pk") - - self.save_file = open(pk_filepath, "wb") - - logger = logging.getLogger() - logger.handlers[0].stream.close() - logger.removeHandler(logger.handlers[0]) - - filename = os.path.join(pk_dir, local_time + ".txt") - file_handler = logging.FileHandler(filename) - file_handler.setLevel(logging.DEBUG) - formatter = logging.Formatter('%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s') - file_handler.setFormatter(formatter) - logger.addHandler(file_handler) - - def print(self, *argv, screen = False): - info = "" - for data in argv: - info += str(data) - if screen: - print(info) - logging.info(info) - - def print_result(self, *argv): - for data in argv: - info = "#Result# %s" % str(data) - #print(info) - logging.info(info) - - def store(self, *argv): - for data in argv: - if data.find(":") < 0: - continue - label, data = data.split(":") - self.store_kv(label, data) - - def write_result(self, *argv): - self.print_result(*argv) - self.store(*argv) - - def store_kv(self, label, data): - self.result.setdefault(label, []) - self.result[label].append(data) - - def write_kv(self, label, data): - self.print_result({label : data}) - #self.print_result(label + ":" + str(data)) - self.store_kv(label, data) - - def dump(self, save_file = None): - if save_file is None: - save_file = self.save_file - pk.dump(self.result, save_file) - - def clock(self, func): - @functools.wraps(func) - def clocked(*args, **kwargs): - t0 = time.perf_counter() - result = func(*args, **kwargs) - elapsed = time.perf_counter() - t0 - - name = func.__name__ - # arg_str = ','.join(repr(arg) for arg in args) - # context = f"{name}: ({arg_str})=>({result}), cost {elapsed}s" - context = f"{name}: ()=>(), cost {elapsed}s" - self.write_kv("func:", context) - - return result - return clocked - - def __del__(self): - self.dump() - -def clocker(*argv): - global recorder - if recorder is None: - recorder = ResultRecorder() - return recorder.clock(*argv) - -def INFO(*argv, screen = False): - global recorder - if recorder is None: - recorder = ResultRecorder() - return recorder.print(*argv, screen = screen) - -def DEBUG(*argv, screen = False): - global recorder - if recorder is None: - recorder = ResultRecorder() - return recorder.print(*argv, screen = screen) - -def logger(): - global recorder - if recorder is None: - recorder = ResultRecorder() - return recorder - -if __name__ == "__main__": - recorder = ResultRecorder() - recorder.write_kv("test", 1) - recorder.set_savefile(pk_dir = "haha") - recorder.write_kv("test", 1) -