finish basic logic of trainer & tester, wait for unittest

action/action.py

@@ -1,3 +1,5 @@
+from saver.logger import Logger
+
 
 class Action(object):
     """
@@ -6,7 +8,7 @@ class Action(object):
 
     def __init__(self):
         super(Action, self).__init__()
-        self.logger = None
+        self.logger = Logger("logger_output.txt")
 
     def load_config(self, args):
         raise NotImplementedError
@@ -14,27 +16,31 @@ class Action(object):
     def load_dataset(self, args):
         raise NotImplementedError
 
-    def log(self, args):
-        print("call logger.log")
+    def log(self, string):
+        self.logger.log(string)
 
-    def batchify(self, X, Y=None):
+    def batchify(self, batch_size, X, Y=None):
         """
-        :param X:
-        :param Y:
+        :param batch_size: int
+        :param X: feature matrix of size [n_sample, m_feature]
+        :param Y: label vector of size [n_sample, 1] (optional)
         :return iteration:int, the number of step in each epoch
                  generator:generator, to generate batch inputs
         """
-        data = X
-        if Y is not None:
-            data = [X, Y]
-        return 2, self._batch_generate(data)
-
-    def _batch_generate(self, data):
-        step = 10
-        for i in range(2):
-            start = i * step
-            end = (i + 1) * step
-            yield data[0][start:end], data[1][start:end]
+        n_samples = X.shape[0]
+        num_iter = n_samples / batch_size
+        if Y is None:
+            generator = self._batch_generate(batch_size, num_iter, X)
+        else:
+            generator = self._batch_generate(batch_size, num_iter, X, Y)
+        return num_iter, generator
+
+    @staticmethod
+    def _batch_generate(batch_size, num_iter, *data):
+        for step in range(num_iter):
+            start = batch_size * step
+            end = (batch_size + 1) * step
+            yield tuple([x[start:end, :] for x in data])
 
     def make_log(self, *args):
         return "log"

action/tester.py

@@ -1,3 +1,5 @@
+from collections import namedtuple
+
 import numpy as np
 
 from action.action import Action
@@ -6,22 +8,39 @@ from action.action import Action
 class Tester(Action):
     """docstring for Tester"""
 
+    TestConfig = namedtuple("config", ["validate_in_training", "save_dev_input", "save_output",
+                                       "save_loss", "batch_size"])
+
     def __init__(self, test_args):
         """
         :param test_args: named tuple
         """
         super(Tester, self).__init__()
-        self.test_args = test_args
-        # self.args_dict = {name: value for name, value in self.test_args.__dict__.iteritems()}
-        self.mean_loss = None
+        self.validate_in_training = test_args.validate_in_training
+        self.save_dev_input = test_args.save_dev_input
+        self.valid_x = None
+        self.valid_y = None
+        self.save_output = test_args.save_output
         self.output = None
+        self.save_loss = test_args.save_loss
+        self.mean_loss = None
+        self.batch_size = test_args.batch_size
 
     def test(self, network, data):
-        # transform into network input and label
-        X, Y = network.prepare_input(data)
+        network.mode(test=True)  # turn on the testing mode
+
+        if not self.save_dev_input:
+            # transform into network input and label
+            valid_x, valid_y = network.prepare_input(data)
+            if self.validate_in_training:
+                self.valid_x = valid_x
+                self.valid_y = valid_y
+        else:
+            valid_x = self.valid_x
+            valid_y = self.valid_y
 
         # split into batches by self.batch_size
-        iterations, test_batch_generator = self.batchify(X, Y)
+        iterations, test_batch_generator = self.batchify(self.batch_size, valid_x, valid_y)
 
         batch_output = list()
         loss_history = list()
@@ -33,16 +52,19 @@ class Tester(Action):
 
             # forward pass from tests input to predicted output
             prediction = network.data_forward(batch_x)
-            batch_output.append(prediction)
 
-            # get the loss
             loss = network.loss(batch_y, prediction)
 
-            loss_history.append(loss)
-            self.log(self.make_log(step, loss))
+            if self.save_output:
+                batch_output.append(prediction)
+            if self.save_loss:
+                loss_history.append(loss)
+                self.log(self.make_log(step, loss))
 
-        self.mean_loss = np.mean(np.array(loss_history))
-        self.output = self.make_output(batch_output)
+        if self.save_loss:
+            self.mean_loss = np.mean(np.array(loss_history))
+        if self.save_output:
+            self.output = self.make_output(batch_output)
 
     @property
     def loss(self):
@@ -55,3 +77,9 @@ class Tester(Action):
     def make_output(self, batch_output):
         # construct full prediction with batch outputs
         return np.concatenate((batch_output[0], batch_output[1]), axis=0)
+
+    def load_config(self, args):
+        raise NotImplementedError
+
+    def load_dataset(self, args):
+        raise NotImplementedError

action/trainer.py

@@ -1,3 +1,5 @@
+from collections import namedtuple
+
 from .action import Action
 from .tester import Tester
 
@@ -6,32 +8,42 @@ class Trainer(Action):
     """
         Trainer for common training logic of all models
     """
+    TrainConfig = namedtuple("config", ["epochs", "validate", "save_when_better", "log_per_step", "log_validation"])
 
     def __init__(self, train_args):
         """
         :param train_args: namedtuple
         """
         super(Trainer, self).__init__()
-        self.train_args = train_args
-        # self.args_dict = {name: value for name, value in self.train_args.__dict__.iteritems()}
-        self.n_epochs = self.train_args.epochs
-        self.validate = self.train_args.validate
-        self.save_when_better = self.train_args.save_when_better
+        self.n_epochs = train_args.epochs
+        self.validate = train_args.validate
+        self.save_when_better = train_args.save_when_better
+        self.log_per_step = train_args.log_per_step
+        self.log_validation = train_args.log_validation
+
+    def train(self, network, train_data, dev_data):
+        """
+        :param network: the model controller
+        :param train_data: raw data for training
+        :param dev_data: raw data for validation
+        :return:
+        """
+        train_x, train_y = network.prepare_input(train_data.train_set, train_data.train_label)
 
-    def train(self, network, data, dev_data):
-        train_x, train_y = network.prepare_input(data.train_set, data.train_label)
-        valid_x, valid_y = network.prepare_input(dev_data.valid_set, dev_data.valid_label)
+        network.mode(test=False)  # turn on the train mode
 
         iterations, train_batch_generator = self.batchify(train_x, train_y)
-        loss_history = list()
-        network.mode(test=False)
 
-        test_args = "..."
+        test_args = Tester.TestConfig(save_output=True, validate_in_training=True,
+                                      save_dev_input=True, save_loss=True, batch_size=16)
         evaluator = Tester(test_args)
+
         best_loss = 1e10
+        loss_history = list()
 
         for epoch in range(self.n_epochs):
 
+            network.define_optimizer()
             for step in range(iterations):
                 batch_x, batch_y = train_batch_generator.__next__()
 
@@ -39,14 +51,18 @@ class Trainer(Action):
 
                 loss = network.loss(batch_y, prediction)
                 network.grad_backward()
-                loss_history.append(loss)
-                self.log(self.make_log(epoch, step, loss))
+
+                if step % self.log_per_step == 0:
+                    loss_history.append(loss)
+                    self.log(self.make_log(epoch, step, loss))
 
             #################### evaluate over dev set  ###################
             if self.validate:
-                evaluator.test(network, [valid_x, valid_y])
+                # give all controls to tester
+                evaluator.test(network, dev_data)
 
-                self.log(self.make_valid_log(epoch, evaluator.loss))
+                if self.log_validation:
+                    self.log(self.make_valid_log(epoch, evaluator.loss))
                 if evaluator.loss < best_loss:
                     best_loss = evaluator.loss
                     if self.save_when_better:
@@ -54,15 +70,20 @@ class Trainer(Action):
 
         # finish training
 
-    @staticmethod
-    def prepare_training(network, data):
-        return network.prepare_training(data)
-
     def make_log(self, *args):
-        print("logged")
+        return "make a log"
 
     def make_valid_log(self, *args):
-        print("logged")
+        return "make a valid log"
 
     def save_model(self, model):
-        print("model saved")
+        model.save()
+
+    def load_data(self, data_name):
+        print("load data")
+
+    def load_config(self, args):
+        raise NotImplementedError
+
+    def load_dataset(self, args):
+        raise NotImplementedError

loader/base_loader.py

@@ -13,3 +13,19 @@ class BaseLoader(object):
         with open(self.data_path, "r", encoding="utf-8") as f:
             text = f.read()
         return text
+
+
+class ToyLoader0(BaseLoader):
+    """
+        For charLM
+    """
+
+    def __init__(self, name, path):
+        super(ToyLoader0, self).__init__(name, path)
+
+    def load(self):
+        with open(self.data_path, 'r') as f:
+            corpus = f.read().lower()
+        import re
+        corpus = re.sub(r"<unk>", "unk", corpus)
+        return corpus.split()

model/char_language_model.py

@@ -14,6 +14,8 @@ from model.base_model import BaseModel
 class CharLM(BaseModel):
     """
         Controller of the Character-level Neural Language Model
+        To do:
+        - where the data goes, call data savers.
     """
 
     def __init__(self):
@@ -28,12 +30,15 @@ class CharLM(BaseModel):
         self.lstm_batch_size = 20
         self.vocab_size = 100
         self.num_char = 150
+        self.max_word_len = 10
+        self.num_epoch = 10
+        self.old_PPL = 100000
+        self.best_PPL = 100000
 
         self.data = None  # named tuple to store all data set
         self.data_ready = False
         self.criterion = nn.CrossEntropyLoss()
         self.loss = None
-        self.optimizer = optim.SGD(self.parameters(), lr=learning_rate, momentum=0.85)
         self.use_gpu = False
         # word_emb_dim == hidden_size / num of hidden units
         self.hidden = (to_var(torch.zeros(2, self.lstm_batch_size, self.word_embed_dim)),
@@ -44,10 +49,17 @@ class CharLM(BaseModel):
                             self.vocab_size,
                             self.num_char,
                             use_gpu=self.use_gpu)
+        for param in self.model.parameters():
+            nn.init.uniform(param.data, -0.05, 0.05)
+
+        self.learning_rate = 0.1
+        self.optimizer = None
 
     def prepare_input(self, raw_text):
         """
-            Do some preparation jobs. Transform raw data into input vectors.
+        :param raw_text: raw input data
+        :return: torch.Tensor, torch.Tensor
+        feature matrix, label vector
         """
         if not self.data_ready:
             # To do: These need to be dropped out from here. (below)
@@ -82,10 +94,20 @@ class CharLM(BaseModel):
             DataTuple = namedtuple("DataTuple", ["feature", "label"])
             self.data = DataTuple(feature=input_vec, label=input_label)
 
-        return self.data.feature, self.data.label
+        feature_input = torch.from_numpy(self.data.feature)
+        label_input = torch.from_numpy(self.data.label)
+        num_seq = feature_input.size()[0] // self.lstm_seq_len
+        feature_input = feature_input[:num_seq * self.lstm_seq_len, :]
+        feature_input = feature_input.view(-1, self.lstm_seq_len, self.max_word_len + 2)
+        self.num_iter_per_epoch = feature_input.size()[0] // self.lstm_batch_size
+
+        return feature_input, label_input
 
     def mode(self, test=False):
-        raise NotImplementedError
+        if test:
+            self.model.eval()
+        else:
+            self.model.train()
 
     def data_forward(self, x):
         # detach hidden state of LSTM from last batch
@@ -103,6 +125,13 @@ class CharLM(BaseModel):
         self.loss = self.criterion(predict, to_var(truth))
         return self.loss
 
+    def define_optimizer(self):
+        # redefine optimizer for every new epoch
+        self.optimizer = optim.SGD(self.model.parameters(), lr=self.learning_rate, momentum=0.85)
+
+    def save(self):
+        torch.save(self.model, "cache/model.pkl")
+
     @staticmethod
     def preprocess():
         word_dict, char_dict = create_word_char_dict("valid.txt", "train.txt", "tests.txt")
@@ -122,23 +151,6 @@ class CharLM(BaseModel):
         torch.save(objects, "cache/prep.pt")
         print("Preprocess done.")
 
-    def forward(self, x, hidden):
-        lstm_batch_size = x.size()[0]
-        lstm_seq_len = x.size()[1]
-        x = x.contiguous().view(-1, x.size()[2])
-        x = self.char_embed(x)
-        x = torch.transpose(x.view(x.size()[0], 1, x.size()[1], -1), 2, 3)
-        x = self.conv_layers(x)
-        x = self.batch_norm(x)
-        x = self.highway1(x)
-        x = self.highway2(x)
-        x = x.contiguous().view(lstm_batch_size, lstm_seq_len, -1)
-        x, hidden = self.lstm(x, hidden)
-        x = self.dropout(x)
-        x = x.contiguous().view(lstm_batch_size * lstm_seq_len, -1)
-        x = self.linear(x)
-        return x, hidden
-
 
 """
     Global Functions

saver/logger.py

@@ -8,4 +8,5 @@ class Logger(BaseSaver):
         super(Logger, self).__init__(save_path)
 
     def log(self, string):
-        raise NotImplementedError
+        with open(self.save_path, "a") as f:
+            f.write(string)

tests/test_charlm.py

@@ -0,0 +1,30 @@
+from action.tester import Tester
+from action.trainer import Trainer
+from loader.base_loader import ToyLoader0
+from model.char_language_model import CharLM
+
+
+def test_charlm():
+    train_config = Trainer.TrainConfig(epochs=1, validate=True, save_when_better=True,
+                                       log_per_step=10, log_validation=True)
+    trainer = Trainer(train_config)
+
+    model = CharLM()
+    train_data = ToyLoader0("load_train", "path_to_train_file").load()
+    valid_data = ToyLoader0("load_valid", "path_to_valid_file").load()
+
+    trainer.train(model, train_data, valid_data)
+
+    trainer.save_model(model)
+
+    test_config = Tester.TestConfig(save_output=True, validate_in_training=True,
+                                    save_dev_input=True, save_loss=True, batch_size=16)
+    tester = Tester(test_config)
+
+    test_data = ToyLoader0("load_test", "path_to_test").load()
+
+    tester.test(model, test_data)
+
+
+if __name__ == "__main__":
+    test_charlm()