fastNLP/test/test_POS_pipeline.py

import sys

sys.path.append("..")

from fastNLP.loader.config_loader import ConfigLoader, ConfigSection
from fastNLP.action.trainer import POSTrainer
from fastNLP.loader.dataset_loader import POSDatasetLoader
from fastNLP.loader.preprocess import POSPreprocess
from fastNLP.saver.model_saver import ModelSaver
from fastNLP.loader.model_loader import ModelLoader
from fastNLP.action.tester import POSTester
from fastNLP.models.sequence_modeling import SeqLabeling
from fastNLP.action.inference import Inference

data_name = "people.txt"
data_path = "data_for_tests/people.txt"
pickle_path = "data_for_tests"


def test_infer():
    # Define the same model
    model = SeqLabeling(hidden_dim=train_args["rnn_hidden_units"], rnn_num_layer=train_args["rnn_layers"],
                        num_classes=train_args["num_classes"], vocab_size=train_args["vocab_size"],
                        word_emb_dim=train_args["word_emb_dim"], bi_direction=train_args["rnn_bi_direction"],
                        rnn_mode="gru", dropout=train_args["dropout"], use_crf=train_args["use_crf"])

    # Dump trained parameters into the model
    ModelLoader("arbitrary_name", "./saved_model.pkl").load_pytorch(model)
    print("model loaded!")

    # Data Loader
    pos_loader = POSDatasetLoader(data_name, data_path)
    infer_data = pos_loader.load_lines()

    # Preprocessor
    POSPreprocess(infer_data, pickle_path)

    # Inference interface
    infer = Inference()
    results = infer.predict(model, infer_data)


if __name__ == "__main__":
    # Config Loader
    train_args = ConfigSection()
    ConfigLoader("config.cfg", "").load_config("./data_for_tests/config", {"POS": train_args})

    # Data Loader
    pos_loader = POSDatasetLoader(data_name, data_path)
    train_data = pos_loader.load_lines()

    # Preprocessor
    p = POSPreprocess(train_data, pickle_path)
    train_args["vocab_size"] = p.vocab_size
    train_args["num_classes"] = p.num_classes

    # Trainer
    trainer = POSTrainer(train_args)

    # Model
    model = SeqLabeling(hidden_dim=train_args["rnn_hidden_units"], rnn_num_layer=train_args["rnn_layers"],
                        num_classes=train_args["num_classes"], vocab_size=train_args["vocab_size"],
                        word_emb_dim=train_args["word_emb_dim"], bi_direction=train_args["rnn_bi_direction"],
                        rnn_mode="gru", dropout=train_args["dropout"], use_crf=train_args["use_crf"])

    # Start training
    trainer.train(model)
    print("Training finished!")

    # Saver
    saver = ModelSaver("./saved_model.pkl")
    saver.save_pytorch(model)
    print("Model saved!")

    del model, trainer, pos_loader

    # Define the same model
    model = SeqLabeling(hidden_dim=train_args["rnn_hidden_units"], rnn_num_layer=train_args["rnn_layers"],
                        num_classes=train_args["num_classes"], vocab_size=train_args["vocab_size"],
                        word_emb_dim=train_args["word_emb_dim"], bi_direction=train_args["rnn_bi_direction"],
                        rnn_mode="gru", dropout=train_args["dropout"], use_crf=train_args["use_crf"])

    # Dump trained parameters into the model
    ModelLoader("arbitrary_name", "./saved_model.pkl").load_pytorch(model)
    print("model loaded!")

    # Load test configuration
    test_args = ConfigSection()
    ConfigLoader("config.cfg", "").load_config("./data_for_tests/config", {"POS_test": test_args})

    # Tester
    tester = POSTester(test_args)

    # Start testing
    tester.test(model)

    # print test results
    print(tester.show_matrices())
    print("model tested!")