[verify] sst2loader

reproduction/text_classification/data/sstloader.py

@@ -0,0 +1,198 @@
+from typing import Iterable
+from nltk import Tree
+from fastNLP.io.base_loader import DataInfo, DataSetLoader
+from fastNLP.core.vocabulary import VocabularyOption, Vocabulary
+from fastNLP import DataSet
+from fastNLP import Instance
+from fastNLP.io.embed_loader import EmbeddingOption, EmbedLoader
+import csv
+from typing import Union, Dict
+from reproduction.utils import check_dataloader_paths, get_tokenizer
+
+
+class SSTLoader(DataSetLoader):
+    URL = 'https://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip'
+    DATA_DIR = 'sst/'
+
+    """
+    别名：:class:`fastNLP.io.SSTLoader` :class:`fastNLP.io.dataset_loader.SSTLoader`
+    读取SST数据集, DataSet包含fields::
+        words: list(str) 需要分类的文本
+        target: str 文本的标签
+    数据来源: https://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip
+    :param subtree: 是否将数据展开为子树，扩充数据量. Default: ``False``
+    :param fine_grained: 是否使用SST-5标准，若 ``False`` , 使用SST-2。Default: ``False``
+    """
+    def __init__(self, subtree=False, fine_grained=False):
+        self.subtree = subtree
+        tag_v = {'0': 'very negative', '1': 'negative', '2': 'neutral',
+                 '3': 'positive', '4': 'very positive'}
+        if not fine_grained:
+            tag_v['0'] = tag_v['1']
+            tag_v['4'] = tag_v['3']
+        self.tag_v = tag_v
+
+    def _load(self, path):
+        """
+        :param str path: 存储数据的路径
+        :return: 一个 :class:`~fastNLP.DataSet` 类型的对象
+        """
+        datalist = []
+        with open(path, 'r', encoding='utf-8') as f:
+            datas = []
+            for l in f:
+                datas.extend([(s, self.tag_v[t])
+                              for s, t in self._get_one(l, self.subtree)])
+        ds = DataSet()
+        for words, tag in datas:
+            ds.append(Instance(words=words, target=tag))
+        return ds
+
+
+    @staticmethod
+    def _get_one(data, subtree):
+        tree = Tree.fromstring(data)
+        if subtree:
+            return [(t.leaves(), t.label()) for t in tree.subtrees()]
+        return [(tree.leaves(), tree.label())]
+
+
+    def process(self,
+                paths,
+                train_ds: Iterable[str] = None,
+                src_vocab_op: VocabularyOption = None,
+                tgt_vocab_op: VocabularyOption = None,
+                src_embed_op: EmbeddingOption = None):
+        input_name, target_name = 'words', 'target'
+        src_vocab = Vocabulary() if src_vocab_op is None else Vocabulary(**src_vocab_op)
+        tgt_vocab = Vocabulary(unknown=None, padding=None) \
+            if tgt_vocab_op is None else Vocabulary(**tgt_vocab_op)
+
+        info = DataInfo(datasets=self.load(paths))
+        _train_ds = [info.datasets[name]
+                     for name in train_ds] if train_ds else info.datasets.values()
+        src_vocab.from_dataset(*_train_ds, field_name=input_name)
+        tgt_vocab.from_dataset(*_train_ds, field_name=target_name)
+        src_vocab.index_dataset(
+            *info.datasets.values(),
+            field_name=input_name, new_field_name=input_name)
+        tgt_vocab.index_dataset(
+            *info.datasets.values(),
+            field_name=target_name, new_field_name=target_name)
+        info.vocabs = {
+            input_name: src_vocab,
+            target_name: tgt_vocab
+        }
+
+
+        if src_embed_op is not None:
+            src_embed_op.vocab = src_vocab
+            init_emb = EmbedLoader.load_with_vocab(**src_embed_op)
+            info.embeddings[input_name] = init_emb
+
+
+        for name, dataset in info.datasets.items():
+            dataset.set_input(input_name)
+            dataset.set_target(target_name)
+        return info
+
+
+
+class sst2Loader(DataSetLoader):
+    '''
+    数据来源"SST":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSST-2.zip?alt=media&token=aabc5f6b-e466-44a2-b9b4-cf6337f84ac8',
+    '''
+
+    def __init__(self):
+        super(sst2Loader, self).__init__()
+        self.tokenizer = get_tokenizer()
+
+
+    def _load(self, path: str) -> DataSet:
+        ds = DataSet()
+        all_count=0
+        csv_reader = csv.reader(open(path, encoding='utf-8'),delimiter='\t')
+        skip_row = 0
+        for idx,row in enumerate(csv_reader):
+            if idx<=skip_row:
+                continue
+            target = row[1]
+            words=self.tokenizer(row[0])
+            ds.append(Instance(words=words,target=target))
+            all_count+=1
+        print("all count:", all_count)
+        return ds
+
+
+
+    def process(self,
+                paths: Union[str, Dict[str, str]],
+                src_vocab_opt: VocabularyOption = None,
+                tgt_vocab_opt: VocabularyOption = None,
+                src_embed_opt: EmbeddingOption = None,
+                char_level_op=False):
+
+        paths = check_dataloader_paths(paths)
+        datasets = {}
+        info = DataInfo()
+        for name, path in paths.items():
+            dataset = self.load(path)
+            datasets[name] = dataset
+
+        def wordtochar(words):
+            chars = []
+            for word in words:
+                word = word.lower()
+                for char in word:
+                    chars.append(char)
+                chars.append('')
+            chars.pop()
+            return chars
+
+        input_name, target_name = 'words', 'target'
+        info.vocabs={}
+
+        # 就分隔为char形式
+        if char_level_op:
+            for dataset in datasets.values():
+                dataset.apply_field(wordtochar, field_name="words", new_field_name='chars')
+        src_vocab = Vocabulary() if src_vocab_opt is None else Vocabulary(**src_vocab_opt)
+        src_vocab.from_dataset(datasets['train'], field_name='words')
+        src_vocab.index_dataset(*datasets.values(), field_name='words')
+
+        tgt_vocab = Vocabulary(unknown=None, padding=None) \
+            if tgt_vocab_opt is None else Vocabulary(**tgt_vocab_opt)
+        tgt_vocab.from_dataset(datasets['train'], field_name='target')
+        tgt_vocab.index_dataset(*datasets.values(), field_name='target')
+
+
+        info.vocabs = {
+            "words": src_vocab,
+            "target": tgt_vocab
+        }
+
+        info.datasets = datasets
+
+        if src_embed_opt is not None:
+            embed = EmbedLoader.load_with_vocab(**src_embed_opt, vocab=src_vocab)
+            info.embeddings['words'] = embed
+
+        for name, dataset in info.datasets.items():
+            dataset.set_input("words")
+            dataset.set_target("target")
+
+        return info
+
+
+
+if __name__=="__main__":
+    datapath = {"train": "/remote-home/ygwang/workspace/GLUE/SST-2/train.tsv",
+                "dev": "/remote-home/ygwang/workspace/GLUE/SST-2/dev.tsv"}
+    datainfo=sst2Loader().process(datapath,char_level_op=True)
+    #print(datainfo.datasets["train"])
+
+    len_count = 0
+    for instance in datainfo.datasets["train"]:
+        len_count += len(instance["chars"])
+    ave_len = len_count / len(datainfo.datasets["train"])
+    print(ave_len)