yuze.zyz 3 years ago
parent
commit
a0d2581028
1 changed files with 11 additions and 0 deletions
  1. +11
    -0
      modelscope/preprocessors/nlp/sentence_piece_preprocessor.py

+ 11
- 0
modelscope/preprocessors/nlp/sentence_piece_preprocessor.py View File

@@ -38,3 +38,14 @@ class SentencePiecePreprocessor(Preprocessor):

def __call__(self, data: str) -> torch.Tensor:
return torch.tensor(self.tokenizer.encode([data]), dtype=torch.long)

def decode(self, tokens, **kwargs):
"""Decode the tokens to real text.

Args:
tokens: The output tokens from model's `forward` and `generate`

Returns:
The actual text.
"""
return self.tokenizer.decode(tokens)

Loading…
Cancel
Save