Browse Source

fix PythonTokenizer bug in windows

pull/14518/head
xulei2020 4 years ago
parent
commit
71380d8760
1 changed files with 2 additions and 2 deletions
  1. +2
    -2
      mindspore/dataset/text/__init__.py

+ 2
- 2
mindspore/dataset/text/__init__.py View File

@@ -19,7 +19,7 @@ utils provides some general methods for NLP text processing.
"""
import platform
from .transforms import Lookup, JiebaTokenizer, UnicodeCharTokenizer, Ngram, WordpieceTokenizer, TruncateSequencePair, \
ToNumber, SlidingWindow, SentencePieceTokenizer
ToNumber, SlidingWindow, SentencePieceTokenizer, PythonTokenizer
from .utils import to_str, to_bytes, JiebaMode, Vocab, NormalizeForm, SentencePieceVocab, SentencePieceModel, \
SPieceTokenizerOutType, SPieceTokenizerLoadType

@@ -33,7 +33,7 @@ __all__ = [

if platform.system().lower() != 'windows':
from .transforms import UnicodeScriptTokenizer, WhitespaceTokenizer, CaseFold, NormalizeUTF8, \
RegexReplace, RegexTokenizer, BasicTokenizer, BertTokenizer, PythonTokenizer
RegexReplace, RegexTokenizer, BasicTokenizer, BertTokenizer

__all__.append(["UnicodeScriptTokenizer", "WhitespaceTokenizer", "CaseFold", "NormalizeUTF8",
"RegexReplace", "RegexTokenizer", "BasicTokenizer", "BertTokenizer"])

Loading…
Cancel
Save