| @@ -91,7 +91,7 @@ class EasyStatisticalChecker(BaseChecker): | |||||
| if stat_spec.get_z().shape[1:] != input_shape: | if stat_spec.get_z().shape[1:] != input_shape: | ||||
| logger.warning(f"The learnware [{learnware.id}] input dimension mismatch with stat specification.") | logger.warning(f"The learnware [{learnware.id}] input dimension mismatch with stat specification.") | ||||
| return self.INVALID_LEARNWARE | return self.INVALID_LEARNWARE | ||||
| def generate_random_text_list(num, text_type="en", min_len=10, max_len=1000): | def generate_random_text_list(num, text_type="en", min_len=10, max_len=1000): | ||||
| text_list = [] | text_list = [] | ||||
| for i in range(num): | for i in range(num): | ||||
| @@ -106,7 +106,7 @@ class EasyStatisticalChecker(BaseChecker): | |||||
| else: | else: | ||||
| raise ValueError("Type should be en or zh") | raise ValueError("Type should be en or zh") | ||||
| return text_list | return text_list | ||||
| if is_text: | if is_text: | ||||
| inputs = generate_random_text_list(10) | inputs = generate_random_text_list(10) | ||||
| else: | else: | ||||
| @@ -1,3 +1,9 @@ | |||||
| from .utils import generate_stat_spec, generate_rkme_spec, generate_rkme_image_spec | from .utils import generate_stat_spec, generate_rkme_spec, generate_rkme_image_spec | ||||
| from .base import Specification, BaseStatSpecification | from .base import Specification, BaseStatSpecification | ||||
| from .regular import RegularStatsSpecification, RKMEStatSpecification, RKMETableSpecification, RKMEImageSpecification, RKMETextSpecification | |||||
| from .regular import ( | |||||
| RegularStatsSpecification, | |||||
| RKMEStatSpecification, | |||||
| RKMETableSpecification, | |||||
| RKMEImageSpecification, | |||||
| RKMETextSpecification, | |||||
| ) | |||||
| @@ -10,6 +10,7 @@ logger = get_module_logger("RKMETextSpecification", "INFO") | |||||
| class RKMETextSpecification(RKMETableSpecification): | class RKMETextSpecification(RKMETableSpecification): | ||||
| """Reduced Kernel Mean Embedding (RKME) Specification for Text""" | """Reduced Kernel Mean Embedding (RKME) Specification for Text""" | ||||
| def __init__(self, gamma: float = 0.1, cuda_idx: int = -1): | def __init__(self, gamma: float = 0.1, cuda_idx: int = -1): | ||||
| RKMETableSpecification.__init__(self, gamma, cuda_idx) | RKMETableSpecification.__init__(self, gamma, cuda_idx) | ||||
| self.language = [] | self.language = [] | ||||
| @@ -59,7 +60,7 @@ class RKMETextSpecification(RKMETableSpecification): | |||||
| @staticmethod | @staticmethod | ||||
| def get_language_ids(X): | def get_language_ids(X): | ||||
| try: | try: | ||||
| text = ' '.join(X) | |||||
| text = " ".join(X) | |||||
| lang = langdetect.detect(text) | lang = langdetect.detect(text) | ||||
| langs = langdetect.detect_langs(text) | langs = langdetect.detect_langs(text) | ||||
| return [l.lang for l in langs] | return [l.lang for l in langs] | ||||