|
|
|
@@ -210,7 +210,7 @@ class Preprocessor(ABC): |
|
|
|
sub_key = 'train' if preprocessor_mode == ModeKeys.TRAIN else 'val' |
|
|
|
|
|
|
|
if not hasattr(cfg, 'preprocessor') or len(cfg.preprocessor) == 0: |
|
|
|
logger.error('No preprocessor field found in cfg.') |
|
|
|
logger.warn('No preprocessor field found in cfg.') |
|
|
|
preprocessor_cfg = ConfigDict() |
|
|
|
else: |
|
|
|
preprocessor_cfg = cfg.preprocessor |
|
|
|
@@ -219,9 +219,8 @@ class Preprocessor(ABC): |
|
|
|
if sub_key in preprocessor_cfg: |
|
|
|
sub_cfg = getattr(preprocessor_cfg, sub_key) |
|
|
|
else: |
|
|
|
logger.error( |
|
|
|
f'No {sub_key} key and type key found in ' |
|
|
|
f'preprocessor domain of configuration.json file.') |
|
|
|
logger.warn(f'No {sub_key} key and type key found in ' |
|
|
|
f'preprocessor domain of configuration.json file.') |
|
|
|
sub_cfg = preprocessor_cfg |
|
|
|
else: |
|
|
|
sub_cfg = preprocessor_cfg |
|
|
|
@@ -237,7 +236,7 @@ class Preprocessor(ABC): |
|
|
|
|
|
|
|
preprocessor = build_preprocessor(sub_cfg, field_name) |
|
|
|
else: |
|
|
|
logger.error( |
|
|
|
logger.warn( |
|
|
|
f'Cannot find available config to build preprocessor at mode {preprocessor_mode}, ' |
|
|
|
f'current config: {sub_cfg}. trying to build by task and model information.' |
|
|
|
) |
|
|
|
@@ -245,13 +244,13 @@ class Preprocessor(ABC): |
|
|
|
model_type = model_cfg.type if hasattr( |
|
|
|
model_cfg, 'type') else getattr(model_cfg, 'model_type', None) |
|
|
|
if task is None or model_type is None: |
|
|
|
logger.error( |
|
|
|
logger.warn( |
|
|
|
f'Find task: {task}, model type: {model_type}. ' |
|
|
|
f'Insufficient information to build preprocessor, skip building preprocessor' |
|
|
|
) |
|
|
|
return None |
|
|
|
if (model_type, task) not in PREPROCESSOR_MAP: |
|
|
|
logger.error( |
|
|
|
logger.warn( |
|
|
|
f'No preprocessor key {(model_type, task)} found in PREPROCESSOR_MAP, ' |
|
|
|
f'skip building preprocessor.') |
|
|
|
return None |
|
|
|
|