From 40b1a9566d29f92aad546855f5ff4853d0ea47c9 Mon Sep 17 00:00:00 2001
From: ly119399 <ly119399@alibaba-inc.com>
Date: Tue, 19 Jul 2022 22:39:54 +0800
Subject: [PATCH] [to #42322933] remove dependency en_core_web_sm from nlp.txt
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

将nlp.txt中的en_core_web_sm依赖删除，在代码中下载响应的依赖
        Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/9432253
---
 .../preprocessors/space/fields/gen_field.py     | 17 +++++++++++++++++
 requirements/nlp.txt                            |  1 -
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/modelscope/preprocessors/space/fields/gen_field.py b/modelscope/preprocessors/space/fields/gen_field.py
index 28928029..9b408acc 100644
--- a/modelscope/preprocessors/space/fields/gen_field.py
+++ b/modelscope/preprocessors/space/fields/gen_field.py
@@ -7,11 +7,14 @@ from itertools import chain
 
 import numpy as np
 
+from ....utils.logger import get_logger
 from ....utils.nlp.space import ontology, utils
 from ....utils.nlp.space.db_ops import MultiWozDB
 from ....utils.nlp.space.utils import list2np
 from ..tokenizer import Tokenizer
 
+logger = get_logger()
+
 
 class BPETextField(object):
 
@@ -306,7 +309,21 @@ class MultiWOZBPETextField(BPETextField):
 
     def __init__(self, model_dir, config):
         super(MultiWOZBPETextField, self).__init__(config)
+
         import spacy
+        try:
+            import en_core_web_sm
+        except ImportError:
+            logger.warn('Miss module en_core_web_sm!')
+            logger.warn('We will download en_core_web_sm automatically.')
+            try:
+                spacy.cli.download('en_core_web_sm')
+            except Exception as e:
+                logger.error(e)
+                raise ImportError(
+                    'Download en_core_web_sm error. '
+                    'Please use \'python -m spacy download en_core_web_sm\' to download it by yourself!'
+                )
         self.nlp = spacy.load('en_core_web_sm')
 
         self.db = MultiWozDB(
diff --git a/requirements/nlp.txt b/requirements/nlp.txt
index 4c881909..a133e451 100644
--- a/requirements/nlp.txt
+++ b/requirements/nlp.txt
@@ -1,3 +1,2 @@
-http://ait-public.oss-cn-hangzhou-zmf.aliyuncs.com/jizhu/en_core_web_sm-2.3.1.tar.gz
 sofa>=1.0.5
 spacy>=2.3.5