!152 fix runtest.sh for python ut and fallback PR140

Merge pull request !152 from changzherui/fix_runtest

mindspore/dataset/engine/datasets.py

@@ -38,7 +38,7 @@ from .iterators import DictIterator, TupleIterator
 from .validators import check, check_batch, check_shuffle, check_map, check_repeat, check_zip, check_rename, \
     check_project, check_imagefolderdatasetv2, check_mnist_cifar_dataset, check_manifestdataset, \
     check_tfrecorddataset, check_vocdataset, check_celebadataset, check_minddataset, check_generatordataset, \
-    check_zip_dataset, check_add_column, check_columns
+    check_zip_dataset
 from ..core.datatypes import mstype_to_detype, mstypelist_to_detypelist
 
 try:
@@ -2334,20 +2334,13 @@ class Schema:
             self.dataset_type = ''
             self.num_rows = 0
         else:
-            if not os.path.isfile(schema_file) or not os.access(schema_file, os.R_OK):
-                raise ValueError("The file %s does not exist or permission denied!" % schema_file)
             try:
                 with open(schema_file, 'r') as load_f:
                     json_obj = json.load(load_f)
+                    self.from_json(json_obj)
             except json.decoder.JSONDecodeError:
-                raise RuntimeError("Schema file failed to load.")
-            except UnicodeDecodeError:
-                raise RuntimeError("Schema file failed to decode.")
-            except Exception:
-                raise RuntimeError("Schema file failed to open.")
-            self.from_json(json_obj)
+                raise RuntimeError("Schema file failed to load")
 
-    @check_add_column
     def add_column(self, name, de_type, shape=None):
         """
         Add new column to the schema.
@@ -2366,8 +2359,10 @@ class Schema:
         if isinstance(de_type, typing.Type):
             de_type = mstype_to_detype(de_type)
             new_column["type"] = str(de_type)
-        else:
+        elif isinstance(de_type, str):
             new_column["type"] = str(DataType(de_type))
+        else:
+            raise ValueError("Unknown column type")
 
         if shape is not None:
             new_column["shape"] = shape
@@ -2404,7 +2399,6 @@ class Schema:
             RuntimeError: If column's name field is missing.
             RuntimeError: If column's type field is missing.
         """
-        check_columns(columns, columns)
         self.columns = []
         for col in columns:
             name = None
@@ -2449,8 +2443,6 @@ class Schema:
             RuntimeError: if dataset type is missing in the object.
             RuntimeError: if columns are missing in the object.
         """
-        if not isinstance(json_obj, dict) or json_obj is None:
-            raise ValueError("Expected non-empty dict.")
         for k, v in json_obj.items():
             if k == "datasetType":
                 self.dataset_type = v

mindspore/dataset/engine/validators.py

@@ -19,15 +19,10 @@ import inspect as ins
 import os
 from functools import wraps
 from multiprocessing import cpu_count
-from mindspore._c_expression import typing
 from . import samplers
 from . import datasets
 
 INT32_MAX = 2147483647
-valid_detype = [
-    "bool", "int8", "int16", "int32", "int64", "uint8", "uint16",
-    "uint32", "uint64", "float16", "float32", "float64"
-]
 
 
 def check(method):
@@ -193,12 +188,6 @@ def check(method):
     return wrapper
 
 
-def check_valid_detype(type_):
-    if type_ not in valid_detype:
-        raise ValueError("Unknown column type")
-    return True
-
-
 def check_filename(path):
     """
     check the filename in the path
@@ -754,41 +743,3 @@ def check_project(method):
         return method(*args, **kwargs)
 
     return new_method
-
-
-def check_shape(shape, name):
-    if isinstance(shape, list):
-        for element in shape:
-            if not isinstance(element, int):
-                raise TypeError(
-                    "Each element in {0} should be of type int. Got {1}.".format(name, type(element)))
-    else:
-        raise TypeError("Expected int list.")
-
-
-def check_add_column(method):
-    """check the input arguments of add_column."""
-    @wraps(method)
-    def new_method(*args, **kwargs):
-        param_dict = make_param_dict(method, args, kwargs)
-
-        # check name; required argument
-        name = param_dict.get("name")
-        if not isinstance(name, str) or not name:
-            raise TypeError("Expected non-empty string.")
-
-        # check type; required argument
-        de_type = param_dict.get("de_type")
-        if not isinstance(de_type, str) or not de_type:
-            raise TypeError("Expected non-empty string.")
-        if not isinstance(de_type, typing.Type) and not check_valid_detype(de_type):
-            raise ValueError("Unknown column type.")
-
-        # check shape
-        shape = param_dict.get("shape")
-        if shape is not None:
-            check_shape(shape, "shape")
-
-        return method(*args, **kwargs)
-
-    return new_method

tests/ut/python/ops/test_ops_check.py

@@ -62,7 +62,7 @@ def test_net_without_construct():
     try:
         _executor.compile(net, inp)
     except RuntimeError as err:
-        if str(err).find("unsupported syntax 'Raise' at ") >= 0:
+        if str(err).find("Unsupported syntax 'Raise' at ") >= 0:
             print(str(err))
         else:
             raise err
@@ -86,7 +86,7 @@ def test_net_with_raise():
     try:
         _executor.compile(net, inp)
     except RuntimeError as err:
-        if str(err).find("unsupported syntax 'Raise' at ") >= 0:
+        if str(err).find("Unsupported syntax 'Raise' at ") >= 0:
             print(str(err))
         else:
             raise err

tests/ut/python/runtest.sh

@@ -40,12 +40,8 @@ fi
 
 if [ $# -gt 0 ]; then
     pytest -s --ignore=$1/pynative_mode --ignore=$1/parallel  --ignore=$1/train  $IGNORE_EXEC $1
-    pytest -n 4 --dist=loadfile -v $1/parallel
-    pytest -n 4 --dist=loadfile -v $1/train
 else
     pytest --ignore=$CURRPATH/pynative_mode --ignore=$CURRPATH/parallel --ignore=$CURRPATH/train $IGNORE_EXEC $CURRPATH
-    pytest -n 4 --dist=loadfile -v $CURRPATH/parallel
-    pytest -n 4 --dist=loadfile -v $CURRPATH/train
 fi
 
 RET=$?
@@ -57,6 +53,18 @@ if [ ${RET} -ne 0 ]; then
     exit ${RET}
 fi
 
+if [ $# -gt 0 ]; then
+    pytest -n 4 --dist=loadfile -v $1/parallel $1/train
+else
+    pytest -n 4 --dist=loadfile -v $CURRPATH/parallel $CURRPATH/train
+fi
+
+RET=$?
+
+if [ ${RET} -ne 0 ]; then
+    exit ${RET}
+fi
+
 if [ $# -gt 0 ]; then
     pytest -s $1/pynative_mode
 else