!30088 [MD] Disable shared memory in Python Multiprocessing UTs for CI

Merge pull request !30088 from cathwong/ckw_ut_fixup_python_multiproc

tests/ut/python/dataset/test_pad_batch.py

@@ -211,12 +211,15 @@ def test_pad_via_map_multiproc():
     Expectation: pad_batch and numpy.pad results are the same
     """
     # Note: Reduce shared memory needed (for CI) by using small num_parallel_workers and max_rowsize values
+    # and disabling the shared memory optimization
+    mem_original = ds.config.get_enable_shared_mem()
+    ds.config.set_enable_shared_mem(False)
     res_from_map = pad_map_config(2, True, 1)
     res_from_batch = pad_batch_config()
     assert len(res_from_batch) == len(res_from_batch)
     for i, _ in enumerate(res_from_map):
         np.testing.assert_array_equal(res_from_map[i], res_from_batch[i])
-
+    ds.config.set_enable_shared_mem(mem_original)
 
 
 if __name__ == '__main__':

tests/ut/python/dataset/test_pyfunc_multiprocess.py

@@ -29,7 +29,7 @@ PYFUNCMAP_DATA_DIR = ["../data/dataset/testPyfuncMap/data.data"]
 PYFUNCMAP_SCHEMA_DIR = "../data/dataset/testPyfuncMap/schema.json"
 
 
-def test_pyfunc_multiproc_shrmem():
+def skip_test_pyfunc_multiproc_shrmem():
     """
     Feature: PyFunc in Map op
     Description: Test python_multiprocessing=True with shared memory enabled
@@ -123,8 +123,11 @@ def test_pyfunc_multiproc_max_rowsize_small():
     Expectation: Number of return data rows is correct
     """
     # Reduce memory needed by reducing queue size
+    # and disabling the shared memory optimization
     prefetch_original = ds.config.get_prefetch_size()
     ds.config.set_prefetch_size(1)
+    mem_original = ds.config.get_enable_shared_mem()
+    ds.config.set_enable_shared_mem(False)
 
     mydata1 = create_dataset_pyop_multiproc(num_parallel_workers=2, max_rowsize=1, num_samples=500)
     mycount1 = 0
@@ -133,6 +136,7 @@ def test_pyfunc_multiproc_max_rowsize_small():
     assert mycount1 == 15
 
     ds.config.set_prefetch_size(prefetch_original)
+    ds.config.set_enable_shared_mem(mem_original)
 
 
 def test_pyfunc_multiproc_max_rowsize_large():
@@ -260,7 +264,7 @@ def test_pyfunc_multiproc_mainproc_exception():
 
 
 if __name__ == '__main__':
-    test_pyfunc_multiproc_shrmem()
+    skip_test_pyfunc_multiproc_shrmem()
     test_pyfunc_multiproc_noshrmem()
     test_pyfunc_multiproc_max_rowsize_small()
     test_pyfunc_multiproc_max_rowsize_large()

tests/ut/python/dataset/test_var_batch_map.py

@@ -117,6 +117,10 @@ def test_variable_size_batch():
 
     # same as test_batch_repeat_with_copy_map except with python multiprocessing enabled
     def test_batch_repeat_with_copy_map_multiproc(gen_num, r, drop, func, num_workers, my_maxrowsize):
+        # Reduce memory required by disabling the shared memory optimization
+        mem_original = ds.config.get_enable_shared_mem()
+        ds.config.set_enable_shared_mem(False)
+
         res = []
         data1 = ds.GeneratorDataset((lambda: gen(gen_num)), ["num"], num_parallel_workers=num_workers,
                                     python_multiprocessing=True, max_rowsize=my_maxrowsize) \
@@ -125,6 +129,8 @@ def test_variable_size_batch():
                    max_rowsize=my_maxrowsize).repeat(r)
         for item in data1.create_dict_iterator(num_epochs=1, output_numpy=True):
             res.append(item["num"])
+
+        ds.config.set_enable_shared_mem(mem_original)
         return res
 
     tst1, tst2, tst3, tst4, tst5, tst6, tst7 = [], [], [], [], [], [], []