wangwei
/
tods
Not watched
1
0
Fork 0
Code
Releases 0 Wiki Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 03efe340c9
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '03efe340c9'
${ noResults }
tods/d3m/tests/test_problem.py

233 lines
13 kB
Raw Blame History 

  1. import os.path

  2. import pickle

  3. import unittest

  4. 

  5. from d3m import utils

  6. from d3m.metadata import problem, pipeline_run

  7. 

  8. 

  9. class TestProblem(unittest.TestCase):

  10.     def test_basic(self):

  11.         self.maxDiff = None

  12. 

  13.         problem_doc_path = os.path.join(os.path.dirname(__file__), 'data', 'problems', 'iris_problem_1', 'problemDoc.json')

  14. 

  15.         problem_uri = 'file://{problem_doc_path}'.format(problem_doc_path=problem_doc_path)

  16. 

  17.         problem_description = problem.Problem.load(problem_uri)

  18. 

  19.         self.assertEqual(problem_description.to_simple_structure(), {

  20.             'id': 'iris_problem_1',

  21.             'digest': '1a12135422967aa0de0c4629f4f58d08d39e97f9133f7b50da71420781aa18a5',

  22.             'version': '4.0.0',

  23.             'location_uris': [

  24.                 problem_uri,

  25.             ],

  26.             'name': 'Distinguish Iris flowers',

  27.             'description': 'Distinguish Iris flowers of three related species.',

  28.             'schema': problem.PROBLEM_SCHEMA_VERSION,

  29.             'problem': {

  30.                 'task_keywords': [problem.TaskKeyword.CLASSIFICATION, problem.TaskKeyword.MULTICLASS],

  31.                 'performance_metrics': [

  32.                     {

  33.                         'metric': problem.PerformanceMetric.ACCURACY,

  34.                     }

  35.                 ]

  36.             },

  37.             'inputs': [

  38.                 {

  39.                     'dataset_id': 'iris_dataset_1',

  40.                     'targets': [

  41.                         {

  42.                             'target_index': 0,

  43.                             'resource_id': 'learningData',

  44.                             'column_index': 5,

  45.                             'column_name': 'species',

  46.                         }

  47.                     ]

  48.                 }

  49.             ],

  50.         })

  51. 

  52.         self.assertEqual(problem_description.to_json_structure(), {

  53.             'id': 'iris_problem_1',

  54.             'digest': '1a12135422967aa0de0c4629f4f58d08d39e97f9133f7b50da71420781aa18a5',

  55.             'version': '4.0.0',

  56.             'location_uris': [

  57.                 problem_uri,

  58.             ],

  59.             'name': 'Distinguish Iris flowers',

  60.             'description': 'Distinguish Iris flowers of three related species.',

  61.             'schema': problem.PROBLEM_SCHEMA_VERSION,

  62.             'problem': {

  63.                 'task_keywords': [problem.TaskKeyword.CLASSIFICATION, problem.TaskKeyword.MULTICLASS],

  64.                 'performance_metrics': [

  65.                     {

  66.                         'metric': problem.PerformanceMetric.ACCURACY,

  67.                     }

  68.                 ]

  69.             },

  70.             'inputs': [

  71.                 {

  72.                     'dataset_id': 'iris_dataset_1',

  73.                     'targets': [

  74.                         {

  75.                             'target_index': 0,

  76.                             'resource_id': 'learningData',

  77.                             'column_index': 5,

  78.                             'column_name': 'species',

  79.                         }

  80.                     ]

  81.                 }

  82.             ],

  83.         })

  84. 

  85.         self.assertEqual(problem_description.to_json_structure(), {

  86.             'id': 'iris_problem_1',

  87.             'digest': '1a12135422967aa0de0c4629f4f58d08d39e97f9133f7b50da71420781aa18a5',

  88.             'version': '4.0.0',

  89.             'location_uris': [

  90.                 problem_uri,

  91.             ],

  92.             'name': 'Distinguish Iris flowers',

  93.             'description': 'Distinguish Iris flowers of three related species.',

  94.             'schema': problem.PROBLEM_SCHEMA_VERSION,

  95.             'problem': {

  96.                 'task_keywords': ['CLASSIFICATION', 'MULTICLASS'],

  97.                 'performance_metrics': [

  98.                     {

  99.                         'metric': 'ACCURACY',

  100.                     }

  101.                 ]

  102.             },

  103.             'inputs': [

  104.                 {

  105.                     'dataset_id': 'iris_dataset_1',

  106.                     'targets': [

  107.                         {

  108.                             'target_index': 0,

  109.                             'resource_id': 'learningData',

  110.                             'column_index': 5,

  111.                             'column_name': 'species',

  112.                         }

  113.                     ]

  114.                 }

  115.             ],

  116.         })

  117. 

  118.         pipeline_run.validate_problem(problem_description.to_json_structure(canonical=True))

  119.         problem.PROBLEM_SCHEMA_VALIDATOR.validate(problem_description.to_json_structure(canonical=True))

  120. 

  121.     def test_conversion(self):

  122.         problem_doc_path = os.path.join(os.path.dirname(__file__), 'data', 'problems', 'iris_problem_1', 'problemDoc.json')

  123. 

  124.         problem_uri = 'file://{problem_doc_path}'.format(problem_doc_path=problem_doc_path)

  125. 

  126.         problem_description = problem.Problem.load(problem_uri)

  127. 

  128.         self.assertEqual(problem_description.to_simple_structure(), problem.Problem.from_json_structure(problem_description.to_json_structure(), strict_digest=True).to_simple_structure())

  129. 

  130.         # Legacy.

  131.         self.assertEqual(utils.to_json_structure(problem_description.to_simple_structure()), problem.Problem.from_json_structure(utils.to_json_structure(problem_description.to_simple_structure()), strict_digest=True).to_simple_structure())

  132. 

  133.         self.assertIs(problem.Problem.from_json_structure(problem_description.to_json_structure(), strict_digest=True)['problem']['task_keywords'][0], problem.TaskKeyword.CLASSIFICATION)

  134. 

  135.     def test_unparse(self):

  136.         self.assertEqual(problem.TaskKeyword.CLASSIFICATION.unparse(), 'classification')

  137.         self.assertEqual(problem.TaskKeyword.MULTICLASS.unparse(), 'multiClass')

  138.         self.assertEqual(problem.PerformanceMetric.ACCURACY.unparse(), 'accuracy')

  139. 

  140.     def test_normalize(self):

  141.         self.assertEqual(problem.PerformanceMetric._normalize(0, 1, 0.5), 0.5)

  142.         self.assertEqual(problem.PerformanceMetric._normalize(0, 2, 0.5), 0.25)

  143.         self.assertEqual(problem.PerformanceMetric._normalize(1, 2, 1.5), 0.5)

  144. 

  145.         self.assertEqual(problem.PerformanceMetric._normalize(-1, 0, -0.5), 0.5)

  146.         self.assertEqual(problem.PerformanceMetric._normalize(-2, 0, -1.5), 0.25)

  147.         self.assertEqual(problem.PerformanceMetric._normalize(-2, -1, -1.5), 0.5)

  148. 

  149.         self.assertEqual(problem.PerformanceMetric._normalize(1, 0, 0.5), 0.5)

  150.         self.assertEqual(problem.PerformanceMetric._normalize(2, 0, 0.5), 0.75)

  151.         self.assertEqual(problem.PerformanceMetric._normalize(2, 1, 1.5), 0.5)

  152. 

  153.         self.assertEqual(problem.PerformanceMetric._normalize(0, -1, -0.5), 0.5)

  154.         self.assertEqual(problem.PerformanceMetric._normalize(0, -2, -1.5), 0.75)

  155.         self.assertEqual(problem.PerformanceMetric._normalize(-1, -2, -1.5), 0.5)

  156. 

  157.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(float('inf'), 0, 0.0), 1.0)

  158.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(float('inf'), 0, 0.5), 0.9997500000052083)

  159.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(float('inf'), 0, 1000.0), 0.5378828427399902)

  160.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(float('inf'), 0, 5000.0), 0.013385701848569713)

  161. 

  162.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(float('inf'), 1, 1.0), 1.0)

  163.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(float('inf'), 1, 1.5), 0.9997500000052083)

  164.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(float('inf'), 1, 1000.0), 0.5382761574524354)

  165.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(float('inf'), 1, 5000.0), 0.013399004523107192)

  166. 

  167.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(float('inf'), -1, -1.0), 1.0)

  168.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(float('inf'), -1, -0.5), 0.9997500000052083)

  169.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(float('inf'), -1, 1000.0), 0.5374897097430198)

  170.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(float('inf'), -1, 5000.0), 0.01337241229216877)

  171.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(float('inf'), -1, 0.0), 0.9995000000416667)

  172. 

  173.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(float('-inf'), 0, 0.0), 1.0)

  174.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(float('-inf'), 0, -0.5), 0.9997500000052083)

  175.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(float('-inf'), 0, -1000.0), 0.5378828427399902)

  176.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(float('-inf'), 0, -5000.0), 0.013385701848569713)

  177. 

  178.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(float('-inf'), 1, 1.0), 1.0)

  179.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(float('-inf'), 1, 0.5), 0.9997500000052083)

  180.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(float('-inf'), 1, -1000.0), 0.5374897097430198)

  181.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(float('-inf'), 1, -5000.0), 0.01337241229216877)

  182.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(float('-inf'), 1, 0.0), 0.9995000000416667)

  183. 

  184.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(float('-inf'), -1, -1.0), 1.0)

  185.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(float('-inf'), -1, -1.5), 0.9997500000052083)

  186.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(float('-inf'), -1, -1000.0), 0.5382761574524354)

  187.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(float('-inf'), -1, -5000.0), 0.013399004523107192)

  188. 

  189.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(0, float('inf'), 0.0), 1 - 1.0)

  190.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(0, float('inf'), 0.5), 1 - 0.9997500000052083)

  191.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(0, float('inf'), 1000.0), 1 - 0.5378828427399902)

  192.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(0, float('inf'), 5000.0), 1 - 0.013385701848569713)

  193. 

  194.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(1, float('inf'), 1.0), 1 - 1.0)

  195.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(1, float('inf'), 1.5), 1 - 0.9997500000052083)

  196.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(1, float('inf'), 1000.0), 1 - 0.5382761574524354)

  197.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(1, float('inf'), 5000.0), 1 - 0.013399004523107192)

  198. 

  199.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(-1, float('inf'), -1.0), 1 - 1.0)

  200.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(-1, float('inf'), -0.5), 1 - 0.9997500000052083)

  201.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(-1, float('inf'), 1000.0), 1 - 0.5374897097430198)

  202.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(-1, float('inf'), 5000.0), 1 - 0.01337241229216877)

  203.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(-1, float('inf'), 0.0), 1 - 0.9995000000416667)

  204. 

  205.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(0, float('-inf'), 0.0), 1 - 1.0)

  206.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(0, float('-inf'), -0.5), 1 - 0.9997500000052083)

  207.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(0, float('-inf'), -1000.0), 1 - 0.5378828427399902)

  208.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(0, float('-inf'), -5000.0), 1 - 0.013385701848569713)

  209. 

  210.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(1, float('-inf'), 1.0), 1 - 1.0)

  211.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(1, float('-inf'), 0.5), 1 - 0.9997500000052083)

  212.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(1, float('-inf'), -1000.0), 1 - 0.5374897097430198)

  213.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(1, float('-inf'), -5000.0), 1 - 0.01337241229216877)

  214.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(1, float('-inf'), 0.0), 1 - 0.9995000000416667)

  215. 

  216.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(-1, float('-inf'), -1.0), 1 - 1.0)

  217.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(-1, float('-inf'), -1.5), 1 - 0.9997500000052083)

  218.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(-1, float('-inf'), -1000.0), 1 - 0.5382761574524354)

  219.         self.assertAlmostEqual(problem.PerformanceMetric._normalize(-1, float('-inf'), -5000.0), 1 - 0.013399004523107192)

  220. 

  221.     def test_pickle(self):

  222.         value = problem.PerformanceMetric.ACCURACY

  223. 

  224.         pickled = pickle.dumps(value)

  225.         unpickled = pickle.loads(pickled)

  226. 

  227.         self.assertEqual(value, unpickled)

  228.         self.assertIs(value.get_class(), unpickled.get_class())

  229. 

  230. 

  231. if __name__ == '__main__':

  232.     unittest.main()