wangwei
/
tods
Not watched
1
0
Fork 0
Code
Releases 0 Wiki Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 03efe340c9
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '03efe340c9'
${ noResults }
tods/d3m/tests/test_metadata.py

2212 lines
84 kB
Raw Blame History 

  1. import unittest

  2. 

  3. import jsonschema

  4. import numpy

  5. 

  6. from d3m import container, utils

  7. from d3m.metadata import base

  8. 

  9. 

  10. def copy_elements_metadata(source_metadata, target_metadata, from_selector, to_selector=(), *, ignore_all_elements=False):

  11.     return source_metadata._copy_elements_metadata(target_metadata, list(from_selector), list(to_selector), [], ignore_all_elements)

  12. 

  13. 

  14. class TestMetadata(unittest.TestCase):

  15.     def test_basic(self):

  16.         md1 = base.Metadata({'value': 'test'})

  17. 

  18.         self.assertEqual(md1.query(()), {'value': 'test'})

  19.         self.assertEqual(md1.query(('foo',)), {})

  20.         self.assertEqual(md1.query(('bar',)), {})

  21. 

  22.         md2 = md1.update((), {'value2': 'test2'})

  23. 

  24.         self.assertEqual(md1.query(()), {'value': 'test'})

  25.         self.assertEqual(md1.query(('foo',)), {})

  26.         self.assertEqual(md1.query(('bar',)), {})

  27.         self.assertEqual(md2.query(()), {'value': 'test', 'value2': 'test2'})

  28. 

  29.         md3 = md2.update(('foo',), {'element': 'one'})

  30. 

  31.         self.assertEqual(md1.query(()), {'value': 'test'})

  32.         self.assertEqual(md1.query(('foo',)), {})

  33.         self.assertEqual(md1.query(('bar',)), {})

  34.         self.assertEqual(md2.query(()), {'value': 'test', 'value2': 'test2'})

  35.         self.assertEqual(md3.query(()), {'value': 'test', 'value2': 'test2'})

  36.         self.assertEqual(md3.query(('foo',)), {'element': 'one'})

  37. 

  38.         md4 = md3.update((base.ALL_ELEMENTS,), {'element': 'two'})

  39. 

  40.         self.assertEqual(md1.query(()), {'value': 'test'})

  41.         self.assertEqual(md1.query(('foo',)), {})

  42.         self.assertEqual(md1.query(('bar',)), {})

  43.         self.assertEqual(md2.query(()), {'value': 'test', 'value2': 'test2'})

  44.         self.assertEqual(md3.query(()), {'value': 'test', 'value2': 'test2'})

  45.         self.assertEqual(md3.query(('foo',)), {'element': 'one'})

  46.         self.assertEqual(md4.query(()), {'value': 'test', 'value2': 'test2'})

  47.         self.assertEqual(md4.query((base.ALL_ELEMENTS,)), {'element': 'two'})

  48.         self.assertEqual(md4.query(('foo',)), {'element': 'two'})

  49. 

  50.         md5 = md4.update(('foo',), {'element': 'three'})

  51. 

  52.         self.assertEqual(md1.query(()), {'value': 'test'})

  53.         self.assertEqual(md1.query(('foo',)), {})

  54.         self.assertEqual(md1.query(('bar',)), {})

  55.         self.assertEqual(md2.query(()), {'value': 'test', 'value2': 'test2'})

  56.         self.assertEqual(md3.query(()), {'value': 'test', 'value2': 'test2'})

  57.         self.assertEqual(md3.query(('foo',)), {'element': 'one'})

  58.         self.assertEqual(md4.query(()), {'value': 'test', 'value2': 'test2'})

  59.         self.assertEqual(md4.query((base.ALL_ELEMENTS,)), {'element': 'two'})

  60.         self.assertEqual(md4.query(('foo',)), {'element': 'two'})

  61.         self.assertEqual(md5.query(()), {'value': 'test', 'value2': 'test2'})

  62.         self.assertEqual(md5.query((base.ALL_ELEMENTS,)), {'element': 'two'})

  63.         self.assertEqual(md5.query(('foo',)), {'element': 'three'})

  64. 

  65.     def test_all_elements(self):

  66.         md1 = base.Metadata()

  67. 

  68.         md2 = md1.update((base.ALL_ELEMENTS, 'bar'), {'value': 'test1'})

  69. 

  70.         self.assertEqual(md2.query(('foo', 'bar')), {'value': 'test1'})

  71. 

  72.         md3 = md2.update(('foo', 'bar'), {'value': 'test2'})

  73. 

  74.         self.assertEqual(md2.query(('foo', 'bar')), {'value': 'test1'})

  75.         self.assertEqual(md3.query(('foo', 'bar')), {'value': 'test2'})

  76. 

  77.         md4 = md3.update((base.ALL_ELEMENTS, 'bar'), {'value': 'test3'})

  78. 

  79.         self.assertEqual(md2.query(('foo', 'bar')), {'value': 'test1'})

  80.         self.assertEqual(md3.query(('foo', 'bar')), {'value': 'test2'})

  81.         self.assertEqual(md4.query(('foo', 'bar')), {'value': 'test3'})

  82. 

  83.         md5 = md4.update(('foo', base.ALL_ELEMENTS), {'value': 'test4'})

  84. 

  85.         self.assertEqual(md2.query(('foo', 'bar')), {'value': 'test1'})

  86.         self.assertEqual(md3.query(('foo', 'bar')), {'value': 'test2'})

  87.         self.assertEqual(md4.query(('foo', 'bar')), {'value': 'test3'})

  88.         self.assertEqual(md5.query(('foo', 'bar')), {'value': 'test4'})

  89. 

  90.         md6 = md5.update(('foo', 'bar'), {'value': 'test5'})

  91. 

  92.         self.assertEqual(md2.query(('foo', 'bar')), {'value': 'test1'})

  93.         self.assertEqual(md3.query(('foo', 'bar')), {'value': 'test2'})

  94.         self.assertEqual(md4.query(('foo', 'bar')), {'value': 'test3'})

  95.         self.assertEqual(md5.query(('foo', 'bar')), {'value': 'test4'})

  96.         self.assertEqual(md6.query(('foo', 'bar')), {'value': 'test5'})

  97. 

  98.         md7 = md6.update((base.ALL_ELEMENTS, base.ALL_ELEMENTS), {'value': 'test6'})

  99. 

  100.         self.assertEqual(md2.query(('foo', 'bar')), {'value': 'test1'})

  101.         self.assertEqual(md3.query(('foo', 'bar')), {'value': 'test2'})

  102.         self.assertEqual(md4.query(('foo', 'bar')), {'value': 'test3'})

  103.         self.assertEqual(md5.query(('foo', 'bar')), {'value': 'test4'})

  104.         self.assertEqual(md6.query(('foo', 'bar')), {'value': 'test5'})

  105.         self.assertEqual(md7.query(('foo', 'bar')), {'value': 'test6'})

  106. 

  107.         md8 = md7.update(('foo', 'bar'), {'value': 'test7'})

  108. 

  109.         self.assertEqual(md2.query(('foo', 'bar')), {'value': 'test1'})

  110.         self.assertEqual(md3.query(('foo', 'bar')), {'value': 'test2'})

  111.         self.assertEqual(md4.query(('foo', 'bar')), {'value': 'test3'})

  112.         self.assertEqual(md5.query(('foo', 'bar')), {'value': 'test4'})

  113.         self.assertEqual(md6.query(('foo', 'bar')), {'value': 'test5'})

  114.         self.assertEqual(md7.query(('foo', 'bar')), {'value': 'test6'})

  115.         self.assertEqual(md8.query(('foo', 'bar')), {'value': 'test7'})

  116. 

  117.         self.assertEqual(md8.to_internal_json_structure(), [{

  118.             'selector': ['__ALL_ELEMENTS__', '__ALL_ELEMENTS__'],

  119.             'metadata': {

  120.                 'value': 'test6'

  121.             }

  122.         }, {

  123.             'selector': ['foo', 'bar'],

  124.             'metadata': {

  125.                 'value': 'test7'

  126.             }

  127.         }])

  128. 

  129.     def test_removal(self):

  130.         md1 = base.Metadata().update((), {'value': 'test1'})

  131. 

  132.         self.assertEqual(md1.query(()), {'value': 'test1'})

  133. 

  134.         md2 = md1.update((), {'value': base.NO_VALUE})

  135. 

  136.         self.assertEqual(md1.query(()), {'value': 'test1'})

  137.         self.assertEqual(md2.query(()), {})

  138.         self.assertEqual(md2.query((), ignore_all_elements=True), {})

  139. 

  140.         md3 = md2.update((), {'value': {'value2': 'test2'}})

  141. 

  142.         self.assertEqual(md1.query(()), {'value': 'test1'})

  143.         self.assertEqual(md2.query(()), {})

  144.         self.assertEqual(md3.query(()), {'value': {'value2': 'test2'}})

  145. 

  146.         md4 = md3.update((), {'value': {'value2': base.NO_VALUE}})

  147. 

  148.         self.assertEqual(md1.query(()), {'value': 'test1'})

  149.         self.assertEqual(md2.query(()), {})

  150.         self.assertEqual(md3.query(()), {'value': {'value2': 'test2'}})

  151.         self.assertEqual(md4.query(()), {})

  152. 

  153.         md5 = md4.update((), {'value': base.NO_VALUE})

  154. 

  155.         self.assertEqual(md1.query(()), {'value': 'test1'})

  156.         self.assertEqual(md2.query(()), {})

  157.         self.assertEqual(md3.query(()), {'value': {'value2': 'test2'}})

  158.         self.assertEqual(md4.query(()), {})

  159.         self.assertEqual(md5.query(()), {})

  160. 

  161.     def test_empty_dict(self):

  162.         md = base.Metadata().update((), {'value': {}})

  163. 

  164.         self.assertEqual(md.query(()), {'value': {}})

  165. 

  166.         md = md.update((), {'value': {'a': '1', 'b': 2}})

  167. 

  168.         self.assertEqual(md.query(()), {'value': {'a': '1', 'b': 2}})

  169. 

  170.         md = md.update((), {'value': {'a': base.NO_VALUE, 'b': base.NO_VALUE}})

  171. 

  172.         self.assertEqual(md.query(()), {})

  173. 

  174.         md = md.update((), {'value': {'a': '1', 'b': 2}})

  175. 

  176.         self.assertEqual(md.query(()), {'value': {'a': '1', 'b': 2}})

  177. 

  178.         md = md.update((), {'value': {'a': base.NO_VALUE}})

  179. 

  180.         self.assertEqual(md.query(()), {'value': {'b': 2}})

  181. 

  182.     def test_remove(self):

  183.         metadata = base.Metadata().update((), {'value': 'test1'})

  184.         metadata = metadata.update(('a',), {'value': 'test2'})

  185.         metadata = metadata.update(('a', 'b'), {'value': 'test3'})

  186.         metadata = metadata.update(('a', 'b', 'c'), {'value': 'test4'})

  187.         metadata = metadata.update((base.ALL_ELEMENTS, 'b', 'd'), {'value': 'test5'})

  188.         metadata = metadata.update((base.ALL_ELEMENTS, 'b', 'e', base.ALL_ELEMENTS), {'value': 'test6'})

  189. 

  190.         self.assertEqual(utils.to_json_structure(metadata.to_internal_simple_structure()), [

  191.             {

  192.                 'selector': [],

  193.                 'metadata': {

  194.                     'value': 'test1',

  195.                 }

  196.             },

  197.             {

  198.                 'selector': ['__ALL_ELEMENTS__', 'b', 'd'],

  199.                 'metadata': {

  200.                     'value': 'test5',

  201.                 },

  202.             },

  203.             {

  204.                 'selector': ['__ALL_ELEMENTS__', 'b', 'e', '__ALL_ELEMENTS__'],

  205.                 'metadata': {

  206.                     'value': 'test6',

  207.                 },

  208.             },

  209.             {

  210.                 'selector': ['a'],

  211.                 'metadata': {

  212.                     'value': 'test2',

  213.                 },

  214.             },

  215.             {

  216.                 'selector': ['a', 'b'],

  217.                 'metadata': {

  218.                     'value': 'test3',

  219.                 },

  220.             },

  221.             {

  222.                 'selector': ['a', 'b', 'c'],

  223.                 'metadata': {

  224.                     'value': 'test4',

  225.                 },

  226.             },

  227.         ])

  228. 

  229.         new_metadata = metadata.remove(('a', 'b'))

  230. 

  231.         self.assertEqual(utils.to_json_structure(new_metadata.to_internal_simple_structure()), [

  232.             {

  233.                 'selector': [],

  234.                 'metadata': {

  235.                     'value': 'test1',

  236.                 },

  237.             },

  238.             {

  239.                 'selector': ['__ALL_ELEMENTS__', 'b', 'd'],

  240.                 'metadata': {

  241.                     'value': 'test5',

  242.                 },

  243.             },

  244.             {

  245.                 'selector': ['__ALL_ELEMENTS__', 'b', 'e', '__ALL_ELEMENTS__'],

  246.                 'metadata': {

  247.                     'value': 'test6',

  248.                 },

  249.             },

  250.             {

  251.                 'selector': ['a'],

  252.                 'metadata': {

  253.                     'value': 'test2',

  254.                 },

  255.             },

  256.             {

  257.                 'selector': ['a', 'b', 'c'],

  258.                 'metadata': {

  259.                     'value': 'test4',

  260.                 },

  261.             },

  262.         ])

  263. 

  264.         new_metadata = metadata.remove(('a', 'b'), recursive=True)

  265. 

  266.         self.assertEqual(utils.to_json_structure(new_metadata.to_internal_simple_structure()), [

  267.             {

  268.                 'selector': [],

  269.                 'metadata': {

  270.                     'value': 'test1',

  271.                 },

  272.             },

  273.             {

  274.                 'selector': ['__ALL_ELEMENTS__', 'b', 'd'],

  275.                 'metadata': {

  276.                     'value': 'test5',

  277.                 },

  278.             },

  279.             {

  280.                 'selector': ['__ALL_ELEMENTS__', 'b', 'e', '__ALL_ELEMENTS__'],

  281.                 'metadata': {

  282.                     'value': 'test6',

  283.                 },

  284.             },

  285.             {

  286.                 'selector': ['a'],

  287.                 'metadata': {

  288.                     'value': 'test2',

  289.                 },

  290.             },

  291.         ])

  292. 

  293.         new_metadata = metadata.remove((), recursive=True)

  294. 

  295.         self.assertEqual(utils.to_json_structure(new_metadata.to_internal_simple_structure()), [])

  296. 

  297.         new_metadata = metadata.remove((base.ALL_ELEMENTS, 'b'))

  298. 

  299.         self.assertEqual(utils.to_json_structure(new_metadata.to_internal_simple_structure()), [

  300.             {

  301.                 'selector': [],

  302.                 'metadata': {

  303.                     'value': 'test1',

  304.                 }

  305.             },

  306.             {

  307.                 'selector': ['__ALL_ELEMENTS__', 'b', 'd'],

  308.                 'metadata': {

  309.                     'value': 'test5',

  310.                 },

  311.             },

  312.             {

  313.                 'selector': ['__ALL_ELEMENTS__', 'b', 'e', '__ALL_ELEMENTS__'],

  314.                 'metadata': {

  315.                     'value': 'test6',

  316.                 },

  317.             },

  318.             {

  319.                 'selector': ['a'],

  320.                 'metadata': {

  321.                     'value': 'test2',

  322.                 },

  323.             },

  324.             {

  325.                 'selector': ['a', 'b', 'c'],

  326.                 'metadata': {

  327.                     'value': 'test4',

  328.                 },

  329.             },

  330.         ])

  331. 

  332.         new_metadata = metadata.remove((base.ALL_ELEMENTS, 'b'), recursive=True)

  333. 

  334.         self.assertEqual(utils.to_json_structure(new_metadata.to_internal_simple_structure()), [

  335.             {

  336.                 'selector': [],

  337.                 'metadata': {

  338.                     'value': 'test1',

  339.                 }

  340.             },

  341.             {

  342.                 'selector': ['a'],

  343.                 'metadata': {

  344.                     'value': 'test2',

  345.                 },

  346.             },

  347.         ])

  348. 

  349.         new_metadata = metadata.remove((base.ALL_ELEMENTS, 'b'), strict_all_elements=True)

  350. 

  351.         self.assertEqual(utils.to_json_structure(new_metadata.to_internal_simple_structure()), [

  352.             {

  353.                 'selector': [],

  354.                 'metadata': {

  355.                     'value': 'test1',

  356.                 }

  357.             },

  358.             {

  359.                 'selector': ['__ALL_ELEMENTS__', 'b', 'd'],

  360.                 'metadata': {

  361.                     'value': 'test5',

  362.                 },

  363.             },

  364.             {

  365.                 'selector': ['__ALL_ELEMENTS__', 'b', 'e', '__ALL_ELEMENTS__'],

  366.                 'metadata': {

  367.                     'value': 'test6',

  368.                 },

  369.             },

  370.             {

  371.                 'selector': ['a'],

  372.                 'metadata': {

  373.                     'value': 'test2',

  374.                 },

  375.             },

  376.             {

  377.                 'selector': ['a', 'b'],

  378.                 'metadata': {

  379.                     'value': 'test3',

  380.                 },

  381.             },

  382.             {

  383.                 'selector': ['a', 'b', 'c'],

  384.                 'metadata': {

  385.                     'value': 'test4',

  386.                 },

  387.             },

  388.         ])

  389. 

  390.         new_metadata = metadata.remove((base.ALL_ELEMENTS, 'b'), recursive=True, strict_all_elements=True)

  391. 

  392.         self.assertEqual(utils.to_json_structure(new_metadata.to_internal_simple_structure()), [

  393.             {

  394.                 'selector': [],

  395.                 'metadata': {

  396.                     'value': 'test1',

  397.                 }

  398.             },

  399.             {

  400.                 'selector': ['a'],

  401.                 'metadata': {

  402.                     'value': 'test2',

  403.                 },

  404.             },

  405.             {

  406.                 'selector': ['a', 'b'],

  407.                 'metadata': {

  408.                     'value': 'test3',

  409.                 },

  410.             },

  411.             {

  412.                 'selector': ['a', 'b', 'c'],

  413.                 'metadata': {

  414.                     'value': 'test4',

  415.                 },

  416.             },

  417.         ])

  418. 

  419.         new_metadata = metadata.remove(('a', base.ALL_ELEMENTS))

  420. 

  421.         self.assertEqual(utils.to_json_structure(new_metadata.to_internal_simple_structure()), [

  422.             {

  423.                 'selector': [],

  424.                 'metadata': {

  425.                     'value': 'test1',

  426.                 }

  427.             },

  428.             {

  429.                 'selector': ['__ALL_ELEMENTS__', 'b', 'd'],

  430.                 'metadata': {

  431.                     'value': 'test5',

  432.                 },

  433.             },

  434.             {

  435.                 'selector': ['__ALL_ELEMENTS__', 'b', 'e', '__ALL_ELEMENTS__'],

  436.                 'metadata': {

  437.                     'value': 'test6',

  438.                 },

  439.             },

  440.             {

  441.                 'selector': ['a'],

  442.                 'metadata': {

  443.                     'value': 'test2',

  444.                 },

  445.             },

  446.             {

  447.                 'selector': ['a', 'b', 'c'],

  448.                 'metadata': {

  449.                     'value': 'test4',

  450.                 },

  451.             },

  452.         ])

  453. 

  454.         new_metadata = metadata.remove(('a', base.ALL_ELEMENTS), strict_all_elements=True)

  455. 

  456.         self.assertEqual(utils.to_json_structure(new_metadata.to_internal_simple_structure()), [

  457.             {

  458.                 'selector': [],

  459.                 'metadata': {

  460.                     'value': 'test1',

  461.                 }

  462.             },

  463.             {

  464.                 'selector': ['__ALL_ELEMENTS__', 'b', 'd'],

  465.                 'metadata': {

  466.                     'value': 'test5',

  467.                 },

  468.             },

  469.             {

  470.                 'selector': ['__ALL_ELEMENTS__', 'b', 'e', '__ALL_ELEMENTS__'],

  471.                 'metadata': {

  472.                     'value': 'test6',

  473.                 },

  474.             },

  475.             {

  476.                 'selector': ['a'],

  477.                 'metadata': {

  478.                     'value': 'test2',

  479.                 },

  480.             },

  481.             {

  482.                 'selector': ['a', 'b'],

  483.                 'metadata': {

  484.                     'value': 'test3',

  485.                 },

  486.             },

  487.             {

  488.                 'selector': ['a', 'b', 'c'],

  489.                 'metadata': {

  490.                     'value': 'test4',

  491.                 },

  492.             },

  493.         ])

  494. 

  495.         new_metadata = metadata.remove(('a', base.ALL_ELEMENTS), recursive=True)

  496. 

  497.         self.assertEqual(utils.to_json_structure(new_metadata.to_internal_simple_structure()), [

  498.             {

  499.                 'selector': [],

  500.                 'metadata': {

  501.                     'value': 'test1',

  502.                 }

  503.             },

  504.             {

  505.                 'selector': ['__ALL_ELEMENTS__', 'b', 'd'],

  506.                 'metadata': {

  507.                     'value': 'test5',

  508.                 },

  509.             },

  510.             {

  511.                 'selector': ['__ALL_ELEMENTS__', 'b', 'e', '__ALL_ELEMENTS__'],

  512.                 'metadata': {

  513.                     'value': 'test6',

  514.                 },

  515.             },

  516.             {

  517.                 'selector': ['a'],

  518.                 'metadata': {

  519.                     'value': 'test2',

  520.                 },

  521.             },

  522.         ])

  523. 

  524.         new_metadata = metadata.remove((base.ALL_ELEMENTS, 'b', base.ALL_ELEMENTS))

  525. 

  526.         self.assertEqual(utils.to_json_structure(new_metadata.to_internal_simple_structure()), [

  527.             {

  528.                 'selector': [],

  529.                 'metadata': {

  530.                     'value': 'test1',

  531.                 }

  532.             },

  533.             {

  534.                 'selector': ['__ALL_ELEMENTS__', 'b', 'e', '__ALL_ELEMENTS__'],

  535.                 'metadata': {

  536.                     'value': 'test6',

  537.                 },

  538.             },

  539.             {

  540.                 'selector': ['a'],

  541.                 'metadata': {

  542.                     'value': 'test2',

  543.                 },

  544.             },

  545.             {

  546.                 'selector': ['a', 'b'],

  547.                 'metadata': {

  548.                     'value': 'test3',

  549.                 },

  550.             },

  551.         ])

  552. 

  553.         new_metadata = metadata.remove((base.ALL_ELEMENTS, 'b', base.ALL_ELEMENTS), recursive=True)

  554. 

  555.         self.assertEqual(utils.to_json_structure(new_metadata.to_internal_simple_structure()), [

  556.             {

  557.                 'selector': [],

  558.                 'metadata': {

  559.                     'value': 'test1',

  560.                 }

  561.             },

  562.             {

  563.                 'selector': ['a'],

  564.                 'metadata': {

  565.                     'value': 'test2',

  566.                 },

  567.             },

  568.             {

  569.                 'selector': ['a', 'b'],

  570.                 'metadata': {

  571.                     'value': 'test3',

  572.                 },

  573.             },

  574.         ])

  575. 

  576.     def test_remove_column(self):

  577.         metadata = base.DataMetadata().update((base.ALL_ELEMENTS, 0), {'name': 'column1'})

  578.         metadata = metadata.update((base.ALL_ELEMENTS, 1), {'name': 'column2'})

  579.         metadata = metadata.update((10, 0), {'value': 'row10.0'})

  580.         metadata = metadata.update((10, 1), {'value': 'row10.1'})

  581. 

  582.         self.assertEqual(utils.to_json_structure(metadata.to_internal_simple_structure()), [{

  583.             'selector': ['__ALL_ELEMENTS__', 0],

  584.             'metadata': {'name': 'column1'},

  585.         }, {

  586.             'selector': ['__ALL_ELEMENTS__', 1],

  587.             'metadata': {'name': 'column2'},

  588.         }, {

  589.             'selector': [10, 0],

  590.             'metadata': {'value': 'row10.0'},

  591.         }, {

  592.             'selector': [10, 1],

  593.             'metadata': {'value': 'row10.1'},

  594.         }])

  595. 

  596.         metadata = metadata.remove_column(0)

  597. 

  598.         self.assertEqual(utils.to_json_structure(metadata.to_internal_simple_structure()), [{

  599.             'selector': ['__ALL_ELEMENTS__', 1],

  600.             'metadata': {'name': 'column2'},

  601.         }, {

  602.             'selector': [10, 1],

  603.             'metadata': {'value': 'row10.1'},

  604.         }])

  605. 

  606.     def test_check(self):

  607.         data = container.Dataset({

  608.             '0': container.ndarray(numpy.array([

  609.                 [1, 2, 3],

  610.                 [4, 5, 6],

  611.             ])),

  612.         })

  613. 

  614.         md1 = base.DataMetadata().update((), {

  615.             'schema': base.CONTAINER_SCHEMA_VERSION,

  616.             'structural_type': type(data),

  617.             'value': 'test'

  618.         })

  619. 

  620.         md1.check(data)

  621. 

  622.         md2 = md1.update(('missing',), {'value': 'test'})

  623. 

  624.         with self.assertRaisesRegex(ValueError, 'cannot be resolved'):

  625.             md2.check(data)

  626. 

  627.         md3 = md1.update(('0', 1), {'value': 'test'})

  628. 

  629.         md4 = md3.update(('0', 2), {'value': 'test'})

  630. 

  631.         with self.assertRaisesRegex(ValueError, 'cannot be resolved'):

  632.             md4.check(data)

  633. 

  634.         md5 = md3.update(('0', 1, 3), {'value': 'test'})

  635. 

  636.         with self.assertRaisesRegex(ValueError, 'cannot be resolved'):

  637.             md5.check(data)

  638. 

  639.         md6 = md3.update(('0', 1, 2, base.ALL_ELEMENTS), {'value': 'test'})

  640. 

  641.         with self.assertRaisesRegex(ValueError, 'ALL_ELEMENTS set but dimension missing at'):

  642.             md6.check(data)

  643. 

  644.     def test_errors(self):

  645.         with self.assertRaisesRegex(TypeError, 'Metadata should be a dict'):

  646.             base.Metadata().update((), None)

  647. 

  648.         class Custom:

  649.             pass

  650. 

  651.         with self.assertRaisesRegex(TypeError, 'is not known to be immutable'):

  652.             base.Metadata().update((), {'foo': Custom()})

  653. 

  654.         with self.assertRaisesRegex(TypeError, 'Selector is not a tuple or a list'):

  655.             base.Metadata().update({}, {'value': 'test'})

  656. 

  657.         with self.assertRaisesRegex(TypeError, 'is not a str, int, or ALL_ELEMENTS'):

  658.             base.Metadata().update((1.0,), {'value': 'test'})

  659. 

  660.         with self.assertRaisesRegex(TypeError, 'is not a str, int, or ALL_ELEMENTS'):

  661.             base.Metadata().update((None,), {'value': 'test'})

  662. 

  663.     def test_data(self):

  664.         data = container.Dataset({

  665.             '0': container.ndarray(numpy.array([

  666.                 [1, 2, 3],

  667.                 [4, 5, 6],

  668.             ])),

  669.         })

  670. 

  671.         md1 = base.DataMetadata()

  672.         md1.update((), {'value': 'test'})

  673. 

  674.         with self.assertRaisesRegex(jsonschema.exceptions.ValidationError, 'is a required property'):

  675.             md1.check(md1)

  676. 

  677.         md1 = base.DataMetadata().generate(data, compact=True)

  678. 

  679.         md2 = md1.update((), {

  680.             'id': 'test-dataset',

  681.             'schema': base.CONTAINER_SCHEMA_VERSION,

  682.             'structural_type': type(data),

  683.             'dimension': {

  684.                 'length': 1

  685.             }

  686.         })

  687. 

  688.         md3 = md2.update(('0',), {

  689.             'structural_type': type(data['0']),

  690.             'dimension': {

  691.                 'length': 2

  692.             }

  693.         })

  694. 

  695.         self.assertEqual(utils.to_json_structure(md3.to_internal_simple_structure()), [{

  696.             'selector': [],

  697.             'metadata': {

  698.                 'id': 'test-dataset',

  699.                 'schema': base.CONTAINER_SCHEMA_VERSION,

  700.                 'structural_type': 'd3m.container.dataset.Dataset',

  701.                 'dimension': {

  702.                     'name': 'resources',

  703.                     'semantic_types': ['https://metadata.datadrivendiscovery.org/types/DatasetResource'],

  704.                     'length': 1

  705.                 },

  706.             },

  707.         }, {

  708.             'selector': ['__ALL_ELEMENTS__'],

  709.             'metadata': {

  710.                 'dimension': {

  711.                     'length': 2,

  712.                      'name': 'rows',

  713.                      'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularRow'],

  714.                 },

  715.                'semantic_types': ['https://metadata.datadrivendiscovery.org/types/Table'],

  716.                'structural_type': 'd3m.container.numpy.ndarray',

  717.             },

  718.         }, {

  719.            'selector': ['__ALL_ELEMENTS__', '__ALL_ELEMENTS__'],

  720.             'metadata':  {

  721.                 'dimension': {

  722.                     'length': 3,

  723.                     'name': 'columns',

  724.                     'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularColumn'],

  725.                 },

  726.             },

  727.         }, {

  728.            'selector': ['__ALL_ELEMENTS__', '__ALL_ELEMENTS__', '__ALL_ELEMENTS__'],

  729.             'metadata': {

  730.                 'structural_type': 'numpy.int64',

  731.             },

  732.         }, {

  733.             'selector': ['0'],

  734.             'metadata': {

  735.                 'structural_type': 'd3m.container.numpy.ndarray',

  736.                 'dimension': {

  737.                     'length': 2,

  738.                 },

  739.             },

  740.         }])

  741. 

  742.         md1 = base.DataMetadata().generate(data, compact=False)

  743. 

  744.         md2 = md1.update((), {

  745.             'id': 'test-dataset',

  746.             'schema': base.CONTAINER_SCHEMA_VERSION,

  747.             'structural_type': type(data),

  748.             'dimension': {

  749.                 'length': 1

  750.             }

  751.         })

  752. 

  753.         md3 = md2.update(('0',), {

  754.             'structural_type': type(data['0']),

  755.             'dimension': {

  756.                 'length': 2

  757.             }

  758.         })

  759. 

  760.         self.assertEqual(utils.to_json_structure(md3.to_internal_simple_structure()), [{

  761.             'selector': [],

  762.             'metadata': {

  763.                 'id': 'test-dataset',

  764.                 'schema': base.CONTAINER_SCHEMA_VERSION,

  765.                 'structural_type': 'd3m.container.dataset.Dataset',

  766.                 'dimension': {

  767.                     'name': 'resources',

  768.                     'semantic_types': ['https://metadata.datadrivendiscovery.org/types/DatasetResource'],

  769.                     'length': 1

  770.                 },

  771.             },

  772.         }, {

  773.             'selector': ['0'],

  774.             'metadata': {

  775.                 'dimension': {

  776.                     'length': 2,

  777.                      'name': 'rows',

  778.                      'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularRow'],

  779.                 },

  780.                'semantic_types': ['https://metadata.datadrivendiscovery.org/types/Table'],

  781.                'structural_type': 'd3m.container.numpy.ndarray',

  782.             },

  783.         }, {

  784.            'selector': ['0', '__ALL_ELEMENTS__'],

  785.             'metadata':  {

  786.                 'dimension': {

  787.                     'length': 3,

  788.                     'name': 'columns',

  789.                     'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularColumn'],

  790.                 },

  791.             },

  792.         }, {

  793.            'selector': ['0', '__ALL_ELEMENTS__', '__ALL_ELEMENTS__'],

  794.             'metadata': {

  795.                 'structural_type': 'numpy.int64',

  796.             },

  797.         }])

  798. 

  799.     def test_prune_bug(self):

  800.         metadata = base.Metadata().update((base.ALL_ELEMENTS, 0), {'foo': 'bar1'})

  801.         metadata = metadata.update((0, 1), {'foo': 'bar2'})

  802.         metadata = metadata.update((1, 1), {'foo': 'bar2'})

  803.         metadata = metadata.update((2, 1), {'foo': 'bar2'})

  804.         metadata = metadata.update((base.ALL_ELEMENTS, base.ALL_ELEMENTS), {'foo': 'bar3'})

  805. 

  806.         self.assertEqual(utils.to_json_structure(metadata.to_internal_simple_structure()), [{

  807.             'selector': ['__ALL_ELEMENTS__', '__ALL_ELEMENTS__'],

  808.             'metadata': {'foo': 'bar3'},

  809.         }])

  810. 

  811.     def test_remove_empty_metadata(self):

  812.         metadata = base.Metadata().update((base.ALL_ELEMENTS,), {

  813.             'foo': {

  814.                 'bar': 42,

  815.             },

  816.             'other': 1,

  817.         })

  818. 

  819.         metadata = metadata.update((base.ALL_ELEMENTS,), {

  820.             'foo': {

  821.                 'bar': base.NO_VALUE,

  822.             },

  823.         })

  824. 

  825.         self.assertEqual(metadata.query((base.ALL_ELEMENTS,)), {

  826.             'other': 1,

  827.         })

  828. 

  829.         metadata = base.Metadata({

  830.             'foo': {

  831.                 'bar': 42,

  832.             },

  833.             'other': 1,

  834.         })

  835. 

  836.         metadata = metadata.update((), {

  837.             'foo': {

  838.                 'bar': base.NO_VALUE,

  839.             },

  840.         })

  841. 

  842.         self.assertEqual(metadata.query(()), {

  843.             'other': 1,

  844.         })

  845. 

  846.         metadata = base.Metadata({

  847.             'foo': {

  848.                 'bar': 42,

  849.             },

  850.         })

  851. 

  852.         metadata = metadata.update((), {

  853.             'foo': {

  854.                 'bar': base.NO_VALUE,

  855.             },

  856.         })

  857. 

  858.         self.assertEqual(metadata.query(()), {})

  859. 

  860.         metadata = base.Metadata().update(('a',), {

  861.             'foo': {

  862.                 'bar': 42,

  863.             },

  864.         })

  865. 

  866.         metadata = metadata.update((base.ALL_ELEMENTS,), {

  867.             'foo': {

  868.                 'bar': base.NO_VALUE,

  869.             },

  870.         })

  871. 

  872.         self.assertEqual(metadata.query(('a',)), {})

  873. 

  874.         self.assertEqual(utils.to_json_structure(metadata.to_internal_simple_structure()), [{

  875.             'selector': ['__ALL_ELEMENTS__'],

  876.             'metadata': {

  877.                 'foo': {

  878.                     'bar': '__NO_VALUE__',

  879.                 },

  880.             },

  881.         }])

  882. 

  883.     def test_ignore_all_elements(self):

  884.         metadata = base.Metadata().update((base.ALL_ELEMENTS,), {

  885.             'foo': 'bar',

  886.             'other': 42,

  887.         })

  888. 

  889.         metadata = metadata.update((0,), {

  890.             'foo': base.NO_VALUE,

  891.         })

  892. 

  893.         metadata = metadata.update((2,), {

  894.             'other2': 43,

  895.         })

  896. 

  897.         self.assertEqual(metadata.query((0,)), {'other': 42})

  898.         self.assertEqual(metadata.query((1,)), {'foo': 'bar', 'other': 42})

  899.         self.assertEqual(metadata.query((2,)), {'foo': 'bar', 'other': 42, 'other2': 43})

  900.         self.assertEqual(metadata.query((0,), ignore_all_elements=True), {})

  901.         self.assertEqual(metadata.query((1,), ignore_all_elements=True), {})

  902.         self.assertEqual(metadata.query((2,), ignore_all_elements=True), {'other2': 43})

  903. 

  904.         metadata = metadata.update((base.ALL_ELEMENTS,), {

  905.             'foo': 'bar2',

  906.         })

  907. 

  908.         self.assertEqual(metadata.query((0,)), {'foo': 'bar2', 'other': 42})

  909.         self.assertEqual(metadata.query((1,)), {'foo': 'bar2', 'other': 42})

  910.         self.assertEqual(metadata.query((2,)), {'foo': 'bar2', 'other': 42, 'other2': 43})

  911.         self.assertEqual(metadata.query((0,), ignore_all_elements=True), {})

  912.         self.assertEqual(metadata.query((1,), ignore_all_elements=True), {})

  913.         self.assertEqual(metadata.query((2,), ignore_all_elements=True), {'other2': 43})

  914. 

  915.     def test_query_with_exceptions(self):

  916.         metadata = base.Metadata().update((base.ALL_ELEMENTS,), {

  917.             'foo': 'bar',

  918.             'other': 42,

  919.         })

  920. 

  921.         metadata = metadata.update((0,), {

  922.             'foo': base.NO_VALUE,

  923.         })

  924. 

  925.         metadata = metadata.update((2,), {

  926.             'other2': 43,

  927.         })

  928. 

  929.         self.assertEqual(metadata.query((0,)), {'other': 42})

  930.         self.assertEqual(metadata.query((1,)), {'foo': 'bar', 'other': 42})

  931.         self.assertEqual(metadata.query((2,)), {'foo': 'bar', 'other': 42, 'other2': 43})

  932. 

  933.         self.assertEqual(metadata.query_with_exceptions((0,)), ({'other': 42}, {}))

  934.         self.assertEqual(metadata.query_with_exceptions((1,)), ({'foo': 'bar', 'other': 42}, {}))

  935.         self.assertEqual(metadata.query_with_exceptions((2,)), ({'foo': 'bar', 'other': 42, 'other2': 43}, {}))

  936. 

  937.         self.assertEqual(metadata.query_with_exceptions((base.ALL_ELEMENTS,)), ({

  938.             'foo': 'bar',

  939.             'other': 42,

  940.         }, {

  941.             (0,): {'other': 42},

  942.             (2,): {'foo': 'bar', 'other': 42, 'other2': 43},

  943.         }))

  944. 

  945.         metadata = metadata.update((base.ALL_ELEMENTS,), {

  946.             'foo': 'bar2',

  947.         })

  948. 

  949.         self.assertEqual(metadata.query_with_exceptions((base.ALL_ELEMENTS,)), ({

  950.             'foo': 'bar2',

  951.             'other': 42,

  952.         }, {

  953.             (2,): {'foo': 'bar2', 'other': 42, 'other2': 43},

  954.         }))

  955. 

  956.         metadata = base.Metadata().update((base.ALL_ELEMENTS, 0), {

  957.             'name': 'bar',

  958.         })

  959. 

  960.         metadata = metadata.update((base.ALL_ELEMENTS, 1), {

  961.             'name': 'foo',

  962.         })

  963. 

  964.         metadata = metadata.update((2, 0), {

  965.             'name': 'bar2',

  966.         })

  967. 

  968.         metadata = metadata.update((2, 2), {

  969.             'name': 'foo2',

  970.         })

  971. 

  972.         self.assertEqual(metadata.query_with_exceptions((base.ALL_ELEMENTS, 0)), ({

  973.             'name': 'bar',

  974.         }, {

  975.             (2, 0): {'name': 'bar2'},

  976.         }))

  977. 

  978.         self.assertEqual(metadata.query_with_exceptions((base.ALL_ELEMENTS, 1)), ({

  979.             'name': 'foo',

  980.         }, {}))

  981. 

  982.         self.assertEqual(metadata.query_with_exceptions((base.ALL_ELEMENTS, 2)), ({}, {

  983.             (2, 2): {'name': 'foo2'},

  984.         }))

  985. 

  986.         self.assertEqual(metadata.query_with_exceptions((2, base.ALL_ELEMENTS)), ({}, {

  987.             (2, 0): {'name': 'bar2'},

  988.             (2, 2): {'name': 'foo2'},

  989.         }))

  990. 

  991.         metadata = base.Metadata().update((base.ALL_ELEMENTS, base.ALL_ELEMENTS), {

  992.             'foo': 'bar',

  993.             'other': 42,

  994.         })

  995. 

  996.         metadata = metadata.update((base.ALL_ELEMENTS, 0), {

  997.             'foo': base.NO_VALUE,

  998.         })

  999. 

  1000.         metadata = metadata.update((base.ALL_ELEMENTS, 2), {

  1001.             'other2': 43,

  1002.         })

  1003. 

  1004.         self.assertEqual(metadata.query((base.ALL_ELEMENTS, 0)), {'other': 42})

  1005.         self.assertEqual(metadata.query((base.ALL_ELEMENTS, 1)), {'foo': 'bar', 'other': 42})

  1006.         self.assertEqual(metadata.query((base.ALL_ELEMENTS, 2)), {'foo': 'bar', 'other': 42, 'other2': 43})

  1007. 

  1008.         self.assertEqual(metadata.query_with_exceptions((base.ALL_ELEMENTS, 0)), ({'other': 42}, {}))

  1009.         self.assertEqual(metadata.query_with_exceptions((base.ALL_ELEMENTS, 1)), ({'foo': 'bar', 'other': 42}, {}))

  1010.         self.assertEqual(metadata.query_with_exceptions((base.ALL_ELEMENTS, 2)), ({'foo': 'bar', 'other': 42, 'other2': 43}, {}))

  1011. 

  1012.         self.assertEqual(metadata.query_with_exceptions((base.ALL_ELEMENTS, base.ALL_ELEMENTS)), ({

  1013.             'foo': 'bar',

  1014.             'other': 42,

  1015.         }, {

  1016.             (base.ALL_ELEMENTS, 0): {'other': 42},

  1017.             (base.ALL_ELEMENTS, 2): {'foo': 'bar', 'other': 42, 'other2': 43},

  1018.         }))

  1019. 

  1020.     def test_semantic_types(self):

  1021.         metadata = base.DataMetadata({

  1022.             'structural_type': container.DataFrame,

  1023.             'schema': base.CONTAINER_SCHEMA_VERSION,

  1024.         })

  1025. 

  1026.         self.assertFalse(metadata.has_semantic_type((), 'https://metadata.datadrivendiscovery.org/types/DatasetResource'))

  1027. 

  1028.         metadata = metadata.add_semantic_type((), 'https://metadata.datadrivendiscovery.org/types/DatasetResource')

  1029. 

  1030.         self.assertTrue(metadata.has_semantic_type((), 'https://metadata.datadrivendiscovery.org/types/DatasetResource'))

  1031. 

  1032.         metadata = metadata.remove_semantic_type((), 'https://metadata.datadrivendiscovery.org/types/DatasetResource')

  1033. 

  1034.         self.assertFalse(metadata.has_semantic_type((), 'https://metadata.datadrivendiscovery.org/types/DatasetResource'))

  1035. 

  1036.         metadata = metadata.add_semantic_type((base.ALL_ELEMENTS, 0), 'https://metadata.datadrivendiscovery.org/types/Attribute')

  1037.         metadata = metadata.add_semantic_type((base.ALL_ELEMENTS, 1), 'https://metadata.datadrivendiscovery.org/types/PrimaryKey')

  1038.         metadata = metadata.add_semantic_type((base.ALL_ELEMENTS, 2), 'https://metadata.datadrivendiscovery.org/types/Attribute')

  1039. 

  1040.         self.assertEqual(metadata.get_elements_with_semantic_type((), 'https://metadata.datadrivendiscovery.org/types/Attribute'), [])

  1041.         self.assertEqual(metadata.get_elements_with_semantic_type((base.ALL_ELEMENTS,), 'https://metadata.datadrivendiscovery.org/types/Attribute'), [0, 2])

  1042. 

  1043.     def test_copy_elements_metadata(self):

  1044.         metadata = base.Metadata()

  1045. 

  1046.         metadata = metadata.update((), {'level0': 'foobar0'})

  1047. 

  1048.         metadata = metadata.update(('level1',), {'level1': 'foobar1'})

  1049. 

  1050.         metadata = metadata.update((base.ALL_ELEMENTS,), {'level1a': 'foobar1a', 'level1b': 'foobar1b'})

  1051. 

  1052.         metadata = metadata.update(('level1',), {'level1b': base.NO_VALUE})

  1053. 

  1054.         metadata = metadata.update(('level1', 'level2'), {'level2': 'foobar2'})

  1055. 

  1056.         metadata = metadata.update((base.ALL_ELEMENTS, base.ALL_ELEMENTS), {'level2a': 'foobar2a', 'level2b': 'foobar2b'})

  1057. 

  1058.         metadata = metadata.update(('level1', 'level2'), {'level2b': base.NO_VALUE})

  1059. 

  1060.         metadata = metadata.update(('level1', 'level2', 'level3'), {'level3': 'foobar3'})

  1061. 

  1062.         metadata = metadata.update((base.ALL_ELEMENTS, base.ALL_ELEMENTS, 'level3'), {'level3a': 'foobar3a'})

  1063. 

  1064.         metadata = metadata.update(('level1', 'level2', 'level3.1'), {'level3.1': 'foobar3.1'})

  1065. 

  1066.         metadata = metadata.update(('level1', 'level2', 'level3', 'level4'), {'level4': 'foobar4'})

  1067. 

  1068.         metadata = metadata.update(('level1', 'level2', 'level3', 'level4.1'), {'level4.1': 'foobar4.1'})

  1069. 

  1070.         self.assertEqual(utils.to_json_structure(metadata.to_internal_simple_structure()), [{

  1071.             'selector': [],

  1072.             'metadata': {'level0': 'foobar0'},

  1073.         }, {

  1074.             'selector': ['__ALL_ELEMENTS__'],

  1075.             'metadata': {'level1a': 'foobar1a', 'level1b': 'foobar1b'},

  1076.         }, {

  1077.             'selector': ['__ALL_ELEMENTS__', '__ALL_ELEMENTS__'],

  1078.             'metadata': {'level2a': 'foobar2a', 'level2b': 'foobar2b'},

  1079.         }, {

  1080.             'selector': ['__ALL_ELEMENTS__', '__ALL_ELEMENTS__', 'level3'],

  1081.             'metadata': {'level3a': 'foobar3a'},

  1082.         }, {

  1083.             'selector': ['level1'],

  1084.             'metadata': {'level1': 'foobar1', 'level1b': '__NO_VALUE__'},

  1085.         }, {

  1086.             'selector': ['level1', 'level2'],

  1087.             'metadata': {'level2': 'foobar2', 'level2b': '__NO_VALUE__'},

  1088.         }, {

  1089.             'selector': ['level1', 'level2', 'level3'],

  1090.             'metadata': {'level3': 'foobar3'},

  1091.         }, {

  1092.             'selector': ['level1', 'level2', 'level3', 'level4'],

  1093.             'metadata': {'level4': 'foobar4'},

  1094.         }, {

  1095.             'selector': ['level1', 'level2', 'level3', 'level4.1'],

  1096.             'metadata': {'level4.1': 'foobar4.1'},

  1097.         }, {

  1098.             'selector': ['level1', 'level2', 'level3.1'],

  1099.             'metadata': {'level3.1': 'foobar3.1'},

  1100.         }])

  1101. 

  1102.         self.assertEqual(metadata.query(('level1', 'level2')), {

  1103.             'level2a': 'foobar2a',

  1104.             'level2': 'foobar2',

  1105.         })

  1106. 

  1107.         target_metadata = base.DataMetadata({

  1108.             'schema': base.CONTAINER_SCHEMA_VERSION,

  1109.             'structural_type': container.DataFrame,

  1110.         })

  1111. 

  1112.         target_metadata = target_metadata.update((), {'level0z': 'foobar0z'})

  1113. 

  1114.         target_metadata = copy_elements_metadata(metadata, target_metadata, ())

  1115. 

  1116.         self.assertEqual(utils.to_json_structure(target_metadata.to_internal_simple_structure()), [{

  1117.             'selector': [],

  1118.             'metadata': {

  1119.                 'level0z': 'foobar0z',

  1120.                 'schema': base.CONTAINER_SCHEMA_VERSION,

  1121.                 'structural_type': 'd3m.container.pandas.DataFrame',

  1122.             },

  1123.         }, {

  1124.             'selector': ['__ALL_ELEMENTS__'],

  1125.             'metadata': {'level1a': 'foobar1a', 'level1b': 'foobar1b'},

  1126.         }, {

  1127.             'selector': ['__ALL_ELEMENTS__', '__ALL_ELEMENTS__'],

  1128.             'metadata': {'level2a': 'foobar2a', 'level2b': 'foobar2b'},

  1129.         }, {

  1130.             'selector': ['__ALL_ELEMENTS__', '__ALL_ELEMENTS__', 'level3'],

  1131.             'metadata': {'level3a': 'foobar3a'},

  1132.         }, {

  1133.             'selector': ['level1'],

  1134.             'metadata': {'level1': 'foobar1', 'level1b': '__NO_VALUE__'},

  1135.         }, {

  1136.             'selector': ['level1', 'level2'],

  1137.             'metadata': {'level2': 'foobar2', 'level2b': '__NO_VALUE__'},

  1138.         }, {

  1139.             'selector': ['level1', 'level2', 'level3'],

  1140.             'metadata': {'level3': 'foobar3'},

  1141.         }, {

  1142.             'selector': ['level1', 'level2', 'level3', 'level4'],

  1143.             'metadata': {'level4': 'foobar4'},

  1144.         }, {

  1145.             'selector': ['level1', 'level2', 'level3', 'level4.1'],

  1146.             'metadata': {'level4.1': 'foobar4.1'},

  1147.         }, {

  1148.             'selector': ['level1', 'level2', 'level3.1'],

  1149.             'metadata': {'level3.1': 'foobar3.1'},

  1150.         }])

  1151. 

  1152.         self.assertEqual(target_metadata.to_json_structure(), [{

  1153.             'selector': [],

  1154.             'metadata': {

  1155.                 'level0z': 'foobar0z',

  1156.                 'schema': base.CONTAINER_SCHEMA_VERSION,

  1157.                 'structural_type': 'd3m.container.pandas.DataFrame',

  1158.             },

  1159.         }, {

  1160.             'selector': ['__ALL_ELEMENTS__'],

  1161.             'metadata': {'level1a': 'foobar1a', 'level1b': 'foobar1b'},

  1162.         }, {

  1163.             'selector': ['__ALL_ELEMENTS__', '__ALL_ELEMENTS__'],

  1164.             'metadata': {'level2a': 'foobar2a', 'level2b': 'foobar2b'},

  1165.         }, {

  1166.             'selector': ['__ALL_ELEMENTS__', '__ALL_ELEMENTS__', 'level3'],

  1167.             'metadata': {'level3a': 'foobar3a'},

  1168.         }, {

  1169.             'selector': ['level1'],

  1170.             'metadata': {'level1': 'foobar1', 'level1a': 'foobar1a'},

  1171.         }, {

  1172.             'selector': ['level1', '__ALL_ELEMENTS__'],

  1173.             'metadata': {'level2a': 'foobar2a', 'level2b': 'foobar2b'},

  1174.         }, {

  1175.             'selector': ['level1', '__ALL_ELEMENTS__', 'level3'],

  1176.             'metadata': {'level3a': 'foobar3a'},

  1177.         }, {

  1178.             'selector': ['level1', 'level2'],

  1179.             'metadata': {'level2': 'foobar2', 'level2a': 'foobar2a'},

  1180.         }, {

  1181.             'selector': ['level1', 'level2', 'level3'],

  1182.             'metadata': {'level3': 'foobar3', 'level3a': 'foobar3a'},

  1183.         }, {

  1184.             'selector': ['level1', 'level2', 'level3', 'level4'],

  1185.             'metadata': {'level4': 'foobar4'},

  1186.         }, {

  1187.             'selector': ['level1', 'level2', 'level3', 'level4.1'],

  1188.             'metadata': {'level4.1': 'foobar4.1'},

  1189.         }, {

  1190.             'selector': ['level1', 'level2', 'level3.1'],

  1191.             'metadata': {'level3.1': 'foobar3.1'},

  1192.         }])

  1193. 

  1194.         target_metadata = base.DataMetadata({

  1195.             'schema': base.CONTAINER_SCHEMA_VERSION,

  1196.             'structural_type': container.DataFrame,

  1197.         })

  1198. 

  1199.         target_metadata = target_metadata.update((), {'level0z': 'foobar0z'})

  1200. 

  1201.         target_metadata = copy_elements_metadata(metadata, target_metadata, ('level1',))

  1202. 

  1203.         self.assertEqual(utils.to_json_structure(target_metadata.to_internal_simple_structure()), [{

  1204.             'selector': [],

  1205.             'metadata': {

  1206.                 'level0z': 'foobar0z',

  1207.                 'schema': base.CONTAINER_SCHEMA_VERSION,

  1208.                 'structural_type': 'd3m.container.pandas.DataFrame',

  1209.             },

  1210.         }, {

  1211.             'selector': ['__ALL_ELEMENTS__'],

  1212.             'metadata': {'level2a': 'foobar2a', 'level2b': 'foobar2b'},

  1213.         }, {

  1214.             'selector': ['__ALL_ELEMENTS__', 'level3'],

  1215.             'metadata': {'level3a': 'foobar3a'},

  1216.         }, {

  1217.             'selector': ['level2'],

  1218.             'metadata': {'level2': 'foobar2', 'level2b': '__NO_VALUE__'},

  1219.         }, {

  1220.             'selector': ['level2', 'level3'],

  1221.             'metadata': {'level3': 'foobar3'},

  1222.         }, {

  1223.             'selector': ['level2', 'level3', 'level4'],

  1224.             'metadata': {'level4': 'foobar4'},

  1225.         }, {

  1226.             'selector': ['level2', 'level3', 'level4.1'],

  1227.             'metadata': {'level4.1': 'foobar4.1'},

  1228.         }, {

  1229.             'selector': ['level2', 'level3.1'],

  1230.             'metadata': {'level3.1': 'foobar3.1'},

  1231.         }])

  1232. 

  1233.         self.assertEqual(target_metadata.query(('level2',)), {

  1234.             'level2a': 'foobar2a',

  1235.             'level2': 'foobar2',

  1236.         })

  1237. 

  1238.         target_metadata = base.DataMetadata({

  1239.             'schema': base.CONTAINER_SCHEMA_VERSION,

  1240.             'structural_type': container.DataFrame,

  1241.         })

  1242. 

  1243.         target_metadata = target_metadata.update((), {'level0z': 'foobar0z'})

  1244.         target_metadata = target_metadata.update(('zlevel',), {'level1z': 'foobar1z'})

  1245. 

  1246.         target_metadata = copy_elements_metadata(metadata, target_metadata, ('level1',), ('zlevel',))

  1247. 

  1248.         self.assertEqual(utils.to_json_structure(target_metadata.to_internal_simple_structure()), [{

  1249.             'selector': [],

  1250.             'metadata': {

  1251.                 'level0z': 'foobar0z',

  1252.                 'schema': base.CONTAINER_SCHEMA_VERSION,

  1253.                 'structural_type': 'd3m.container.pandas.DataFrame',

  1254.             },

  1255.         }, {

  1256.             'selector': ['zlevel'],

  1257.             'metadata': {'level1z': 'foobar1z'},

  1258.         }, {

  1259.             'selector': ['zlevel', '__ALL_ELEMENTS__'],

  1260.             'metadata': {'level2a': 'foobar2a', 'level2b': 'foobar2b'},

  1261.         }, {

  1262.             'selector': ['zlevel', '__ALL_ELEMENTS__', 'level3'],

  1263.             'metadata': {'level3a': 'foobar3a'},

  1264.         }, {

  1265.             'selector': ['zlevel', 'level2'],

  1266.             'metadata': {'level2': 'foobar2', 'level2b': '__NO_VALUE__'},

  1267.         }, {

  1268.             'selector': ['zlevel', 'level2', 'level3'],

  1269.             'metadata': {'level3': 'foobar3'},

  1270.         }, {

  1271.             'selector': ['zlevel', 'level2', 'level3', 'level4'],

  1272.             'metadata': {'level4': 'foobar4'},

  1273.         }, {

  1274.             'selector': ['zlevel', 'level2', 'level3', 'level4.1'],

  1275.             'metadata': {'level4.1': 'foobar4.1'},

  1276.         }, {

  1277.             'selector': ['zlevel', 'level2', 'level3.1'],

  1278.             'metadata': {'level3.1': 'foobar3.1'},

  1279.         }])

  1280. 

  1281.         self.assertEqual(target_metadata.query(('zlevel', 'level2',)), {

  1282.             'level2a': 'foobar2a',

  1283.             'level2': 'foobar2',

  1284.         })

  1285. 

  1286.         target_metadata = base.DataMetadata({

  1287.             'schema': base.CONTAINER_SCHEMA_VERSION,

  1288.             'structural_type': container.DataFrame,

  1289.         })

  1290. 

  1291.         target_metadata = target_metadata.update((), {'level0z': 'foobar0z'})

  1292. 

  1293.         target_metadata = copy_elements_metadata(metadata, target_metadata, ('level1', 'level2'))

  1294. 

  1295.         self.assertEqual(utils.to_json_structure(target_metadata.to_internal_simple_structure()), [{

  1296.             'selector': [],

  1297.             'metadata': {

  1298.                 'level0z': 'foobar0z',

  1299.                 'schema': base.CONTAINER_SCHEMA_VERSION,

  1300.                 'structural_type': 'd3m.container.pandas.DataFrame',

  1301.             },

  1302.         }, {

  1303.             'selector': ['level3'],

  1304.             'metadata': {'level3': 'foobar3', 'level3a': 'foobar3a'},

  1305.         }, {

  1306.             'selector': ['level3', 'level4'],

  1307.             'metadata': {'level4': 'foobar4'},

  1308.         }, {

  1309.             'selector': ['level3', 'level4.1'],

  1310.             'metadata': {'level4.1': 'foobar4.1'},

  1311.         }, {

  1312.             'selector': ['level3.1'],

  1313.             'metadata': {'level3.1': 'foobar3.1'},

  1314.         }])

  1315. 

  1316.         target_metadata = base.DataMetadata({

  1317.             'schema': base.CONTAINER_SCHEMA_VERSION,

  1318.             'structural_type': container.DataFrame,

  1319.         })

  1320. 

  1321.         target_metadata = target_metadata.update((), {'level0z': 'foobar0z'})

  1322.         target_metadata = target_metadata.update(('zlevel',), {'level1z': 'foobar1z'})

  1323. 

  1324.         target_metadata = copy_elements_metadata(metadata, target_metadata, ('level1', 'level2'), ('zlevel',))

  1325. 

  1326.         self.assertEqual(utils.to_json_structure(target_metadata.to_internal_simple_structure()), [{

  1327.             'selector': [],

  1328.             'metadata': {

  1329.                 'level0z': 'foobar0z',

  1330.                 'schema': base.CONTAINER_SCHEMA_VERSION,

  1331.                 'structural_type': 'd3m.container.pandas.DataFrame',

  1332.             },

  1333.         }, {

  1334.             'selector': ['zlevel'],

  1335.             'metadata': {'level1z': 'foobar1z'},

  1336.         }, {

  1337.             'selector': ['zlevel', 'level3'],

  1338.             'metadata': {'level3': 'foobar3', 'level3a': 'foobar3a'},

  1339.         }, {

  1340.             'selector': ['zlevel', 'level3', 'level4'],

  1341.             'metadata': {'level4': 'foobar4'},

  1342.         }, {

  1343.             'selector': ['zlevel', 'level3', 'level4.1'],

  1344.             'metadata': {'level4.1': 'foobar4.1'},

  1345.         }, {

  1346.             'selector': ['zlevel', 'level3.1'],

  1347.             'metadata': {'level3.1': 'foobar3.1'},

  1348.         }])

  1349. 

  1350.     def test_copy_metadata(self):

  1351.         metadata = base.Metadata()

  1352. 

  1353.         metadata = metadata.update((), {'level0': 'foobar0'})

  1354. 

  1355.         metadata = metadata.update(('level1',), {'level1': 'foobar1'})

  1356. 

  1357.         metadata = metadata.update((base.ALL_ELEMENTS,), {'level1a': 'foobar1a', 'level1b': 'foobar1b'})

  1358. 

  1359.         metadata = metadata.update(('level1',), {'level1b': base.NO_VALUE})

  1360. 

  1361.         metadata = metadata.update(('level1', 'level2'), {'level2': 'foobar2'})

  1362. 

  1363.         metadata = metadata.update((base.ALL_ELEMENTS, base.ALL_ELEMENTS), {'level2a': 'foobar2a', 'level2b': 'foobar2b'})

  1364. 

  1365.         metadata = metadata.update(('level1', 'level2'), {'level2b': base.NO_VALUE})

  1366. 

  1367.         metadata = metadata.update(('level1', 'level2', 'level3'), {'level3': 'foobar3'})

  1368. 

  1369.         metadata = metadata.update((base.ALL_ELEMENTS, base.ALL_ELEMENTS, 'level3'), {'level3a': 'foobar3a'})

  1370. 

  1371.         metadata = metadata.update(('level1', 'level2', 'level3.1'), {'level3.1': 'foobar3.1'})

  1372. 

  1373.         metadata = metadata.update(('level1', 'level2', 'level3', 'level4'), {'level4': 'foobar4'})

  1374. 

  1375.         metadata = metadata.update(('level1', 'level2', 'level3', 'level4.1'), {'level4.1': 'foobar4.1'})

  1376. 

  1377.         self.assertEqual(utils.to_json_structure(metadata.to_internal_simple_structure()), [{

  1378.             'selector': [],

  1379.             'metadata': {'level0': 'foobar0'},

  1380.         }, {

  1381.             'selector': ['__ALL_ELEMENTS__'],

  1382.             'metadata': {'level1a': 'foobar1a', 'level1b': 'foobar1b'},

  1383.         }, {

  1384.             'selector': ['__ALL_ELEMENTS__', '__ALL_ELEMENTS__'],

  1385.             'metadata': {'level2a': 'foobar2a', 'level2b': 'foobar2b'},

  1386.         }, {

  1387.             'selector': ['__ALL_ELEMENTS__', '__ALL_ELEMENTS__', 'level3'],

  1388.             'metadata': {'level3a': 'foobar3a'},

  1389.         }, {

  1390.             'selector': ['level1'],

  1391.             'metadata': {'level1': 'foobar1', 'level1b': '__NO_VALUE__'},

  1392.         }, {

  1393.             'selector': ['level1', 'level2'],

  1394.             'metadata': {'level2': 'foobar2', 'level2b': '__NO_VALUE__'},

  1395.         }, {

  1396.             'selector': ['level1', 'level2', 'level3'],

  1397.             'metadata': {'level3': 'foobar3'},

  1398.         }, {

  1399.             'selector': ['level1', 'level2', 'level3', 'level4'],

  1400.             'metadata': {'level4': 'foobar4'},

  1401.         }, {

  1402.             'selector': ['level1', 'level2', 'level3', 'level4.1'],

  1403.             'metadata': {'level4.1': 'foobar4.1'},

  1404.         }, {

  1405.             'selector': ['level1', 'level2', 'level3.1'],

  1406.             'metadata': {'level3.1': 'foobar3.1'},

  1407.         }])

  1408. 

  1409.         self.assertEqual(metadata.query(('level1', 'level2')), {

  1410.             'level2a': 'foobar2a',

  1411.             'level2': 'foobar2',

  1412.         })

  1413. 

  1414.         target_metadata = base.DataMetadata({

  1415.             'schema': base.CONTAINER_SCHEMA_VERSION,

  1416.             'structural_type': container.DataFrame,

  1417.         })

  1418. 

  1419.         target_metadata = target_metadata.update((), {'level0z': 'foobar0z'})

  1420. 

  1421.         target_metadata = metadata.copy_to(target_metadata, ())

  1422. 

  1423.         self.assertEqual(utils.to_json_structure(target_metadata.to_internal_simple_structure()), [{

  1424.             'selector': [],

  1425.             'metadata': {

  1426.                 'level0': 'foobar0',

  1427.                 'level0z': 'foobar0z',

  1428.                 'schema': base.CONTAINER_SCHEMA_VERSION,

  1429.                 'structural_type': 'd3m.container.pandas.DataFrame',

  1430.             },

  1431.         }, {

  1432.             'selector': ['__ALL_ELEMENTS__'],

  1433.             'metadata': {'level1a': 'foobar1a', 'level1b': 'foobar1b'},

  1434.         }, {

  1435.             'selector': ['__ALL_ELEMENTS__', '__ALL_ELEMENTS__'],

  1436.             'metadata': {'level2a': 'foobar2a', 'level2b': 'foobar2b'},

  1437.         }, {

  1438.             'selector': ['__ALL_ELEMENTS__', '__ALL_ELEMENTS__', 'level3'],

  1439.             'metadata': {'level3a': 'foobar3a'},

  1440.         }, {

  1441.             'selector': ['level1'],

  1442.             'metadata': {'level1': 'foobar1', 'level1b': '__NO_VALUE__'},

  1443.         }, {

  1444.             'selector': ['level1', 'level2'],

  1445.             'metadata': {'level2': 'foobar2', 'level2b': '__NO_VALUE__'},

  1446.         }, {

  1447.             'selector': ['level1', 'level2', 'level3'],

  1448.             'metadata': {'level3': 'foobar3'},

  1449.         }, {

  1450.             'selector': ['level1', 'level2', 'level3', 'level4'],

  1451.             'metadata': {'level4': 'foobar4'},

  1452.         }, {

  1453.             'selector': ['level1', 'level2', 'level3', 'level4.1'],

  1454.             'metadata': {'level4.1': 'foobar4.1'},

  1455.         }, {

  1456.             'selector': ['level1', 'level2', 'level3.1'],

  1457.             'metadata': {'level3.1': 'foobar3.1'},

  1458.         }])

  1459. 

  1460.         target_metadata = base.DataMetadata({

  1461.             'schema': base.CONTAINER_SCHEMA_VERSION,

  1462.             'structural_type': container.DataFrame,

  1463.         })

  1464. 

  1465.         target_metadata = target_metadata.update((), {'level0z': 'foobar0z'})

  1466. 

  1467.         target_metadata = metadata.copy_to(target_metadata, ('level1',))

  1468. 

  1469.         self.assertEqual(utils.to_json_structure(target_metadata.to_internal_simple_structure()), [{

  1470.             'selector': [],

  1471.             'metadata': {

  1472.                 'level0z': 'foobar0z',

  1473.                 'level1': 'foobar1',

  1474.                 'level1b': '__NO_VALUE__',

  1475.                 'level1a': 'foobar1a',

  1476.                 'schema': base.CONTAINER_SCHEMA_VERSION,

  1477.                 'structural_type': 'd3m.container.pandas.DataFrame',

  1478.             },

  1479.         }, {

  1480.             'selector': ['__ALL_ELEMENTS__'],

  1481.             'metadata': {'level2a': 'foobar2a', 'level2b': 'foobar2b'},

  1482.         }, {

  1483.             'selector': ['__ALL_ELEMENTS__', 'level3'],

  1484.             'metadata': {'level3a': 'foobar3a'},

  1485.         }, {

  1486.             'selector': ['level2'],

  1487.             'metadata': {'level2': 'foobar2', 'level2b': '__NO_VALUE__'},

  1488.         }, {

  1489.             'selector': ['level2', 'level3'],

  1490.             'metadata': {'level3': 'foobar3'},

  1491.         }, {

  1492.             'selector': ['level2', 'level3', 'level4'],

  1493.             'metadata': {'level4': 'foobar4'},

  1494.         }, {

  1495.             'selector': ['level2', 'level3', 'level4.1'],

  1496.             'metadata': {'level4.1': 'foobar4.1'},

  1497.         }, {

  1498.             'selector': ['level2', 'level3.1'],

  1499.             'metadata': {'level3.1': 'foobar3.1'},

  1500.         }])

  1501. 

  1502.         self.assertEqual(target_metadata.query(('level2',)), {

  1503.             'level2a': 'foobar2a',

  1504.             'level2': 'foobar2',

  1505.         })

  1506. 

  1507.         target_metadata = base.DataMetadata({

  1508.             'schema': base.CONTAINER_SCHEMA_VERSION,

  1509.             'structural_type': container.DataFrame,

  1510.         })

  1511. 

  1512.         target_metadata = target_metadata.update((), {'level0z': 'foobar0z'})

  1513.         target_metadata = target_metadata.update(('zlevel',), {'level1z': 'foobar1z'})

  1514. 

  1515.         target_metadata = metadata.copy_to(target_metadata, ('level1',), ('zlevel',))

  1516. 

  1517.         self.assertEqual(utils.to_json_structure(target_metadata.to_internal_simple_structure()), [{

  1518.             'selector': [],

  1519.             'metadata': {

  1520.                 'level0z': 'foobar0z',

  1521.                 'schema': base.CONTAINER_SCHEMA_VERSION,

  1522.                 'structural_type': 'd3m.container.pandas.DataFrame',

  1523.             },

  1524.         }, {

  1525.             'selector': ['zlevel'],

  1526.             'metadata': {'level1z': 'foobar1z', 'level1': 'foobar1', 'level1b': '__NO_VALUE__', 'level1a': 'foobar1a'},

  1527.         }, {

  1528.             'selector': ['zlevel', '__ALL_ELEMENTS__'],

  1529.             'metadata': {'level2a': 'foobar2a', 'level2b': 'foobar2b'},

  1530.         }, {

  1531.             'selector': ['zlevel', '__ALL_ELEMENTS__', 'level3'],

  1532.             'metadata': {'level3a': 'foobar3a'},

  1533.         }, {

  1534.             'selector': ['zlevel', 'level2'],

  1535.             'metadata': {'level2': 'foobar2', 'level2b': '__NO_VALUE__'},

  1536.         }, {

  1537.             'selector': ['zlevel', 'level2', 'level3'],

  1538.             'metadata': {'level3': 'foobar3'},

  1539.         }, {

  1540.             'selector': ['zlevel', 'level2', 'level3', 'level4'],

  1541.             'metadata': {'level4': 'foobar4'},

  1542.         }, {

  1543.             'selector': ['zlevel', 'level2', 'level3', 'level4.1'],

  1544.             'metadata': {'level4.1': 'foobar4.1'},

  1545.         }, {

  1546.             'selector': ['zlevel', 'level2', 'level3.1'],

  1547.             'metadata': {'level3.1': 'foobar3.1'},

  1548.         }])

  1549. 

  1550.         self.assertEqual(target_metadata.query(('zlevel', 'level2',)), {

  1551.             'level2a': 'foobar2a',

  1552.             'level2': 'foobar2',

  1553.         })

  1554. 

  1555.         target_metadata = base.DataMetadata({

  1556.             'schema': base.CONTAINER_SCHEMA_VERSION,

  1557.             'structural_type': container.DataFrame,

  1558.         })

  1559. 

  1560.         target_metadata = target_metadata.update((), {'level0z': 'foobar0z'})

  1561. 

  1562.         target_metadata = metadata.copy_to(target_metadata, ('level1', 'level2'))

  1563. 

  1564.         self.assertEqual(utils.to_json_structure(target_metadata.to_internal_simple_structure()), [{

  1565.             'selector': [],

  1566.             'metadata': {

  1567.                 'level0z': 'foobar0z',

  1568.                 'level2': 'foobar2',

  1569.                 'level2b': '__NO_VALUE__',

  1570.                 'level2a': 'foobar2a',

  1571.                 'schema': base.CONTAINER_SCHEMA_VERSION,

  1572.                 'structural_type': 'd3m.container.pandas.DataFrame',

  1573.             },

  1574.         }, {

  1575.             'selector': ['level3'],

  1576.             'metadata': {'level3': 'foobar3', 'level3a': 'foobar3a'},

  1577.         }, {

  1578.             'selector': ['level3', 'level4'],

  1579.             'metadata': {'level4': 'foobar4'},

  1580.         }, {

  1581.             'selector': ['level3', 'level4.1'],

  1582.             'metadata': {'level4.1': 'foobar4.1'},

  1583.         }, {

  1584.             'selector': ['level3.1'],

  1585.             'metadata': {'level3.1': 'foobar3.1'},

  1586.         }])

  1587. 

  1588.         target_metadata = base.DataMetadata({

  1589.             'schema': base.CONTAINER_SCHEMA_VERSION,

  1590.             'structural_type': container.DataFrame,

  1591.         })

  1592. 

  1593.         target_metadata = target_metadata.update((), {'level0z': 'foobar0z'})

  1594.         target_metadata = target_metadata.update(('zlevel',), {'level1z': 'foobar1z'})

  1595. 

  1596.         target_metadata = metadata.copy_to(target_metadata, ('level1', 'level2'), ('zlevel',))

  1597. 

  1598.         self.assertEqual(utils.to_json_structure(target_metadata.to_internal_simple_structure()), [{

  1599.             'selector': [],

  1600.             'metadata': {

  1601.                 'level0z': 'foobar0z',

  1602.                 'schema': base.CONTAINER_SCHEMA_VERSION,

  1603.                 'structural_type': 'd3m.container.pandas.DataFrame',

  1604.             },

  1605.         }, {

  1606.             'selector': ['zlevel'],

  1607.             'metadata': {'level1z': 'foobar1z', 'level2': 'foobar2', 'level2b': '__NO_VALUE__', 'level2a': 'foobar2a'},

  1608.         }, {

  1609.             'selector': ['zlevel', 'level3'],

  1610.             'metadata': {'level3': 'foobar3', 'level3a': 'foobar3a'},

  1611.         }, {

  1612.             'selector': ['zlevel', 'level3', 'level4'],

  1613.             'metadata': {'level4': 'foobar4'},

  1614.         }, {

  1615.             'selector': ['zlevel', 'level3', 'level4.1'],

  1616.             'metadata': {'level4.1': 'foobar4.1'},

  1617.         }, {

  1618.             'selector': ['zlevel', 'level3.1'],

  1619.             'metadata': {'level3.1': 'foobar3.1'},

  1620.         }])

  1621. 

  1622.     def test_get_index_columns(self):

  1623.         main = container.DataFrame({'a1': [1, 2, 3], 'b1': [4, 5, 6]}, generate_metadata=True)

  1624. 

  1625.         main.metadata = main.metadata.update((base.ALL_ELEMENTS, 0), {

  1626.             'name': 'image',

  1627.             'semantic_types': ['https://metadata.datadrivendiscovery.org/types/PrimaryKey'],

  1628.         })

  1629.         main.metadata = main.metadata.update((base.ALL_ELEMENTS, 1), {

  1630.             'name': 'd3mIndex',

  1631.             'semantic_types': ['https://metadata.datadrivendiscovery.org/types/PrimaryKey'],

  1632.         })

  1633. 

  1634.         self.assertEqual(main.metadata.get_index_columns(), [1, 0])

  1635. 

  1636.     def test_query_field(self):

  1637.         md = base.Metadata()

  1638.         md = md.update((1,), {'key': 'value'})

  1639. 

  1640.         self.assertEqual(md.query_field((1,), 'key', strict_all_elements=False), 'value')

  1641.         self.assertEqual(md.query_field((1,), 'key', strict_all_elements=True), 'value')

  1642. 

  1643.         with self.assertRaises(KeyError):

  1644.             self.assertEqual(md.query_field((base.ALL_ELEMENTS,), 'key', strict_all_elements=False), 'value')

  1645.         self.assertEqual(md.query_field((base.ALL_ELEMENTS,), 'key', strict_all_elements=True), 'value')

  1646. 

  1647.         with self.assertRaises(KeyError):

  1648.             self.assertEqual(md.query_field((base.ALL_ELEMENTS,), 'key2', strict_all_elements=True), 'value')

  1649. 

  1650.         md = md.update((2,), {'key': 'value'})

  1651. 

  1652.         with self.assertRaises(KeyError):

  1653.             self.assertEqual(md.query_field((base.ALL_ELEMENTS,), 'key', strict_all_elements=False), 'value')

  1654.         self.assertEqual(md.query_field((base.ALL_ELEMENTS,), 'key', strict_all_elements=True), 'value')

  1655. 

  1656.         md = md.update((3,), {'key': 'value2'})

  1657. 

  1658.         with self.assertRaises(KeyError):

  1659.             self.assertEqual(md.query_field((base.ALL_ELEMENTS,), 'key', strict_all_elements=False), 'value')

  1660.         with self.assertRaises(KeyError):

  1661.             self.assertEqual(md.query_field((base.ALL_ELEMENTS,), 'key', strict_all_elements=True), 'value')

  1662. 

  1663.         md = md.update((base.ALL_ELEMENTS,), {'key': 'value'})

  1664. 

  1665.         self.assertEqual(md.query_field((base.ALL_ELEMENTS,), 'key', strict_all_elements=False), 'value')

  1666.         self.assertEqual(md.query_field((base.ALL_ELEMENTS,), 'key', strict_all_elements=True), 'value')

  1667. 

  1668.         self.assertEqual(md.query_field((1,), 'key', strict_all_elements=False), 'value')

  1669.         self.assertEqual(md.query_field((1,), 'key', strict_all_elements=True), 'value')

  1670. 

  1671.         md = md.update((3,), {'key': 'value2'})

  1672. 

  1673.         self.assertEqual(md.query_field((base.ALL_ELEMENTS,), 'key', strict_all_elements=False), 'value')

  1674.         with self.assertRaises(KeyError):

  1675.             self.assertEqual(md.query_field((base.ALL_ELEMENTS,), 'key', strict_all_elements=True), 'value')

  1676. 

  1677.     def test_query_field_with_exceptions(self):

  1678.         md = base.Metadata()

  1679.         md = md.update((1,), {'key': 'value'})

  1680.         md = md.update((2,), {'key': 'value2'})

  1681. 

  1682.         self.assertEqual(md.query_field_with_exceptions((1,), 'key'), ('value', {}))

  1683.         self.assertEqual(md.query_field_with_exceptions((2,), 'key'), ('value2', {}))

  1684.         with self.assertRaises(KeyError):

  1685.             md.query_field_with_exceptions((3,), 'key')

  1686. 

  1687.         self.assertEqual(md.query_field_with_exceptions((base.ALL_ELEMENTS,), 'key'), (base.NO_VALUE, {(1,): 'value', (2,): 'value2'}))

  1688. 

  1689.         # All elements ar require "key" field when there is no explicit ALL_ELEMENTS metadata.

  1690.         md = md.update((3,), {'key2': 'value'})

  1691. 

  1692.         with self.assertRaises(KeyError):

  1693.             md.query_field_with_exceptions((base.ALL_ELEMENTS,), 'key')

  1694. 

  1695.         md = md.update((base.ALL_ELEMENTS,), {'key': 'value'})

  1696. 

  1697.         self.assertEqual(md.query_field_with_exceptions((1,), 'key'), ('value', {}))

  1698.         self.assertEqual(md.query_field_with_exceptions((2,), 'key'), ('value', {}))

  1699. 

  1700.         self.assertEqual(md.query_field_with_exceptions((base.ALL_ELEMENTS,), 'key'), ('value', {}))

  1701. 

  1702.         md = md.update((3,), {'key': 'value2'})

  1703. 

  1704.         self.assertEqual(md.query_field_with_exceptions((base.ALL_ELEMENTS,), 'key'), ('value', {(3,): 'value2'}))

  1705. 

  1706.         # Setting same value as what ALL_ELEMENTS has should not add additional exception.

  1707.         md = md.update((4,), {'key': 'value'})

  1708. 

  1709.         self.assertEqual(md.query_field_with_exceptions((base.ALL_ELEMENTS,), 'key'), ('value', {(3,): 'value2'}))

  1710. 

  1711.         # Because ALL_ELEMENTS is set, any additional elements without "key" field are ignored.

  1712.         md = md.update((5,), {'key2': 'value'})

  1713. 

  1714.         self.assertEqual(md.query_field_with_exceptions((base.ALL_ELEMENTS,), 'key'), ('value', {(3,): 'value2'}))

  1715. 

  1716.     def test_compact_generated_metadata(self):

  1717.         ALL_GENERATED_KEYS = ['foo', 'name', 'other', 'structural_type']

  1718. 

  1719.         compacted_metadata = base.DataMetadata._compact_metadata({}, ALL_GENERATED_KEYS)

  1720. 

  1721.         self.assertEqual(compacted_metadata, {})

  1722. 

  1723.         # All equal.

  1724.         new_metadata = {

  1725.             ('a',): {'foo': 'bar', 'other': 1},

  1726.             ('b',): {'foo': 'bar', 'other': 2},

  1727.             ('c',): {'foo': 'bar', 'other': 3},

  1728.         }

  1729. 

  1730.         compacted_metadata = base.DataMetadata._compact_metadata(new_metadata, ALL_GENERATED_KEYS)

  1731. 

  1732.         self.assertEqual(compacted_metadata, {

  1733.             (base.ALL_ELEMENTS,): {'foo': 'bar'},

  1734.             ('a',): {'other': 1},

  1735.             ('b',): {'other': 2},

  1736.             ('c',): {'other': 3},

  1737.         })

  1738. 

  1739.         # One different.

  1740.         new_metadata = {

  1741.             ('a',): {'foo': 'bar', 'other': 1},

  1742.             ('b',): {'foo': 'bar', 'other': 2},

  1743.             ('c',): {'foo': 'bar2', 'other': 3,},

  1744.         }

  1745. 

  1746.         compacted_metadata = base.DataMetadata._compact_metadata(new_metadata, ALL_GENERATED_KEYS)

  1747. 

  1748.         self.assertEqual(compacted_metadata, {

  1749.             ('a',): {'foo': 'bar', 'other': 1},

  1750.             ('b',): {'foo': 'bar', 'other': 2},

  1751.             ('c',): {'foo': 'bar2', 'other': 3,},

  1752.         })

  1753. 

  1754.         # Recursive.

  1755.         new_metadata = {

  1756.             ('deep', 'a'): {'foo': 'bar', 'other': 1},

  1757.             ('deep', 'b'): {'foo': 'bar', 'other': 2},

  1758.             ('deep', 'c'): {'foo': 'bar', 'other': 3},

  1759.         }

  1760. 

  1761.         compacted_metadata = base.DataMetadata._compact_metadata(new_metadata, ALL_GENERATED_KEYS)

  1762. 

  1763.         self.assertEqual(compacted_metadata, {

  1764.             (base.ALL_ELEMENTS, base.ALL_ELEMENTS): {'foo': 'bar'},

  1765.             (base.ALL_ELEMENTS, 'a'): {'other': 1},

  1766.             (base.ALL_ELEMENTS, 'b'): {'other': 2},

  1767.             (base.ALL_ELEMENTS, 'c'): {'other': 3},

  1768.         })

  1769. 

  1770.         new_metadata = {

  1771.             ('deep', 'a'): {'foo': 'bar', 'other': 1},

  1772.             ('deep', 'b'): {'foo': 'bar', 'other': 2},

  1773.             ('deep', 'c'): {'foo': 'bar', 'other': 3},

  1774.             ('deep2', 'd'): {'foo': 'bar', 'other': 4},

  1775.         }

  1776. 

  1777.         compacted_metadata = base.DataMetadata._compact_metadata(new_metadata, ALL_GENERATED_KEYS)

  1778. 

  1779.         self.assertEqual(compacted_metadata, {

  1780.             (base.ALL_ELEMENTS, base.ALL_ELEMENTS): {'foo': 'bar'},

  1781.             (base.ALL_ELEMENTS, 'a'): {'other': 1},

  1782.             (base.ALL_ELEMENTS, 'b'): {'other': 2},

  1783.             (base.ALL_ELEMENTS, 'c'): {'other': 3},

  1784.             (base.ALL_ELEMENTS, 'd'): {'other': 4},

  1785.         })

  1786. 

  1787.         new_metadata = {

  1788.             ('deep', 'a'): {'foo': 'bar', 'other': 1},

  1789.             ('deep', 'b'): {'foo': 'bar', 'other': 2},

  1790.             ('deep', 'c'): {'foo': 'bar', 'other': 3},

  1791.             ('deep2', 'a'): {'foo': 'bar', 'other': 4},

  1792.             ('deep2', 'b'): {'foo': 'bar', 'other': 5},

  1793.         }

  1794. 

  1795.         compacted_metadata = base.DataMetadata._compact_metadata(new_metadata, ALL_GENERATED_KEYS)

  1796. 

  1797.         self.assertEqual(compacted_metadata, {

  1798.             (base.ALL_ELEMENTS, base.ALL_ELEMENTS): {'foo': 'bar'},

  1799.             (base.ALL_ELEMENTS, 'c'): {'other': 3},

  1800.             ('deep', 'a'): {'other': 1},

  1801.             ('deep', 'b'): {'other': 2},

  1802.             ('deep2', 'a'): {'other': 4},

  1803.             ('deep2', 'b'): {'other': 5},

  1804.         })

  1805. 

  1806.         new_metadata = {

  1807.             ('deep', 'a'): {'foo': 'bar', 'other': 1},

  1808.             ('deep', 'b'): {'foo': 'bar', 'other': 2},

  1809.             ('deep', 'c'): {'foo': 'bar2', 'other': 3},

  1810.             ('deep2', 'a'): {'foo': 'bar', 'other': 4},

  1811.             ('deep2', 'b'): {'foo': 'bar', 'other': 5},

  1812.         }

  1813. 

  1814.         compacted_metadata = base.DataMetadata._compact_metadata(new_metadata, ALL_GENERATED_KEYS)

  1815. 

  1816.         self.assertEqual(compacted_metadata, {

  1817.             (base.ALL_ELEMENTS, 'a'): {'foo': 'bar'},

  1818.             (base.ALL_ELEMENTS, 'b'): {'foo': 'bar'},

  1819.             (base.ALL_ELEMENTS, 'c'): {'foo': 'bar2', 'other': 3},

  1820.             ('deep', 'a'): {'other': 1},

  1821.             ('deep', 'b'): {'other': 2},

  1822.             ('deep2', 'a'): {'other': 4},

  1823.             ('deep2', 'b'): {'other': 5},

  1824.         })

  1825. 

  1826.         new_metadata = {

  1827.             ('a', 'deep'): {'foo': 'bar', 'other': 1},

  1828.             ('b', 'deep'): {'foo': 'bar', 'other': 2},

  1829.             ('c', 'deep'): {'foo': 'bar2', 'other': 3},

  1830.             ('a', 'deep2'): {'foo': 'bar', 'other': 4},

  1831.             ('b', 'deep2'): {'foo': 'bar', 'other': 5},

  1832.         }

  1833. 

  1834.         compacted_metadata = base.DataMetadata._compact_metadata(new_metadata, ALL_GENERATED_KEYS)

  1835. 

  1836.         self.assertEqual(compacted_metadata, {

  1837.             ('a', base.ALL_ELEMENTS): {'foo': 'bar'},

  1838.             ('a', 'deep'): {'other': 1},

  1839.             ('a', 'deep2'): {'other': 4},

  1840.             ('b', base.ALL_ELEMENTS): {'foo': 'bar'},

  1841.             ('b', 'deep'): {'other': 2},

  1842.             ('b', 'deep2'): {'other': 5},

  1843.             ('c', base.ALL_ELEMENTS): {'foo': 'bar2', 'other': 3},

  1844.         })

  1845. 

  1846.         new_metadata = {

  1847.             (base.ALL_ELEMENTS, 'a'): {'foo': 'bar', 'other': 1},

  1848.             (base.ALL_ELEMENTS, 'b'): {'foo': 'bar', 'other': 2},

  1849.             (base.ALL_ELEMENTS, 'c'): {'foo': 'bar', 'other': 3},

  1850.         }

  1851. 

  1852.         compacted_metadata = base.DataMetadata._compact_metadata(new_metadata, ALL_GENERATED_KEYS)

  1853. 

  1854.         self.assertEqual(compacted_metadata, {

  1855.             (base.ALL_ELEMENTS, base.ALL_ELEMENTS): {'foo': 'bar'},

  1856.             (base.ALL_ELEMENTS, 'a'): {'other': 1},

  1857.             (base.ALL_ELEMENTS, 'b'): {'other': 2},

  1858.             (base.ALL_ELEMENTS, 'c'): {'other': 3},

  1859.         })

  1860. 

  1861.         new_metadata = {

  1862.             (base.ALL_ELEMENTS, 0): {'foo': 'bar1'},

  1863.             (0, 1): {'foo': 'bar2'},

  1864.             (1, 1): {'foo': 'bar2'},

  1865.             (2, 1): {'foo': 'bar2'},

  1866.         }

  1867. 

  1868.         compacted_metadata = base.DataMetadata._compact_metadata(new_metadata, ALL_GENERATED_KEYS)

  1869. 

  1870.         self.assertEqual(compacted_metadata, {

  1871.             (base.ALL_ELEMENTS, 0): {'foo': 'bar1'},

  1872.             (base.ALL_ELEMENTS, 1): {'foo': 'bar2'},

  1873.         })

  1874. 

  1875.         new_metadata = {

  1876.             ('deep1', 'a'): {'foo': 'bar', 'other': 1},

  1877.             ('deep1', 'b'): {'foo': 'bar2', 'other': 2},

  1878.             ('deep2', 'a'): {'foo': 'bar', 'other': 3},

  1879.             ('deep2', 'b'): {'foo': 'bar2', 'other': 4},

  1880.             ('deep3', 'a'): {'foo': 'bar', 'other': 5},

  1881.             ('deep3', 'b'): {'foo': 'bar2', 'other': 6},

  1882.         }

  1883. 

  1884.         compacted_metadata = base.DataMetadata._compact_metadata(new_metadata, ALL_GENERATED_KEYS)

  1885. 

  1886.         self.assertEqual(compacted_metadata, {

  1887.             (base.ALL_ELEMENTS, 'a'): {'foo': 'bar'},

  1888.             (base.ALL_ELEMENTS, 'b'): {'foo': 'bar2'},

  1889.             ('deep1', 'a'): {'other': 1},

  1890.             ('deep1', 'b'): {'other': 2},

  1891.             ('deep2', 'a'): {'other': 3},

  1892.             ('deep2', 'b'): {'other': 4},

  1893.             ('deep3', 'a'): {'other': 5},

  1894.             ('deep3', 'b'): {'other': 6},

  1895.         })

  1896. 

  1897.         new_metadata = {

  1898.             ('deep1', 'a'): {'foo': 'bar', 'other': 1},

  1899.             ('deep1', 'b'): {'foo': 'bar', 'other': 2},

  1900.             ('deep2', 'c'): {'foo': 'bar', 'other': 3},

  1901.             ('deep2', 'd'): {'foo': 'bar', 'other': 4},

  1902.             ('deep3', 'e'): {'foo': 'bar', 'other': 5},

  1903.             ('deep3', 'f'): {'foo': 'bar', 'other': 6},

  1904.         }

  1905. 

  1906.         compacted_metadata = base.DataMetadata._compact_metadata(new_metadata, ALL_GENERATED_KEYS)

  1907. 

  1908.         self.assertEqual(compacted_metadata, {

  1909.             (base.ALL_ELEMENTS, base.ALL_ELEMENTS): {'foo': 'bar'},

  1910.             (base.ALL_ELEMENTS, 'a'): {'other': 1},

  1911.             (base.ALL_ELEMENTS, 'b'): {'other': 2},

  1912.             (base.ALL_ELEMENTS, 'c'): {'other': 3},

  1913.             (base.ALL_ELEMENTS, 'd'): {'other': 4},

  1914.             (base.ALL_ELEMENTS, 'e'): {'other': 5},

  1915.             (base.ALL_ELEMENTS, 'f'): {'other': 6},

  1916.         })

  1917. 

  1918.         new_metadata = {

  1919.             ('deep1', 'a', 1): {'foo': 'bar1', 'other': 1},

  1920.             ('deep2', 'a', 2): {'foo': 'bar1', 'other': 2},

  1921.             ('deep3', 'a', 3): {'foo': 'bar1', 'other': 3},

  1922.             ('deep4', 'a', 4): {'foo': 'bar1', 'other': 4},

  1923.             ('deep1', 'b', 1): {'foo': 'bar2', 'other': 5},

  1924.         }

  1925. 

  1926.         compacted_metadata = base.DataMetadata._compact_metadata(new_metadata, ALL_GENERATED_KEYS)

  1927. 

  1928.         self.assertEqual(compacted_metadata, {

  1929.             (base.ALL_ELEMENTS, base.ALL_ELEMENTS, 2): {'other': 2},

  1930.             (base.ALL_ELEMENTS, base.ALL_ELEMENTS, 3): {'other': 3},

  1931.             (base.ALL_ELEMENTS, base.ALL_ELEMENTS, 4): {'other': 4},

  1932.             (base.ALL_ELEMENTS, 'a', base.ALL_ELEMENTS): {'foo': 'bar1'},

  1933.             (base.ALL_ELEMENTS, 'a', 1): {'other': 1},

  1934.             (base.ALL_ELEMENTS, 'b', base.ALL_ELEMENTS): {'foo': 'bar2', 'other': 5},

  1935.         })

  1936. 

  1937.         new_metadata = {

  1938.             ('deep', 'a', 1): {'foo': 'bar1', 'other': 1},

  1939.             ('deep', 'a', 2): {'foo': 'bar1', 'other': 2},

  1940.             ('deep', 'b', 1): {'foo': 'bar2', 'other': 3},

  1941.             ('deep', 'b', 2): {'foo': 'bar2', 'other': 4},

  1942.         }

  1943. 

  1944.         compacted_metadata = base.DataMetadata._compact_metadata(new_metadata, ALL_GENERATED_KEYS)

  1945. 

  1946.         self.assertEqual(compacted_metadata, {

  1947.             (base.ALL_ELEMENTS, 'a', base.ALL_ELEMENTS): {'foo': 'bar1'},

  1948.             (base.ALL_ELEMENTS, 'a', 1): {'other': 1},

  1949.             (base.ALL_ELEMENTS, 'a', 2): {'other': 2},

  1950.             (base.ALL_ELEMENTS, 'b', base.ALL_ELEMENTS): {'foo': 'bar2'},

  1951.             (base.ALL_ELEMENTS, 'b', 1): {'other': 3},

  1952.             (base.ALL_ELEMENTS, 'b', 2): {'other': 4},

  1953.         })

  1954. 

  1955.         new_metadata =  {

  1956.             ('deep', 'a', 1): {'foo': 'bar1', 'other': 'bar1'},

  1957.             ('deep', 'a', 2): {'foo': 'bar1', 'other': 'bar2'},

  1958.             ('deep', 'b', 1): {'foo': 'bar2', 'other': 'bar1'},

  1959.             ('deep', 'b', 2): {'foo': 'bar2', 'other': 'bar2'},

  1960.         }

  1961. 

  1962.         compacted_metadata = base.DataMetadata._compact_metadata(new_metadata, ALL_GENERATED_KEYS)

  1963. 

  1964.         self.assertEqual(compacted_metadata, {

  1965.             (base.ALL_ELEMENTS, base.ALL_ELEMENTS, 1): {'other': 'bar1'},

  1966.             (base.ALL_ELEMENTS, base.ALL_ELEMENTS, 2): {'other': 'bar2'},

  1967.             (base.ALL_ELEMENTS, 'a', base.ALL_ELEMENTS): {'foo': 'bar1'},

  1968.             (base.ALL_ELEMENTS, 'b', base.ALL_ELEMENTS): {'foo': 'bar2'},

  1969.         })

  1970. 

  1971.         new_metadata = {

  1972.             ('deep1', 'a', 1): {'foo': 'bar1', 'other': 1},

  1973.             ('deep1', 'a', 2): {'foo': 'bar1', 'other': 2},

  1974.             ('deep2', 'a', 3): {'foo': 'bar1', 'other': 3},

  1975.             ('deep2', 'a', 4): {'foo': 'bar1', 'other': 4},

  1976.             ('deep1', 'b', 1): {'foo': 'bar2', 'other': 1},

  1977.             ('deep1', 'b', 2): {'foo': 'bar2', 'other': 2},

  1978.             ('deep2', 'b', 3): {'foo': 'bar2', 'other': 3},

  1979.             ('deep2', 'b', 4): {'foo': 'bar2', 'other': 4},

  1980.         }

  1981. 

  1982.         compacted_metadata = base.DataMetadata._compact_metadata(new_metadata, ALL_GENERATED_KEYS)

  1983. 

  1984.         self.assertEqual(compacted_metadata, {

  1985.             (base.ALL_ELEMENTS, base.ALL_ELEMENTS, 1): {'other': 1},

  1986.             (base.ALL_ELEMENTS, base.ALL_ELEMENTS, 2): {'other': 2},

  1987.             (base.ALL_ELEMENTS, base.ALL_ELEMENTS, 3): {'other': 3},

  1988.             (base.ALL_ELEMENTS, base.ALL_ELEMENTS, 4): {'other': 4},

  1989.             (base.ALL_ELEMENTS, 'a', base.ALL_ELEMENTS): {'foo': 'bar1'},

  1990.             (base.ALL_ELEMENTS, 'b', base.ALL_ELEMENTS): {'foo': 'bar2'},

  1991.         })

  1992. 

  1993.         new_metadata = {

  1994.             ('deep', 'a'): {'foo': 'bar', 'other': 1},

  1995.             ('deep', 'b'): {'foo': 'bar', 'other': 2},

  1996.             ('deep2', 'b'): {'other': 3},

  1997.             ('deep2', 'c'): {'foo': 'bar', 'other': 4},

  1998.         }

  1999. 

  2000.         compacted_metadata = base.DataMetadata._compact_metadata(new_metadata, ALL_GENERATED_KEYS)

  2001. 

  2002.         self.assertEqual(compacted_metadata, {

  2003.             (base.ALL_ELEMENTS, 'a'): {'other': 1},

  2004.             (base.ALL_ELEMENTS, 'c'): {'foo': 'bar', 'other': 4},

  2005.             ('deep',base.ALL_ELEMENTS): {'foo': 'bar'},

  2006.             ('deep', 'b'): {'other': 2},

  2007.             ('deep2', 'b'): {'other': 3},

  2008.         })

  2009. 

  2010.         new_metadata = {

  2011.             ('deep', 'a'): {'foo': 'bar', 'other': 1},

  2012.             ('deep', 'b'): {'foo': 'bar', 'other': 2},

  2013.             ('deep', 'c'): {'other': 3},

  2014.             ('deep2', 'd'): {'foo': 'bar', 'other': 4},

  2015.         }

  2016. 

  2017.         compacted_metadata = base.DataMetadata._compact_metadata(new_metadata, ALL_GENERATED_KEYS)

  2018. 

  2019.         self.assertEqual(compacted_metadata, {

  2020.             (base.ALL_ELEMENTS, 'a'): {'foo': 'bar', 'other': 1},

  2021.             (base.ALL_ELEMENTS, 'b'): {'foo': 'bar', 'other': 2},

  2022.             (base.ALL_ELEMENTS, 'c'): {'other': 3},

  2023.             (base.ALL_ELEMENTS, 'd'): {'foo':'bar', 'other': 4},

  2024.         })

  2025. 

  2026.         new_metadata = {

  2027.             (base.ALL_ELEMENTS, 0): {'structural_type': 'numpy.int64'},

  2028.             (0, 1): {'structural_type': 'str'},

  2029.             (1, 1): {'structural_type': 'str'},

  2030.             (2, 1): {'structural_type': 'str'},

  2031.             (base.ALL_ELEMENTS, 1): {'name': 'B'},

  2032.             (0, 0): {'structural_type': 'numpy.int64'},

  2033.             (1, 0): {'structural_type': 'numpy.int64'},

  2034.             (2, 0): {'structural_type': 'numpy.int64'},

  2035.         }

  2036. 

  2037.         compacted_metadata = base.DataMetadata._compact_metadata(new_metadata, ALL_GENERATED_KEYS)

  2038. 

  2039.         self.assertEqual(compacted_metadata, {

  2040.             (base.ALL_ELEMENTS, 0): {'structural_type': 'numpy.int64'},

  2041.             (base.ALL_ELEMENTS, 1): {'name': 'B', 'structural_type': 'str'},

  2042.         })

  2043. 

  2044.         new_metadata = {

  2045.             (base.ALL_ELEMENTS, base.ALL_ELEMENTS, 0): {'structural_type': 'numpy.int64'},

  2046.             (base.ALL_ELEMENTS, base.ALL_ELEMENTS, 1): {'structural_type': 'str'},

  2047.             ('0', base.ALL_ELEMENTS, 0): {'name': 'A', 'structural_type': 'numpy.int64'},

  2048.             ('0', base.ALL_ELEMENTS, 1): {'name': 'B', 'structural_type': 'str'},

  2049.         }

  2050. 

  2051.         compacted_metadata = base.DataMetadata._compact_metadata(new_metadata, ALL_GENERATED_KEYS)

  2052. 

  2053.         self.assertEqual(compacted_metadata, {

  2054.             (base.ALL_ELEMENTS, base.ALL_ELEMENTS, 0): {'structural_type': 'numpy.int64', 'name': 'A'},

  2055.             (base.ALL_ELEMENTS, base.ALL_ELEMENTS, 1): {'structural_type': 'str', 'name': 'B'},

  2056.         })

  2057. 

  2058.     def test_greedy_prune_metadata(self):

  2059.         # Warmup test 1.

  2060.         selectors_to_compact = [('a',), ('b',),('c',)]

  2061.         compacted_selector = [(base.ALL_ELEMENTS, )]

  2062. 

  2063.         pruned_selectors = base.DataMetadata._greedy_prune_selector(compacted_selector, selectors_to_compact)

  2064. 

  2065.         self.assertEqual(pruned_selectors, [

  2066.             (base.ALL_ELEMENTS, )

  2067.         ])

  2068. 

  2069.         # Warmup test 2.

  2070.         selectors_to_compact = [('deep', 'a'), ('deep', 'b'), ('deep', 'c'), ('deep2', 'd')]

  2071.         compacted_selector = [(base.ALL_ELEMENTS, base.ALL_ELEMENTS,)]

  2072. 

  2073.         pruned_selectors = base.DataMetadata._greedy_prune_selector(compacted_selector, selectors_to_compact)

  2074. 

  2075.         self.assertEqual(pruned_selectors, [

  2076.             (base.ALL_ELEMENTS, base.ALL_ELEMENTS,)

  2077.         ])

  2078. 

  2079.         # Check if it can remove unnecessary outputs.

  2080.         selectors_to_compact = [('deep', 'a'), ('deep', 'b'), ('deep2', 'a'), ('deep2', 'b')]

  2081.         compacted_selector = [(base.ALL_ELEMENTS, 'a'), (base.ALL_ELEMENTS, 'b'), ('deep', 'a'), ('deep2', 'b')]

  2082. 

  2083.         pruned_selectors = base.DataMetadata._greedy_prune_selector(compacted_selector, selectors_to_compact)

  2084. 

  2085.         self.assertEqual(pruned_selectors, [

  2086.             (base.ALL_ELEMENTS, 'a'), (base.ALL_ELEMENTS, 'b')

  2087.         ])

  2088. 

  2089.         # Case when compacted_selector overlaps.

  2090.         selectors_to_compact = [('a', 'deep'), ('b', 'deep'), ('a', 'deep2'), ('b', 'deep2')]

  2091.         compacted_selector = [('a', base.ALL_ELEMENTS), ('b', base.ALL_ELEMENTS), (base.ALL_ELEMENTS, 'deep2')]

  2092. 

  2093.         pruned_selectors = base.DataMetadata._greedy_prune_selector(compacted_selector, selectors_to_compact)

  2094. 

  2095.         self.assertEqual(pruned_selectors, [

  2096.             ('a', base.ALL_ELEMENTS), ('b', base.ALL_ELEMENTS)

  2097.         ])

  2098. 

  2099.         # Check the order.

  2100.         selectors_to_compact = [('a', 'deep'), ('b', 'deep'), ('a', 'deep2'), ('b', 'deep2')]

  2101.         compacted_selector = [(base.ALL_ELEMENTS, 'deep2'), ('a', base.ALL_ELEMENTS), ('b', base.ALL_ELEMENTS),]

  2102. 

  2103.         pruned_selectors = base.DataMetadata._greedy_prune_selector(compacted_selector, selectors_to_compact)

  2104. 

  2105.         self.assertEqual(pruned_selectors, [

  2106.             ('a', base.ALL_ELEMENTS), ('b', base.ALL_ELEMENTS)

  2107.         ])

  2108. 

  2109.         # More complex compacted_selectors.

  2110.         selectors_to_compact = [('a', 'deep'), ('b', 'deep'), ('a', 'deep2'), ('b', 'deep2')]

  2111.         compacted_selector = [(base.ALL_ELEMENTS, 'deep2'), ('a', base.ALL_ELEMENTS),

  2112.                               (base.ALL_ELEMENTS, 'deep'), ('b', base.ALL_ELEMENTS),]

  2113. 

  2114.         pruned_selectors = base.DataMetadata._greedy_prune_selector(compacted_selector, selectors_to_compact)

  2115. 

  2116.         self.assertEqual(pruned_selectors, [

  2117.             (base.ALL_ELEMENTS, 'deep2'), (base.ALL_ELEMENTS, 'deep')

  2118.         ])

  2119. 

  2120.         # All-elements in selectors_to_compact.

  2121.         selectors_to_compact = [('deep', 'a', 1), ('deep', base.ALL_ELEMENTS, 2)]

  2122.         compacted_selector = [(base.ALL_ELEMENTS, 'a', base.ALL_ELEMENTS), ('deep', base.ALL_ELEMENTS, 2)]

  2123. 

  2124.         pruned_selectors = base.DataMetadata._greedy_prune_selector(compacted_selector, selectors_to_compact)

  2125. 

  2126.         self.assertEqual(pruned_selectors, [

  2127.             (base.ALL_ELEMENTS, 'a', base.ALL_ELEMENTS), ('deep', base.ALL_ELEMENTS, 2)

  2128.         ])

  2129. 

  2130.     def test_semantic_types_merge(self):

  2131.         metadata = base.DataMetadata().update(('0',), {

  2132.             'semantic_types': ['https://metadata.datadrivendiscovery.org/types/DatasetEntryPoint'],

  2133.         })

  2134. 

  2135.         metadata_regular = metadata.update((base.ALL_ELEMENTS,), {

  2136.             'semantic_types': ['https://metadata.datadrivendiscovery.org/types/Table'],

  2137.         })

  2138. 

  2139.         self.assertEqual(metadata_regular.query(('0',)).get('semantic_types', None), ('https://metadata.datadrivendiscovery.org/types/Table',))

  2140. 

  2141.         metadata._update_with_generated_metadata({

  2142.             (base.ALL_ELEMENTS,): {

  2143.                 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/Table'],

  2144.             },

  2145.         })

  2146. 

  2147.         self.assertEqual(metadata.query(('0',)).get('semantic_types', None), ('https://metadata.datadrivendiscovery.org/types/DatasetEntryPoint', 'https://metadata.datadrivendiscovery.org/types/Table',))

  2148. 

  2149.     def test_compact(self):

  2150.         md = base.Metadata().update(('0',), {

  2151.             'key': 'value',

  2152.         })

  2153.         md = md.update(('1',), {

  2154.             'key': 'value',

  2155.         })

  2156. 

  2157.         md = md.compact(['key'])

  2158. 

  2159.         self.assertEqual(md.to_internal_json_structure(), [{

  2160.             'selector': ['__ALL_ELEMENTS__'],

  2161.             'metadata': {'key': 'value'},

  2162.         }])

  2163. 

  2164.         md = base.Metadata().update(('0',), {

  2165.             'key': 'value',

  2166.         })

  2167.         md = md.update(('1',), {

  2168.             'key': 'value',

  2169.         })

  2170.         md = md.update(('2',), {

  2171.             'key': 'value2',

  2172.         })

  2173. 

  2174.         md = md.compact(['key'])

  2175. 

  2176.         self.assertEqual(md.to_internal_json_structure(), [{

  2177.             'selector': ['0'],

  2178.             'metadata': {'key': 'value'},

  2179.         }, {

  2180.             'selector': ['1'],

  2181.             'metadata': {'key': 'value'},

  2182.         }, {

  2183.             'selector': ['2'],

  2184.             'metadata': {'key': 'value2'},

  2185.         }])

  2186. 

  2187.         md = base.Metadata().update(('0',), {

  2188.             'key': 'value',

  2189.             'key2': 'value',

  2190.         })

  2191.         md = md.update(('1',), {

  2192.             'key': 'value',

  2193.             'key2': 'value',

  2194.         })

  2195. 

  2196.         md = md.compact(['key'])

  2197. 

  2198.         self.assertEqual(md.to_internal_json_structure(), [{

  2199.             'selector': ['__ALL_ELEMENTS__'],

  2200.             'metadata': {'key': 'value'},

  2201.         }, {

  2202.             'selector': ['0'],

  2203.             'metadata': {'key2': 'value'},

  2204.         }, {

  2205.             'selector': ['1'],

  2206.             'metadata': {'key2': 'value'},

  2207.         }])

  2208. 

  2209. 

  2210. if __name__ == '__main__':

  2211.     unittest.main()