wangwei
/
tods
Not watched
1
0
Fork 0
Code
Releases 0 Wiki Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: cf2b170182
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'cf2b170182'
${ noResults }
tods/d3m/tests/test_base_utils.py

1036 lines
40 kB
Raw Blame History 

  1. import unittest

  2. 

  3. from d3m import container, utils as d3m_utils

  4. from d3m.base import utils

  5. from d3m.metadata import base as metadata_base

  6. 

  7. 

  8. class TestBaseUtils(unittest.TestCase):

  9.     def test_combine_columns_compact_metadata(self):

  10.         main = container.DataFrame({'a1': [1, 2, 3], 'b1': [4, 5, 6], 'c1': [7, 8, 9], 'd1': [10, 11, 12], 'e1': [13, 14, 15]}, {

  11.             'top_level': 'main',

  12.         }, generate_metadata=False)

  13.         main.metadata = main.metadata.generate(main, compact=True)

  14.         main.metadata = main.metadata.update_column(0, {'name': 'aaa111'})

  15.         main.metadata = main.metadata.update_column(1, {'name': 'bbb111', 'extra': 'b_column'})

  16.         main.metadata = main.metadata.update_column(2, {'name': 'ccc111'})

  17. 

  18.         columns2 = container.DataFrame({'a2': [21, 22, 23], 'b2': [24, 25, 26]}, {

  19.             'top_level': 'columns2',

  20.         }, generate_metadata=False)

  21.         columns2.metadata = columns2.metadata.generate(columns2, compact=True)

  22.         columns2.metadata = columns2.metadata.update_column(0, {'name': 'aaa222'})

  23.         columns2.metadata = columns2.metadata.update_column(1, {'name': 'bbb222'})

  24. 

  25.         columns3 = container.DataFrame({'a3': [31, 32, 33], 'b3': [34, 35, 36]}, {

  26.             'top_level': 'columns3',

  27.         }, generate_metadata=False)

  28.         columns3.metadata = columns3.metadata.generate(columns3, compact=True)

  29.         columns3.metadata = columns3.metadata.update_column(0, {'name': 'aaa333'})

  30.         columns3.metadata = columns3.metadata.update_column(1, {'name': 'bbb333'})

  31. 

  32.         result = utils.combine_columns(main, [1, 2], [columns2, columns3], return_result='append', add_index_columns=False)

  33. 

  34.         self.assertEqual(result.values.tolist(), [

  35.             [1, 4, 7, 10, 13, 21, 24, 31, 34],

  36.             [2, 5, 8, 11, 14, 22, 25, 32, 35],

  37.             [3, 6, 9, 12, 15, 23, 26, 33, 36],

  38.         ])

  39. 

  40.         self.assertEqual(d3m_utils.to_json_structure(result.metadata.to_internal_simple_structure()), [{

  41.             'selector': [],

  42.             'metadata': {

  43.                 'top_level': 'main',

  44.                 'schema': metadata_base.CONTAINER_SCHEMA_VERSION,

  45.                 'structural_type': 'd3m.container.pandas.DataFrame',

  46.                 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/Table'],

  47.                 'dimension': {

  48.                     'name': 'rows',

  49.                     'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularRow'],

  50.                     'length': 3,

  51.                 },

  52.             },

  53.         }, {

  54.             'selector': ['__ALL_ELEMENTS__'],

  55.             'metadata': {

  56.                 'dimension': {

  57.                     'name': 'columns',

  58.                     'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularColumn'],

  59.                     'length': 9,

  60.                 },

  61.             },

  62.         }, {

  63.             'selector': ['__ALL_ELEMENTS__', '__ALL_ELEMENTS__'],

  64.             'metadata': {

  65.                 'structural_type': 'numpy.int64',

  66.             },

  67.         }, {

  68.             'selector': ['__ALL_ELEMENTS__', 0],

  69.             'metadata': {

  70.                 'name': 'aaa111',

  71.             },

  72.         }, {

  73.             'selector': ['__ALL_ELEMENTS__', 1],

  74.             'metadata': {

  75.                 'name': 'bbb111',

  76.                 'extra': 'b_column',

  77.             },

  78.         }, {

  79.             'selector': ['__ALL_ELEMENTS__', 2],

  80.             'metadata': {

  81.                 'name': 'ccc111',

  82.             },

  83.         }, {

  84.             'selector': ['__ALL_ELEMENTS__', 3],

  85.             'metadata': {

  86.                 'name': 'd1',

  87.             },

  88.         }, {

  89.             'selector': ['__ALL_ELEMENTS__', 4],

  90.             'metadata': {

  91.                 'name': 'e1',

  92.             },

  93.         }, {

  94.             'selector': ['__ALL_ELEMENTS__', 5],

  95.             'metadata': {

  96.                 'name': 'aaa222',

  97.                 'structural_type': 'numpy.int64',

  98.             },

  99.         }, {

  100.             'selector': ['__ALL_ELEMENTS__', 6],

  101.             'metadata': {

  102.                 'name': 'bbb222',

  103.                 'structural_type': 'numpy.int64',

  104.             },

  105.         }, {

  106.             'selector': ['__ALL_ELEMENTS__', 7],

  107.             'metadata': {

  108.                 'name': 'aaa333',

  109.                 'structural_type': 'numpy.int64',

  110.             },

  111.         }, {

  112.             'selector': ['__ALL_ELEMENTS__', 8],

  113.             'metadata': {

  114.                 'name': 'bbb333',

  115.                 'structural_type': 'numpy.int64',

  116.             },

  117.         }])

  118. 

  119.         result = utils.combine_columns(main, [1, 2], [columns2, columns3], return_result='new', add_index_columns=False)

  120. 

  121.         self.assertEqual(result.values.tolist(), [

  122.             [21, 24, 31, 34],

  123.             [22, 25, 32, 35],

  124.             [23, 26, 33, 36],

  125.         ])

  126. 

  127.         self.assertEqual(d3m_utils.to_json_structure(result.metadata.to_internal_simple_structure()), [{

  128.             'selector': [],

  129.             'metadata': {

  130.                 'top_level': 'columns2',

  131.                 'schema': metadata_base.CONTAINER_SCHEMA_VERSION,

  132.                 'structural_type': 'd3m.container.pandas.DataFrame',

  133.                 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/Table'],

  134.                 'dimension': {

  135.                     'name': 'rows',

  136.                     'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularRow'],

  137.                     'length': 3,

  138.                 },

  139.             },

  140.         }, {

  141.             'selector': ['__ALL_ELEMENTS__'],

  142.             'metadata': {

  143.                 'dimension': {

  144.                     'name': 'columns',

  145.                     'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularColumn'],

  146.                     'length': 4,

  147.                 },

  148.             },

  149.         }, {

  150.             'selector': ['__ALL_ELEMENTS__', '__ALL_ELEMENTS__'],

  151.             'metadata': {

  152.                 'structural_type': 'numpy.int64',

  153.             },

  154.         }, {

  155.             'selector': ['__ALL_ELEMENTS__', 0],

  156.             'metadata': {

  157.                 'name': 'aaa222',

  158.             },

  159.         }, {

  160.             'selector': ['__ALL_ELEMENTS__', 1],

  161.             'metadata': {

  162.                 'name': 'bbb222',

  163.             },

  164.         }, {

  165.             'selector': ['__ALL_ELEMENTS__', 2],

  166.             'metadata': {

  167.                 'name': 'aaa333',

  168.                 'structural_type': 'numpy.int64',

  169.             },

  170.         }, {

  171.             'selector': ['__ALL_ELEMENTS__', 3],

  172.             'metadata': {

  173.                 'name': 'bbb333',

  174.                 'structural_type': 'numpy.int64',

  175.             },

  176.         }])

  177. 

  178.         result = utils.combine_columns(main, [1, 2], [columns2, columns3], return_result='replace', add_index_columns=False)

  179. 

  180.         self.assertEqual(result.values.tolist(), [

  181.             [1, 21, 24, 31, 34, 10, 13],

  182.             [2, 22, 25, 32, 35, 11, 14],

  183.             [3, 23, 26, 33, 36, 12, 15],

  184.         ])

  185. 

  186.         self.assertEqual(d3m_utils.to_json_structure(result.metadata.to_internal_simple_structure()), [{

  187.             'selector': [],

  188.             'metadata': {

  189.                 'top_level': 'main',

  190.                 'schema': metadata_base.CONTAINER_SCHEMA_VERSION,

  191.                 'structural_type': 'd3m.container.pandas.DataFrame',

  192.                 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/Table'],

  193.                 'dimension': {

  194.                     'name': 'rows',

  195.                     'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularRow'],

  196.                     'length': 3,

  197.                 },

  198.             },

  199.         }, {

  200.             'selector': ['__ALL_ELEMENTS__'],

  201.             'metadata': {

  202.                 'dimension': {

  203.                     'name': 'columns',

  204.                     'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularColumn'],

  205.                     'length': 7,

  206.                 },

  207.             },

  208.         }, {

  209.             'selector': ['__ALL_ELEMENTS__', '__ALL_ELEMENTS__'],

  210.             'metadata': {

  211.                 'structural_type': 'numpy.int64',

  212.             },

  213.         }, {

  214.             'selector': ['__ALL_ELEMENTS__', 0],

  215.             'metadata': {

  216.                 'name': 'aaa111',

  217.             },

  218.         }, {

  219.             'selector': ['__ALL_ELEMENTS__', 1],

  220.             'metadata': {

  221.                 'name': 'aaa222',

  222.                 'structural_type': 'numpy.int64',

  223.             },

  224.         }, {

  225.             'selector': ['__ALL_ELEMENTS__', 2],

  226.             'metadata': {

  227.                 'name': 'bbb222',

  228.                 'structural_type': 'numpy.int64',

  229.             },

  230.         }, {

  231.             'selector': ['__ALL_ELEMENTS__', 3],

  232.             'metadata': {

  233.                 'name': 'aaa333',

  234.                 'structural_type': 'numpy.int64',

  235.             },

  236.         }, {

  237.             'selector': ['__ALL_ELEMENTS__', 4],

  238.             'metadata': {

  239.                 'name': 'bbb333',

  240.                 'structural_type': 'numpy.int64',

  241.             },

  242.         }, {

  243.             'selector': ['__ALL_ELEMENTS__', 5],

  244.             'metadata': {

  245.                 'name': 'd1',

  246.                 'structural_type': 'numpy.int64',

  247.             },

  248.         }, {

  249.             'selector': ['__ALL_ELEMENTS__', 6],

  250.             'metadata': {

  251.                 'name': 'e1',

  252.                 'structural_type': 'numpy.int64',

  253.             },

  254.         }])

  255. 

  256.         result = utils.combine_columns(main, [0, 1, 2, 3, 4], [columns2, columns3], return_result='replace', add_index_columns=False)

  257. 

  258.         self.assertEqual(result.values.tolist(), [

  259.             [21, 24, 31, 34],

  260.             [22, 25, 32, 35],

  261.             [23, 26, 33, 36],

  262.         ])

  263. 

  264.         self.assertEqual(d3m_utils.to_json_structure(result.metadata.to_internal_simple_structure()), [{

  265.             'selector': [],

  266.             'metadata': {

  267.                 'top_level': 'main',

  268.                 'schema': metadata_base.CONTAINER_SCHEMA_VERSION,

  269.                 'structural_type': 'd3m.container.pandas.DataFrame',

  270.                 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/Table'],

  271.                 'dimension': {

  272.                     'name': 'rows',

  273.                     'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularRow'],

  274.                     'length': 3,

  275.                 },

  276.             },

  277.         }, {

  278.             'selector': ['__ALL_ELEMENTS__'],

  279.             'metadata': {

  280.                 'dimension': {

  281.                     'name': 'columns',

  282.                     'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularColumn'],

  283.                     'length': 4,

  284.                 },

  285.             },

  286.         }, {

  287.             'selector': ['__ALL_ELEMENTS__', '__ALL_ELEMENTS__'],

  288.             'metadata': {

  289.                 'structural_type': 'numpy.int64',

  290.             },

  291.         }, {

  292.             'selector': ['__ALL_ELEMENTS__', 0],

  293.             'metadata': {

  294.                 'name': 'aaa222',

  295.                 'structural_type': 'numpy.int64',

  296.             },

  297.         }, {

  298.             'selector': ['__ALL_ELEMENTS__', 1],

  299.             'metadata': {

  300.                 'name': 'bbb222',

  301.                 'structural_type': 'numpy.int64',

  302.             },

  303.         }, {

  304.             'selector': ['__ALL_ELEMENTS__', 2],

  305.             'metadata': {

  306.                 'name': 'aaa333',

  307.                 'structural_type': 'numpy.int64',

  308.             },

  309.         }, {

  310.             'selector': ['__ALL_ELEMENTS__', 3],

  311.             'metadata': {

  312.                 'name': 'bbb333',

  313.                 'structural_type': 'numpy.int64',

  314.             },

  315.         }])

  316. 

  317.         result = utils.combine_columns(main, [4], [columns2, columns3], return_result='replace', add_index_columns=False)

  318. 

  319.         self.assertEqual(result.values.tolist(), [

  320.             [1, 4, 7, 10, 21, 24, 31, 34],

  321.             [2, 5, 8, 11, 22, 25, 32, 35],

  322.             [3, 6, 9, 12, 23, 26, 33, 36],

  323.         ])

  324. 

  325.         self.assertEqual(d3m_utils.to_json_structure(result.metadata.to_internal_simple_structure()), [{

  326.             'selector': [],

  327.             'metadata': {

  328.                 'top_level': 'main',

  329.                 'schema': metadata_base.CONTAINER_SCHEMA_VERSION,

  330.                 'structural_type': 'd3m.container.pandas.DataFrame',

  331.                 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/Table'],

  332.                 'dimension': {

  333.                     'name': 'rows',

  334.                     'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularRow'],

  335.                     'length': 3,

  336.                 },

  337.             },

  338.         }, {

  339.             'selector': ['__ALL_ELEMENTS__'],

  340.             'metadata': {

  341.                 'dimension': {

  342.                     'name': 'columns',

  343.                     'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularColumn'],

  344.                     'length': 8,

  345.                 },

  346.             },

  347.         }, {

  348.             'selector': ['__ALL_ELEMENTS__', '__ALL_ELEMENTS__'],

  349.             'metadata': {

  350.                 'structural_type': 'numpy.int64',

  351.             },

  352.         }, {

  353.             'selector': ['__ALL_ELEMENTS__', 0],

  354.             'metadata': {

  355.                 'name': 'aaa111',

  356.             },

  357.         }, {

  358.             'selector': ['__ALL_ELEMENTS__', 1],

  359.             'metadata': {

  360.                 'name': 'bbb111',

  361.                 'extra': 'b_column',

  362.             },

  363.         }, {

  364.             'selector': ['__ALL_ELEMENTS__', 2],

  365.             'metadata': {

  366.                 'name': 'ccc111',

  367.             },

  368.         }, {

  369.             'selector': ['__ALL_ELEMENTS__', 3],

  370.             'metadata': {

  371.                 'name': 'd1',

  372.             },

  373.         }, {

  374.             'selector': ['__ALL_ELEMENTS__', 4],

  375.             'metadata': {

  376.                 'structural_type': 'numpy.int64',

  377.                 'name': 'aaa222',

  378.             },

  379.         }, {

  380.             'selector': ['__ALL_ELEMENTS__', 5],

  381.             'metadata': {

  382.                 'structural_type': 'numpy.int64',

  383.                 'name': 'bbb222',

  384.             },

  385.         }, {

  386.             'selector': ['__ALL_ELEMENTS__', 6],

  387.             'metadata': {

  388.                 'structural_type': 'numpy.int64',

  389.                 'name': 'aaa333',

  390.             },

  391.         }, {

  392.             'selector': ['__ALL_ELEMENTS__', 7],

  393.             'metadata': {

  394.                 'structural_type': 'numpy.int64',

  395.                 'name': 'bbb333',

  396.             },

  397.         }])

  398. 

  399.         result = utils.combine_columns(main, [0, 2, 4], [columns2, columns3], return_result='replace', add_index_columns=False)

  400. 

  401.         self.assertEqual(result.values.tolist(), [

  402.             [21, 4, 24, 10, 31, 34],

  403.             [22, 5, 25, 11, 32, 35],

  404.             [23, 6, 26, 12, 33, 36],

  405.         ])

  406. 

  407.         self.assertEqual(d3m_utils.to_json_structure(result.metadata.to_internal_simple_structure()), [{

  408.             'selector': [],

  409.             'metadata': {

  410.                 'top_level': 'main',

  411.                 'schema': metadata_base.CONTAINER_SCHEMA_VERSION,

  412.                 'structural_type': 'd3m.container.pandas.DataFrame',

  413.                 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/Table'],

  414.                 'dimension': {

  415.                     'name': 'rows',

  416.                     'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularRow'],

  417.                     'length': 3,

  418.                 },

  419.             },

  420.         }, {

  421.             'selector': ['__ALL_ELEMENTS__'],

  422.             'metadata': {

  423.                 'dimension': {

  424.                     'name': 'columns',

  425.                     'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularColumn'],

  426.                     'length': 6,

  427.                 },

  428.             },

  429.         }, {

  430.             'selector': ['__ALL_ELEMENTS__', '__ALL_ELEMENTS__'],

  431.             'metadata': {

  432.                 'structural_type': 'numpy.int64',

  433.             },

  434.         }, {

  435.             'selector': ['__ALL_ELEMENTS__', 0],

  436.             'metadata': {

  437.                 'name': 'aaa222',

  438.                 'structural_type': 'numpy.int64',

  439.             },

  440.         }, {

  441.             'selector': ['__ALL_ELEMENTS__', 1],

  442.             'metadata': {

  443.                 'name': 'bbb111',

  444.                 'extra': 'b_column',

  445.             },

  446.         }, {

  447.             'selector': ['__ALL_ELEMENTS__', 2],

  448.             'metadata': {

  449.                 'name': 'bbb222',

  450.                 'structural_type': 'numpy.int64',

  451.             },

  452.         }, {

  453.             'selector': ['__ALL_ELEMENTS__', 3],

  454.             'metadata': {

  455.                 'name': 'd1',

  456.             },

  457.         }, {

  458.             'selector': ['__ALL_ELEMENTS__', 4],

  459.             'metadata': {

  460.                 'name': 'aaa333',

  461.                 'structural_type': 'numpy.int64',

  462.             },

  463.         }, {

  464.             'selector': ['__ALL_ELEMENTS__', 5],

  465.             'metadata': {

  466.                 'name': 'bbb333',

  467.                 'structural_type': 'numpy.int64',

  468.             },

  469.         }])

  470.     def test_combine_columns_noncompact_metadata(self):

  471.         main = container.DataFrame({'a1': [1, 2, 3], 'b1': [4, 5, 6], 'c1': [7, 8, 9], 'd1': [10, 11, 12], 'e1': [13, 14, 15]}, {

  472.             'top_level': 'main',

  473.         }, generate_metadata=False)

  474.         main.metadata = main.metadata.generate(main, compact=False)

  475.         main.metadata = main.metadata.update_column(0, {'name': 'aaa111'})

  476.         main.metadata = main.metadata.update_column(1, {'name': 'bbb111', 'extra': 'b_column'})

  477.         main.metadata = main.metadata.update_column(2, {'name': 'ccc111'})

  478. 

  479.         columns2 = container.DataFrame({'a2': [21, 22, 23], 'b2': [24, 25, 26]}, {

  480.             'top_level': 'columns2',

  481.         }, generate_metadata=False)

  482.         columns2.metadata = columns2.metadata.generate(columns2, compact=False)

  483.         columns2.metadata = columns2.metadata.update_column(0, {'name': 'aaa222'})

  484.         columns2.metadata = columns2.metadata.update_column(1, {'name': 'bbb222'})

  485. 

  486.         columns3 = container.DataFrame({'a3': [31, 32, 33], 'b3': [34, 35, 36]}, {

  487.             'top_level': 'columns3',

  488.         }, generate_metadata=False)

  489.         columns3.metadata = columns3.metadata.generate(columns3, compact=False)

  490.         columns3.metadata = columns3.metadata.update_column(0, {'name': 'aaa333'})

  491.         columns3.metadata = columns3.metadata.update_column(1, {'name': 'bbb333'})

  492. 

  493.         result = utils.combine_columns(main, [1, 2], [columns2, columns3], return_result='append', add_index_columns=False)

  494. 

  495.         self.assertEqual(result.values.tolist(), [

  496.             [1, 4, 7, 10, 13, 21, 24, 31, 34],

  497.             [2, 5, 8, 11, 14, 22, 25, 32, 35],

  498.             [3, 6, 9, 12, 15, 23, 26, 33, 36],

  499.         ])

  500. 

  501.         self.assertEqual(d3m_utils.to_json_structure(result.metadata.to_internal_simple_structure()), [{

  502.             'selector': [],

  503.             'metadata': {

  504.                 'top_level': 'main',

  505.                 'schema': metadata_base.CONTAINER_SCHEMA_VERSION,

  506.                 'structural_type': 'd3m.container.pandas.DataFrame',

  507.                 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/Table'],

  508.                 'dimension': {

  509.                     'name': 'rows',

  510.                     'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularRow'],

  511.                     'length': 3,

  512.                 },

  513.             },

  514.         }, {

  515.             'selector': ['__ALL_ELEMENTS__'],

  516.             'metadata': {

  517.                 'dimension': {

  518.                     'name': 'columns',

  519.                     'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularColumn'],

  520.                     'length': 9,

  521.                 },

  522.             },

  523.         }, {

  524.             'selector': ['__ALL_ELEMENTS__', 0],

  525.             'metadata': {

  526.                 'name': 'aaa111',

  527.                 'structural_type': 'numpy.int64',

  528.             },

  529.         }, {

  530.             'selector': ['__ALL_ELEMENTS__', 1],

  531.             'metadata': {

  532.                 'name': 'bbb111',

  533.                 'extra': 'b_column',

  534.                 'structural_type': 'numpy.int64',

  535.             },

  536.         }, {

  537.             'selector': ['__ALL_ELEMENTS__', 2],

  538.             'metadata': {

  539.                 'name': 'ccc111',

  540.                 'structural_type': 'numpy.int64',

  541.             },

  542.         }, {

  543.             'selector': ['__ALL_ELEMENTS__', 3],

  544.             'metadata': {

  545.                 'name': 'd1',

  546.                 'structural_type': 'numpy.int64',

  547.             },

  548.         }, {

  549.             'selector': ['__ALL_ELEMENTS__', 4],

  550.             'metadata': {

  551.                 'name': 'e1',

  552.                 'structural_type': 'numpy.int64',

  553.             },

  554.         }, {

  555.             'selector': ['__ALL_ELEMENTS__', 5],

  556.             'metadata': {

  557.                 'name': 'aaa222',

  558.                 'structural_type': 'numpy.int64',

  559.             },

  560.         }, {

  561.             'selector': ['__ALL_ELEMENTS__', 6],

  562.             'metadata': {

  563.                 'name': 'bbb222',

  564.                 'structural_type': 'numpy.int64',

  565.             },

  566.         }, {

  567.             'selector': ['__ALL_ELEMENTS__', 7],

  568.             'metadata': {

  569.                 'name': 'aaa333',

  570.                 'structural_type': 'numpy.int64',

  571.             },

  572.         }, {

  573.             'selector': ['__ALL_ELEMENTS__', 8],

  574.             'metadata': {

  575.                 'name': 'bbb333',

  576.                 'structural_type': 'numpy.int64',

  577.             },

  578.         }])

  579. 

  580.         result = utils.combine_columns(main, [1, 2], [columns2, columns3], return_result='new', add_index_columns=False)

  581. 

  582.         self.assertEqual(result.values.tolist(), [

  583.             [21, 24, 31, 34],

  584.             [22, 25, 32, 35],

  585.             [23, 26, 33, 36],

  586.         ])

  587. 

  588.         self.assertEqual(d3m_utils.to_json_structure(result.metadata.to_internal_simple_structure()), [{

  589.             'selector': [],

  590.             'metadata': {

  591.                 'top_level': 'columns2',

  592.                 'schema': metadata_base.CONTAINER_SCHEMA_VERSION,

  593.                 'structural_type': 'd3m.container.pandas.DataFrame',

  594.                 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/Table'],

  595.                 'dimension': {

  596.                     'name': 'rows',

  597.                     'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularRow'],

  598.                     'length': 3,

  599.                 },

  600.             },

  601.         }, {

  602.             'selector': ['__ALL_ELEMENTS__'],

  603.             'metadata': {

  604.                 'dimension': {

  605.                     'name': 'columns',

  606.                     'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularColumn'],

  607.                     'length': 4,

  608.                 },

  609.             },

  610.         }, {

  611.             'selector': ['__ALL_ELEMENTS__', 0],

  612.             'metadata': {

  613.                 'name': 'aaa222',

  614.                 'structural_type': 'numpy.int64',

  615.             },

  616.         }, {

  617.             'selector': ['__ALL_ELEMENTS__', 1],

  618.             'metadata': {

  619.                 'name': 'bbb222',

  620.                 'structural_type': 'numpy.int64',

  621.             },

  622.         }, {

  623.             'selector': ['__ALL_ELEMENTS__', 2],

  624.             'metadata': {

  625.                 'name': 'aaa333',

  626.                 'structural_type': 'numpy.int64',

  627.             },

  628.         }, {

  629.             'selector': ['__ALL_ELEMENTS__', 3],

  630.             'metadata': {

  631.                 'name': 'bbb333',

  632.                 'structural_type': 'numpy.int64',

  633.             },

  634.         }])

  635. 

  636.         result = utils.combine_columns(main, [1, 2], [columns2, columns3], return_result='replace', add_index_columns=False)

  637. 

  638.         self.assertEqual(result.values.tolist(), [

  639.             [1, 21, 24, 31, 34, 10, 13],

  640.             [2, 22, 25, 32, 35, 11, 14],

  641.             [3, 23, 26, 33, 36, 12, 15],

  642.         ])

  643. 

  644.         self.assertEqual(d3m_utils.to_json_structure(result.metadata.to_internal_simple_structure()), [{

  645.             'selector': [],

  646.             'metadata': {

  647.                 'top_level': 'main',

  648.                 'schema': metadata_base.CONTAINER_SCHEMA_VERSION,

  649.                 'structural_type': 'd3m.container.pandas.DataFrame',

  650.                 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/Table'],

  651.                 'dimension': {

  652.                     'name': 'rows',

  653.                     'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularRow'],

  654.                     'length': 3,

  655.                 },

  656.             },

  657.         }, {

  658.             'selector': ['__ALL_ELEMENTS__'],

  659.             'metadata': {

  660.                 'dimension': {

  661.                     'name': 'columns',

  662.                     'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularColumn'],

  663.                     'length': 7,

  664.                 },

  665.             },

  666.         }, {

  667.             'selector': ['__ALL_ELEMENTS__', 0],

  668.             'metadata': {

  669.                 'name': 'aaa111',

  670.                 'structural_type': 'numpy.int64',

  671.             },

  672.         }, {

  673.             'selector': ['__ALL_ELEMENTS__', 1],

  674.             'metadata': {

  675.                 'name': 'aaa222',

  676.                 'structural_type': 'numpy.int64',

  677.             },

  678.         }, {

  679.             'selector': ['__ALL_ELEMENTS__', 2],

  680.             'metadata': {

  681.                 'name': 'bbb222',

  682.                 'structural_type': 'numpy.int64',

  683.             },

  684.         }, {

  685.             'selector': ['__ALL_ELEMENTS__', 3],

  686.             'metadata': {

  687.                 'name': 'aaa333',

  688.                 'structural_type': 'numpy.int64',

  689.             },

  690.         }, {

  691.             'selector': ['__ALL_ELEMENTS__', 4],

  692.             'metadata': {

  693.                 'name': 'bbb333',

  694.                 'structural_type': 'numpy.int64',

  695.             },

  696.         }, {

  697.             'selector': ['__ALL_ELEMENTS__', 5],

  698.             'metadata': {

  699.                 'name': 'd1',

  700.                 'structural_type': 'numpy.int64',

  701.             },

  702.         }, {

  703.             'selector': ['__ALL_ELEMENTS__', 6],

  704.             'metadata': {

  705.                 'name': 'e1',

  706.                 'structural_type': 'numpy.int64',

  707.             },

  708.         }])

  709. 

  710.         result = utils.combine_columns(main, [0, 1, 2, 3, 4], [columns2, columns3], return_result='replace', add_index_columns=False)

  711. 

  712.         self.assertEqual(result.values.tolist(), [

  713.             [21, 24, 31, 34],

  714.             [22, 25, 32, 35],

  715.             [23, 26, 33, 36],

  716.         ])

  717. 

  718.         self.assertEqual(d3m_utils.to_json_structure(result.metadata.to_internal_simple_structure()), [{

  719.             'selector': [],

  720.             'metadata': {

  721.                 'top_level': 'main',

  722.                 'schema': metadata_base.CONTAINER_SCHEMA_VERSION,

  723.                 'structural_type': 'd3m.container.pandas.DataFrame',

  724.                 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/Table'],

  725.                 'dimension': {

  726.                     'name': 'rows',

  727.                     'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularRow'],

  728.                     'length': 3,

  729.                 },

  730.             },

  731.         }, {

  732.             'selector': ['__ALL_ELEMENTS__'],

  733.             'metadata': {

  734.                 'dimension': {

  735.                     'name': 'columns',

  736.                     'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularColumn'],

  737.                     'length': 4,

  738.                 },

  739.             },

  740.         }, {

  741.             'selector': ['__ALL_ELEMENTS__', 0],

  742.             'metadata': {

  743.                 'name': 'aaa222',

  744.                 'structural_type': 'numpy.int64',

  745.             },

  746.         }, {

  747.             'selector': ['__ALL_ELEMENTS__', 1],

  748.             'metadata': {

  749.                 'name': 'bbb222',

  750.                 'structural_type': 'numpy.int64',

  751.             },

  752.         }, {

  753.             'selector': ['__ALL_ELEMENTS__', 2],

  754.             'metadata': {

  755.                 'name': 'aaa333',

  756.                 'structural_type': 'numpy.int64',

  757.             },

  758.         }, {

  759.             'selector': ['__ALL_ELEMENTS__', 3],

  760.             'metadata': {

  761.                 'name': 'bbb333',

  762.                 'structural_type': 'numpy.int64',

  763.             },

  764.         }])

  765. 

  766.         result = utils.combine_columns(main, [4], [columns2, columns3], return_result='replace', add_index_columns=False)

  767. 

  768.         self.assertEqual(result.values.tolist(), [

  769.             [1, 4, 7, 10, 21, 24, 31, 34],

  770.             [2, 5, 8, 11, 22, 25, 32, 35],

  771.             [3, 6, 9, 12, 23, 26, 33, 36],

  772.         ])

  773. 

  774.         self.assertEqual(d3m_utils.to_json_structure(result.metadata.to_internal_simple_structure()), [{

  775.             'selector': [],

  776.             'metadata': {

  777.                 'top_level': 'main',

  778.                 'schema': metadata_base.CONTAINER_SCHEMA_VERSION,

  779.                 'structural_type': 'd3m.container.pandas.DataFrame',

  780.                 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/Table'],

  781.                 'dimension': {

  782.                     'name': 'rows',

  783.                     'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularRow'],

  784.                     'length': 3,

  785.                 },

  786.             },

  787.         }, {

  788.             'selector': ['__ALL_ELEMENTS__'],

  789.             'metadata': {

  790.                 'dimension': {

  791.                     'name': 'columns',

  792.                     'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularColumn'],

  793.                     'length': 8,

  794.                 },

  795.             },

  796.         }, {

  797.             'selector': ['__ALL_ELEMENTS__', 0],

  798.             'metadata': {

  799.                 'name': 'aaa111',

  800.                 'structural_type': 'numpy.int64',

  801.             },

  802.         }, {

  803.             'selector': ['__ALL_ELEMENTS__', 1],

  804.             'metadata': {

  805.                 'name': 'bbb111',

  806.                 'extra': 'b_column',

  807.                 'structural_type': 'numpy.int64',

  808.             },

  809.         }, {

  810.             'selector': ['__ALL_ELEMENTS__', 2],

  811.             'metadata': {

  812.                 'name': 'ccc111',

  813.                 'structural_type': 'numpy.int64',

  814.             },

  815.         }, {

  816.             'selector': ['__ALL_ELEMENTS__', 3],

  817.             'metadata': {

  818.                 'name': 'd1',

  819.                 'structural_type': 'numpy.int64',

  820.             },

  821.         }, {

  822.             'selector': ['__ALL_ELEMENTS__', 4],

  823.             'metadata': {

  824.                 'structural_type': 'numpy.int64',

  825.                 'name': 'aaa222',

  826.             },

  827.         }, {

  828.             'selector': ['__ALL_ELEMENTS__', 5],

  829.             'metadata': {

  830.                 'structural_type': 'numpy.int64',

  831.                 'name': 'bbb222',

  832.             },

  833.         }, {

  834.             'selector': ['__ALL_ELEMENTS__', 6],

  835.             'metadata': {

  836.                 'structural_type': 'numpy.int64',

  837.                 'name': 'aaa333',

  838.             },

  839.         }, {

  840.             'selector': ['__ALL_ELEMENTS__', 7],

  841.             'metadata': {

  842.                 'structural_type': 'numpy.int64',

  843.                 'name': 'bbb333',

  844.             },

  845.         }])

  846. 

  847.         result = utils.combine_columns(main, [0, 2, 4], [columns2, columns3], return_result='replace', add_index_columns=False)

  848. 

  849.         self.assertEqual(result.values.tolist(), [

  850.             [21, 4, 24, 10, 31, 34],

  851.             [22, 5, 25, 11, 32, 35],

  852.             [23, 6, 26, 12, 33, 36],

  853.         ])

  854. 

  855.         self.assertEqual(d3m_utils.to_json_structure(result.metadata.to_internal_simple_structure()), [{

  856.             'selector': [],

  857.             'metadata': {

  858.                 'top_level': 'main',

  859.                 'schema': metadata_base.CONTAINER_SCHEMA_VERSION,

  860.                 'structural_type': 'd3m.container.pandas.DataFrame',

  861.                 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/Table'],

  862.                 'dimension': {

  863.                     'name': 'rows',

  864.                     'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularRow'],

  865.                     'length': 3,

  866.                 },

  867.             },

  868.         }, {

  869.             'selector': ['__ALL_ELEMENTS__'],

  870.             'metadata': {

  871.                 'dimension': {

  872.                     'name': 'columns',

  873.                     'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularColumn'],

  874.                     'length': 6,

  875.                 },

  876.             },

  877.         }, {

  878.             'selector': ['__ALL_ELEMENTS__', 0],

  879.             'metadata': {

  880.                 'name': 'aaa222',

  881.                 'structural_type': 'numpy.int64',

  882.             },

  883.         }, {

  884.             'selector': ['__ALL_ELEMENTS__', 1],

  885.             'metadata': {

  886.                 'name': 'bbb111',

  887.                 'extra': 'b_column',

  888.                 'structural_type': 'numpy.int64',

  889.             },

  890.         }, {

  891.             'selector': ['__ALL_ELEMENTS__', 2],

  892.             'metadata': {

  893.                 'name': 'bbb222',

  894.                 'structural_type': 'numpy.int64',

  895.             },

  896.         }, {

  897.             'selector': ['__ALL_ELEMENTS__', 3],

  898.             'metadata': {

  899.                 'name': 'd1',

  900.                 'structural_type': 'numpy.int64',

  901.             },

  902.         }, {

  903.             'selector': ['__ALL_ELEMENTS__', 4],

  904.             'metadata': {

  905.                 'name': 'aaa333',

  906.                 'structural_type': 'numpy.int64',

  907.             },

  908.         }, {

  909.             'selector': ['__ALL_ELEMENTS__', 5],

  910.             'metadata': {

  911.                 'name': 'bbb333',

  912.                 'structural_type': 'numpy.int64',

  913.             },

  914.         }])

  915. 

  916.     def test_combine_columns_new_with_index_compact_metadata(self):

  917.         main = container.DataFrame({'d3mIndex': [1, 2, 3], 'b1': [4, 5, 6], 'c1': [7, 8, 9]}, columns=['d3mIndex', 'b1', 'c1'], generate_metadata=False)

  918.         main.metadata = main.metadata.generate(main, compact=True)

  919.         main.metadata = main.metadata.update_column(0, {'name': 'd3mIndex', 'semantic_types': ['http://schema.org/Integer', 'https://metadata.datadrivendiscovery.org/types/PrimaryKey']})

  920.         main.metadata = main.metadata.update_column(1, {'name': 'b1', 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/Attribute']})

  921.         main.metadata = main.metadata.update_column(2, {'name': 'c1', 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/Attribute']})

  922. 

  923.         columns = container.DataFrame({'d3mIndex': [1, 2, 3], 'b2': [4, 5, 6]}, columns=['d3mIndex', 'b2'], generate_metadata=False)

  924.         columns.metadata = columns.metadata.generate(columns, compact=True)

  925.         columns.metadata = columns.metadata.update_column(0, {'name': 'd3mIndex', 'semantic_types': ['http://schema.org/Integer', 'https://metadata.datadrivendiscovery.org/types/PrimaryKey']})

  926.         columns.metadata = columns.metadata.update_column(1, {'name': 'b2', 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/Attribute']})

  927. 

  928.         result = utils.combine_columns(main, [], [columns], return_result='new', add_index_columns=True)

  929. 

  930.         self.assertEqual(result.values.tolist(), [

  931.             [1, 4],

  932.             [2, 5],

  933.             [3, 6],

  934.         ])

  935. 

  936.         self.assertEqual(d3m_utils.to_json_structure(result.metadata.to_internal_simple_structure()), [{

  937.             'selector': [],

  938.             'metadata': {

  939.                 'schema': metadata_base.CONTAINER_SCHEMA_VERSION,

  940.                 'structural_type': 'd3m.container.pandas.DataFrame',

  941.                 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/Table'],

  942.                 'dimension': {

  943.                     'name': 'rows',

  944.                     'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularRow'],

  945.                     'length': 3,

  946.                 },

  947.             },

  948.         }, {

  949.             'selector': ['__ALL_ELEMENTS__'],

  950.             'metadata': {

  951.                 'dimension': {

  952.                     'name': 'columns',

  953.                     'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularColumn'],

  954.                     'length': 2,

  955.                 },

  956.             },

  957.         }, {

  958.             'selector': ['__ALL_ELEMENTS__', '__ALL_ELEMENTS__'],

  959.             'metadata': {

  960.                 'structural_type': 'numpy.int64',

  961.             },

  962.         }, {

  963.             'selector': ['__ALL_ELEMENTS__', 0],

  964.             'metadata': {

  965.                 'name': 'd3mIndex',

  966.                 'semantic_types': ['http://schema.org/Integer', 'https://metadata.datadrivendiscovery.org/types/PrimaryKey'],

  967.             },

  968.         }, {

  969.             'selector': ['__ALL_ELEMENTS__', 1],

  970.             'metadata': {

  971.                 'name': 'b2',

  972.                 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/Attribute'],

  973.             },

  974.         }])

  975. 

  976.     def test_combine_columns_new_with_index_noncompact_metadata(self):

  977.         main = container.DataFrame({'d3mIndex': [1, 2, 3], 'b1': [4, 5, 6], 'c1': [7, 8, 9]}, columns=['d3mIndex', 'b1', 'c1'], generate_metadata=False)

  978.         main.metadata = main.metadata.generate(main, compact=False)

  979.         main.metadata = main.metadata.update_column(0, {'name': 'd3mIndex', 'semantic_types': ['http://schema.org/Integer', 'https://metadata.datadrivendiscovery.org/types/PrimaryKey']})

  980.         main.metadata = main.metadata.update_column(1, {'name': 'b1', 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/Attribute']})

  981.         main.metadata = main.metadata.update_column(2, {'name': 'c1', 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/Attribute']})

  982. 

  983.         columns = container.DataFrame({'d3mIndex': [1, 2, 3], 'b2': [4, 5, 6]}, columns=['d3mIndex', 'b2'], generate_metadata=False)

  984.         columns.metadata = columns.metadata.generate(columns, compact=False)

  985.         columns.metadata = columns.metadata.update_column(0, {'name': 'd3mIndex', 'semantic_types': ['http://schema.org/Integer', 'https://metadata.datadrivendiscovery.org/types/PrimaryKey']})

  986.         columns.metadata = columns.metadata.update_column(1, {'name': 'b2', 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/Attribute']})

  987. 

  988.         result = utils.combine_columns(main, [], [columns], return_result='new', add_index_columns=True)

  989. 

  990.         self.assertEqual(result.values.tolist(), [

  991.             [1, 4],

  992.             [2, 5],

  993.             [3, 6],

  994.         ])

  995. 

  996.         self.assertEqual(d3m_utils.to_json_structure(result.metadata.to_internal_simple_structure()), [{

  997.             'selector': [],

  998.             'metadata': {

  999.                 'schema': metadata_base.CONTAINER_SCHEMA_VERSION,

  1000.                 'structural_type': 'd3m.container.pandas.DataFrame',

  1001.                 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/Table'],

  1002.                 'dimension': {

  1003.                     'name': 'rows',

  1004.                     'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularRow'],

  1005.                     'length': 3,

  1006.                 },

  1007.             },

  1008.         }, {

  1009.             'selector': ['__ALL_ELEMENTS__'],

  1010.             'metadata': {

  1011.                 'dimension': {

  1012.                     'name': 'columns',

  1013.                     'semantic_types': ['https://metadata.datadrivendiscovery.org/types/TabularColumn'],

  1014.                     'length': 2,

  1015.                 },

  1016.             },

  1017.         }, {

  1018.             'selector': ['__ALL_ELEMENTS__', 0],

  1019.             'metadata': {

  1020.                 'name': 'd3mIndex',

  1021.                 'semantic_types': ['http://schema.org/Integer', 'https://metadata.datadrivendiscovery.org/types/PrimaryKey'],

  1022.                 'structural_type': 'numpy.int64',

  1023.             },

  1024.         }, {

  1025.             'selector': ['__ALL_ELEMENTS__', 1],

  1026.             'metadata': {

  1027.                 'name': 'b2',

  1028.                 'semantic_types': ['https://metadata.datadrivendiscovery.org/types/Attribute'],

  1029.                 'structural_type': 'numpy.int64',

  1030.             },

  1031.         }])

  1032. 

  1033. 

  1034. if __name__ == '__main__':

  1035.     unittest.main()