tods/d3m/tests/test_hyperparams.py

import json
import logging
import os
import typing
import pickle
import subprocess
import sys
import unittest
from collections import OrderedDict

import frozendict
import numpy
from sklearn.utils import validation as sklearn_validation

from d3m import container, exceptions, index, utils
from d3m.metadata import base as metadata_base, hyperparams
from d3m.primitive_interfaces import base, transformer

TEST_PRIMITIVES_DIR = os.path.join(os.path.dirname(__file__), 'data', 'primitives')

sys.path.insert(0, TEST_PRIMITIVES_DIR)

from test_primitives.monomial import MonomialPrimitive
from test_primitives.random import RandomPrimitive
from test_primitives.sum import SumPrimitive
from test_primitives.increment import IncrementPrimitive


# It's defined at global scope so it can be pickled.
class TestPicklingHyperparams(hyperparams.Hyperparams):
    choice = hyperparams.Choice(
        choices={
            'alpha': hyperparams.Hyperparams.define(OrderedDict(
                value=hyperparams.Union(
                    OrderedDict(
                        float=hyperparams.Hyperparameter[float](0),
                        int=hyperparams.Hyperparameter[int](0)
                    ),
                    default='float'
                ),
            ))
        },
        default='alpha',
        semantic_types=['https://metadata.datadrivendiscovery.org/types/TuningParameter']
    )


class TestHyperparams(unittest.TestCase):
    def test_hyperparameter(self):
        hyperparameter = hyperparams.Hyperparameter[str]('nothing')

        self.assertEqual(hyperparameter.get_default(), 'nothing')
        with self.assertLogs(hyperparams.logger) as cm:
            self.assertEqual(hyperparameter.sample(42), 'nothing')
        self.assertEqual(len(cm.records), 1)
        with self.assertLogs(hyperparams.logger) as cm:
            self.assertEqual(hyperparameter.sample_multiple(0, 1, 42), ('nothing',))
        self.assertEqual(len(cm.records), 1)

        with self.assertLogs(hyperparams.logger) as cm:
            self.assertEqual(hyperparameter.sample_multiple(0, 0, 42), ())
        self.assertEqual(len(cm.records), 1)

        self.assertEqual(hyperparameter.to_simple_structure(), {
            'default': 'nothing',
            'semantic_types': [],
            'structural_type': str,
            'type': hyperparams.Hyperparameter,
        })

        self.assertEqual(hyperparameter.value_to_json_structure(hyperparameter.get_default()), 'nothing')
        with self.assertLogs(hyperparams.logger) as cm:
            self.assertEqual(hyperparameter.value_to_json_structure(hyperparameter.sample(42)), 'nothing')
        self.assertEqual(len(cm.records), 1)

        self.assertEqual(hyperparameter.value_from_json_structure(hyperparameter.value_to_json_structure(hyperparameter.get_default())), hyperparameter.get_default())
        with self.assertLogs(hyperparams.logger) as cm:
            self.assertEqual(hyperparameter.value_from_json_structure(hyperparameter.value_to_json_structure(hyperparameter.sample(42))), hyperparameter.sample(42))
        self.assertEqual(len(cm.records), 1)

        with self.assertRaisesRegex(TypeError, 'Value \'.*\' is not an instance of the structural type'):
            hyperparams.Hyperparameter[int]('nothing')

        with self.assertRaisesRegex(ValueError, '\'max_samples\' cannot be larger than'):
            hyperparameter.sample_multiple(0, 2, 42)

    def test_constant(self):
        hyperparameter = hyperparams.Constant(12345)

        self.assertEqual(hyperparameter.get_default(), 12345)
        self.assertEqual(hyperparameter.sample(), 12345)
        self.assertEqual(hyperparameter.sample_multiple(0, 1, 42), (12345,))

        self.assertEqual(hyperparameter.sample_multiple(0, 0, 42), ())

        self.assertEqual(hyperparameter.to_simple_structure(), {
            'default': 12345,
            'semantic_types': [],
            'structural_type': int,
            'type': hyperparams.Constant,
        })

        self.assertEqual(hyperparameter.value_to_json_structure(hyperparameter.get_default()), 12345)
        self.assertEqual(hyperparameter.value_to_json_structure(hyperparameter.sample(42)), 12345)
        self.assertEqual(hyperparameter.value_from_json_structure(hyperparameter.value_to_json_structure(hyperparameter.get_default())), hyperparameter.get_default())
        self.assertEqual(hyperparameter.value_from_json_structure(hyperparameter.value_to_json_structure(hyperparameter.sample(42))), hyperparameter.sample(42))

        with self.assertRaisesRegex(TypeError, 'Value \'.*\' is not an instance of the structural type'):
            hyperparams.Hyperparameter[int]('different')

        with self.assertRaisesRegex(ValueError, 'Value \'.*\' is not the constant default value'):
            hyperparameter.validate(54321)

        with self.assertRaisesRegex(ValueError, '\'max_samples\' cannot be larger than'):
            self.assertEqual(hyperparameter.sample_multiple(0, 2, 42), {12345})

        hyperparameter = hyperparams.Constant('constant')

        with self.assertRaisesRegex(ValueError, 'Value \'.*\' is not the constant default value'):
            hyperparameter.validate('different')

    def test_bounded(self):
        hyperparameter = hyperparams.Bounded[float](0.0, 1.0, 0.2)

        self.assertEqual(hyperparameter.get_default(), 0.2)
        with self.assertLogs(hyperparams.logger) as cm:
            self.assertEqual(hyperparameter.sample(42), 0.37454011884736255)
        self.assertEqual(len(cm.records), 1)
        with self.assertLogs(hyperparams.logger) as cm:
            self.assertEqual(hyperparameter.sample_multiple(0, 1, 7), (0.22733907982646523,))
        self.assertEqual(len(cm.records), 1)

        self.assertEqual(hyperparameter.sample_multiple(0, 0, 42), ())

        self.assertEqual(hyperparameter.to_simple_structure(), {
            'default': 0.2,
            'semantic_types': [],
            'structural_type': float,
            'type': hyperparams.Bounded,
            'lower': 0.0,
            'upper': 1.0,
            'lower_inclusive': True,
            'upper_inclusive': True,
        })

        self.assertEqual(hyperparameter.value_to_json_structure(hyperparameter.get_default()), 0.2)
        with self.assertLogs(hyperparams.logger) as cm:
            self.assertEqual(hyperparameter.value_to_json_structure(hyperparameter.sample(42)), 0.37454011884736255)
        self.assertEqual(len(cm.records), 1)

        self.assertEqual(hyperparameter.value_from_json_structure(hyperparameter.value_to_json_structure(hyperparameter.get_default())), hyperparameter.get_default())
        with self.assertLogs(hyperparams.logger) as cm:
            self.assertEqual(hyperparameter.value_from_json_structure(hyperparameter.value_to_json_structure(hyperparameter.sample(42))), hyperparameter.sample(42))
        self.assertEqual(len(cm.records), 1)

        with self.assertRaisesRegex(TypeError, 'Value \'.*\' is not an instance of the structural type'):
            hyperparams.Bounded[str]('lower', 'upper', 0.2)

        with self.assertRaisesRegex(TypeError, 'Lower bound \'.*\' is not an instance of the structural type'):
            hyperparams.Bounded[str](0.0, 'upper', 'default')

        with self.assertRaisesRegex(TypeError, 'Upper bound \'.*\' is not an instance of the structural type'):
            hyperparams.Bounded[str]('lower', 1.0, 'default')

        with self.assertRaisesRegex(ValueError, 'Value \'.*\' is outside of range'):
            hyperparams.Bounded[str]('lower', 'upper', 'default')

        with self.assertRaisesRegex(ValueError, 'Value \'.*\' is outside of range'):
            hyperparams.Bounded[float](0.0, 1.0, 1.2)

        hyperparams.Bounded[typing.Optional[float]](0.0, None, 0.2)
        hyperparams.Bounded[typing.Optional[float]](None, 1.0, 0.2)

        with self.assertRaisesRegex(ValueError, 'Lower and upper bounds cannot both be None'):
            hyperparams.Bounded[typing.Optional[float]](None, None, 0.2)

        with self.assertRaisesRegex(TypeError, 'Value \'.*\' is not an instance of the structural type'):
            hyperparams.Bounded[float](0.0, 1.0, None)

        with self.assertRaises(TypeError):
            hyperparams.Bounded[typing.Optional[float]](0.0, 1.0, None)

        hyperparams.Bounded[typing.Optional[float]](None, 1.0, None)
        hyperparams.Bounded[typing.Optional[float]](0.0, None, None)

        hyperparameter = hyperparams.Bounded[float](0.0, None, 0.2)

        with self.assertRaisesRegex(ValueError, '\'max_samples\' cannot be larger than'):
            hyperparameter.sample_multiple(0, 2, 42)

        with self.assertRaisesRegex(exceptions.InvalidArgumentValueError, 'must be finite'):
            hyperparams.Bounded[typing.Optional[float]](0.0, numpy.nan, 0)

        with self.assertRaisesRegex(exceptions.InvalidArgumentValueError, 'must be finite'):
            hyperparams.Bounded[typing.Optional[float]](numpy.inf, 0.0, 0)

    def test_enumeration(self):
        hyperparameter = hyperparams.Enumeration(['a', 'b', 1, 2, None], None)

        self.assertEqual(hyperparameter.get_default(), None)
        self.assertEqual(hyperparameter.sample(42), 2)
        self.assertEqual(hyperparameter.sample_multiple(0, 1, 42), ())
        self.assertEqual(hyperparameter.sample_multiple(0, 2, 42), ('b', None))
        self.assertEqual(hyperparameter.sample_multiple(0, 3, 42), ('b', None))

        self.assertEqual(hyperparameter.to_simple_structure(), {
            'default': None,
            'semantic_types': [],
            'structural_type': typing.Union[str, int, type(None)],
            'type': hyperparams.Enumeration,
            'values': ['a', 'b', 1, 2, None],
        })

        self.assertEqual(hyperparameter.value_to_json_structure(hyperparameter.get_default()), None)
        self.assertEqual(hyperparameter.value_to_json_structure(hyperparameter.sample(42)), 2)

        self.assertEqual(hyperparameter.value_from_json_structure(hyperparameter.value_to_json_structure(hyperparameter.get_default())), hyperparameter.get_default())
        self.assertEqual(hyperparameter.value_from_json_structure(hyperparameter.value_to_json_structure(hyperparameter.sample(42))), hyperparameter.sample(42))

        with self.assertRaisesRegex(ValueError, 'Value \'.*\' is not among values'):
            hyperparams.Enumeration(['a', 'b', 1, 2], None)

        with self.assertRaisesRegex(TypeError, 'Value \'.*\' is not an instance of the structural type'):
            hyperparams.Enumeration[typing.Union[str, int]](['a', 'b', 1, 2, None], None)

        with self.assertRaisesRegex(ValueError, '\'max_samples\' cannot be larger than'):
            self.assertEqual(hyperparameter.sample_multiple(0, 6, 42), ())

        hyperparameter = hyperparams.Enumeration(['a', 'b', 'c'], 'a')

        self.assertEqual(hyperparameter.value_to_json_structure('c'), 'c')
        self.assertEqual(hyperparameter.value_from_json_structure(hyperparameter.value_to_json_structure('c')), 'c')

        with self.assertRaisesRegex(exceptions.InvalidArgumentValueError, 'contain duplicates'):
            hyperparams.Enumeration([1.0, 1], 1)

        hyperparameter = hyperparams.Enumeration([1.0, float('nan'), float('infinity'), float('-infinity')], 1.0)

        hyperparameter.validate(float('nan'))

        self.assertEqual(utils.to_json_structure(hyperparameter.to_simple_structure()), {
            'type': 'd3m.metadata.hyperparams.Enumeration',
            'default': 1.0,
            'structural_type': 'float',
            'semantic_types': [],
            'values': [1.0, 'nan', 'inf', '-inf'],
        })

        self.assertEqual(json.dumps(hyperparameter.value_to_json_structure(float('nan')), allow_nan=False), '{"encoding": "pickle", "value": "gANHf/gAAAAAAAAu"}')
        self.assertEqual(json.dumps(hyperparameter.value_to_json_structure(float('inf')), allow_nan=False), '{"encoding": "pickle", "value": "gANHf/AAAAAAAAAu"}')

    def test_other(self):
        hyperparameter = hyperparams.UniformInt(1, 10, 2)

        self.assertEqual(hyperparameter.get_default(), 2)
        self.assertEqual(hyperparameter.sample(42), 7)
        self.assertEqual(hyperparameter.sample_multiple(0, 1, 42), ())
        self.assertEqual(hyperparameter.sample_multiple(0, 2, 42), (4, 8))

        self.assertEqual(hyperparameter.to_simple_structure(), {
            'default': 2,
            'semantic_types': [],
            'structural_type': int,
            'type': hyperparams.UniformInt,
            'lower': 1,
            'upper': 10,
            'lower_inclusive': True,
            'upper_inclusive': False,
        })

        with self.assertRaisesRegex(ValueError, 'Value \'.*\' is outside of range'):
            hyperparams.UniformInt(1, 10, 0)

        with self.assertRaisesRegex(ValueError, '\'max_samples\' cannot be larger than'):
            self.assertEqual(hyperparameter.sample_multiple(0, 10, 42), ())

        hyperparameter = hyperparams.Uniform(1.0, 10.0, 2.0)

        self.assertEqual(hyperparameter.get_default(), 2.0)
        self.assertEqual(hyperparameter.sample(42), 4.370861069626263)

        self.assertEqual(hyperparameter.to_simple_structure(), {
            'default': 2.0,
            'semantic_types': [],
            'structural_type': float,
            'type': hyperparams.Uniform,
            'lower': 1.0,
            'upper': 10.0,
            'lower_inclusive': True,
            'upper_inclusive': False,
        })

        with self.assertRaisesRegex(ValueError, 'Value \'.*\' is outside of range'):
            hyperparams.Uniform(1.0, 10.0, 0.0)

        hyperparameter = hyperparams.LogUniform(1.0, 10.0, 2.0)

        self.assertEqual(hyperparameter.get_default(), 2.0)
        self.assertEqual(hyperparameter.sample(42), 2.368863950364078)

        self.assertEqual(hyperparameter.to_simple_structure(), {
            'default': 2.0,
            'semantic_types': [],
            'structural_type': float,
            'type': hyperparams.LogUniform,
            'lower': 1.0,
            'upper': 10.0,
            'lower_inclusive': True,
            'upper_inclusive': False,
        })

        with self.assertRaisesRegex(ValueError, 'Value \'.*\' is outside of range'):
            hyperparams.LogUniform(1.0, 10.0, 0.0)

        hyperparameter = hyperparams.UniformBool(True)

        self.assertEqual(hyperparameter.get_default(), True)
        self.assertEqual(hyperparameter.sample(42), True)

        self.assertEqual(hyperparameter.to_simple_structure(), {
            'default': True,
            'semantic_types': [],
            'structural_type': bool,
            'type': hyperparams.UniformBool,
        })

        with self.assertRaises(exceptions.InvalidArgumentValueError):
            hyperparams.UniformInt(0, 1, 1, lower_inclusive=False, upper_inclusive=False)

        hyperparameter = hyperparams.UniformInt(0, 2, 1, lower_inclusive=False, upper_inclusive=False)

        self.assertEqual(hyperparameter.sample(42), 1)

        with self.assertRaises(exceptions.InvalidArgumentValueError):
            hyperparameter.sample_multiple(2, 2, 42)

        self.assertEqual(hyperparameter.sample_multiple(2, 2, 42, with_replacement=True), (1, 1))

    def test_union(self):
        hyperparameter = hyperparams.Union(
            OrderedDict(
                none=hyperparams.Hyperparameter(None),
                range=hyperparams.UniformInt(1, 10, 2)
            ),
            'none',
        )

        self.assertEqual(hyperparameter.get_default(), None)
        self.assertEqual(hyperparameter.sample(45), 4)

        self.assertEqual(hyperparameter.to_simple_structure(), {
            'default': None,
            'semantic_types': [],
            'structural_type': typing.Optional[int],
            'type': hyperparams.Union,
            'configuration': {
                'none': {
                    'default': None,
                    'semantic_types': [],
                    'structural_type': type(None),
                    'type': hyperparams.Hyperparameter,
                },
                'range': {
                    'default': 2,
                    'semantic_types': [],
                    'structural_type': int,
                    'type': hyperparams.UniformInt,
                    'lower': 1,
                    'upper': 10,
                    'lower_inclusive': True,
                    'upper_inclusive': False,
                }
            }
        })

        self.assertEqual(hyperparameter.value_to_json_structure(hyperparameter.get_default()), {'case': 'none', 'value': None})
        self.assertEqual(hyperparameter.value_to_json_structure(hyperparameter.sample(45)), {'case': 'range', 'value': 4})

        self.assertEqual(hyperparameter.value_from_json_structure(hyperparameter.value_to_json_structure(hyperparameter.get_default())), hyperparameter.get_default())
        with self.assertLogs(hyperparams.logger) as cm:
            self.assertEqual(hyperparameter.value_from_json_structure(hyperparameter.value_to_json_structure(hyperparameter.sample(42))), hyperparameter.sample(42))
        self.assertEqual(len(cm.records), 1)

        with self.assertRaisesRegex(TypeError, 'Hyper-parameter name is not a string'):
            hyperparams.Union(OrderedDict({1: hyperparams.Hyperparameter(None)}), 1)

        with self.assertRaisesRegex(TypeError, 'Hyper-parameter description is not an instance of the Hyperparameter class'):
            hyperparams.Union(OrderedDict(none=None), 'none')

        with self.assertRaisesRegex(ValueError, 'Default value \'.*\' is not in configuration'):
            hyperparams.Union(OrderedDict(range=hyperparams.UniformInt(1, 10, 2)), 'none')

        hyperparams.Union(OrderedDict(range=hyperparams.UniformInt(1, 10, 2), default=hyperparams.Hyperparameter('nothing')), 'default')
        hyperparams.Union[typing.Union[str, int]](OrderedDict(range=hyperparams.UniformInt(1, 10, 2), default=hyperparams.Hyperparameter('nothing')), 'default')

        with self.assertRaisesRegex(TypeError, 'Hyper-parameter \'.*\' is not a subclass of the structural type'):
            hyperparams.Union[str](OrderedDict(range=hyperparams.UniformInt(1, 10, 2), default=hyperparams.Hyperparameter('nothing')), 'default')

    def test_hyperparams(self):
        class TestHyperparams(hyperparams.Hyperparams):
            a = hyperparams.Union(OrderedDict(
                range=hyperparams.UniformInt(1, 10, 2),
                none=hyperparams.Hyperparameter(None),
            ), 'range')
            b = hyperparams.Uniform(1.0, 10.0, 2.0)

        testCls = hyperparams.Hyperparams.define(OrderedDict(
            a=hyperparams.Union(OrderedDict(
                range=hyperparams.UniformInt(1, 10, 2),
                none=hyperparams.Hyperparameter(None),
            ), 'range'),
            b=hyperparams.Uniform(1.0, 10.0, 2.0),
        ), set_names=True)

        for cls in (TestHyperparams, testCls):
            self.assertEqual(cls.configuration['a'].name, 'a', cls)

            self.assertEqual(cls.defaults(), {'a': 2, 'b': 2.0}, cls)
            self.assertEqual(cls.defaults(), cls({'a': 2, 'b': 2.0}), cls)
            self.assertEqual(cls.sample(42), {'a': 4, 'b': 9.556428757689245}, cls)
            self.assertEqual(cls.sample(42), cls({'a': 4, 'b': 9.556428757689245}), cls)
            self.assertEqual(cls(cls.defaults(), b=3.0), {'a': 2, 'b': 3.0}, cls)
            self.assertEqual(cls(cls.defaults(), **{'b': 4.0}), {'a': 2, 'b': 4.0}, cls)
            self.assertEqual(cls.defaults('a'), 2, cls)
            self.assertEqual(cls.defaults('b'), 2.0, cls)

            self.assertEqual(cls.to_simple_structure(), {
                'a': {
                    'default': 2,
                    'semantic_types': [],
                    'structural_type': typing.Optional[int],
                    'type': hyperparams.Union,
                    'configuration': {
                        'none': {
                            'default': None,
                            'semantic_types': [],
                            'structural_type': type(None),
                            'type': hyperparams.Hyperparameter,
                        },
                        'range': {
                            'default': 2,
                            'lower': 1,
                            'semantic_types': [],
                            'structural_type': int,
                            'type': hyperparams.UniformInt,
                            'upper': 10,
                            'lower_inclusive': True,
                            'upper_inclusive': False,
                        },
                    },
                },
                'b': {
                    'default': 2.0,
                    'semantic_types': [],
                    'structural_type': float,
                    'type': hyperparams.Uniform,
                    'lower': 1.0,
                    'upper': 10.0,
                    'lower_inclusive': True,
                    'upper_inclusive': False,
                }
            }, cls)

            test_hyperparams = cls({'a': cls.configuration['a'].get_default(), 'b': cls.configuration['b'].get_default()})

            self.assertEqual(test_hyperparams['a'], 2, cls)
            self.assertEqual(test_hyperparams['b'], 2.0, cls)

            self.assertEqual(test_hyperparams.values_to_json_structure(), {'a': {'case': 'range', 'value': 2}, 'b': 2.0})
            self.assertEqual(cls.values_from_json_structure(test_hyperparams.values_to_json_structure()), test_hyperparams)

            with self.assertRaisesRegex(ValueError, 'Not all hyper-parameters are specified', msg=cls):
                cls({'a': cls.configuration['a'].get_default()})

            with self.assertRaisesRegex(ValueError, 'Additional hyper-parameters are specified', msg=cls):
                cls({'a': cls.configuration['a'].get_default(), 'b': cls.configuration['b'].get_default(), 'c': 'two'})

            cls({'a': 3, 'b': 3.0})
            cls({'a': None, 'b': 3.0})

            test_hyperparams = cls(a=None, b=3.0)
            self.assertEqual(test_hyperparams['a'], None, cls)
            self.assertEqual(test_hyperparams['b'], 3.0, cls)

            with self.assertRaisesRegex(ValueError, 'Value \'.*\' for hyper-parameter \'.*\' has not validated with any of configured hyper-parameters', msg=cls):
                cls({'a': 0, 'b': 3.0})

            with self.assertRaisesRegex(ValueError, 'Value \'.*\' for hyper-parameter \'.*\' is outside of range', msg=cls):
                cls({'a': 3, 'b': 100.0})

            class SubTestHyperparams(cls):
                c = hyperparams.Hyperparameter[int](0)

            self.assertEqual(SubTestHyperparams.defaults(), {'a': 2, 'b': 2.0, 'c': 0}, cls)

            testSubCls = cls.define(OrderedDict(
                c=hyperparams.Hyperparameter[int](0),
            ), set_names=True)

            self.assertEqual(testSubCls.defaults(), {'a': 2, 'b': 2.0, 'c': 0}, cls)

        class ConfigurationHyperparams(hyperparams.Hyperparams):
            configuration = hyperparams.Uniform(1.0, 10.0, 2.0)

        self.assertEqual(ConfigurationHyperparams.configuration['configuration'].to_simple_structure(), hyperparams.Uniform(1.0, 10.0, 2.0).to_simple_structure())

    def test_numpy(self):
        class TestHyperparams(hyperparams.Hyperparams):
            value = hyperparams.Hyperparameter[container.ndarray](
                default=container.ndarray([0], generate_metadata=True),
            )

        values = TestHyperparams(value=container.ndarray([1, 2, 3], generate_metadata=True))

        self.assertEqual(values.values_to_json_structure(), {'value': {'encoding': 'pickle', 'value': 'gANjbnVtcHkuY29yZS5tdWx0aWFycmF5Cl9yZWNvbnN0cnVjdApxAGNkM20uY29udGFpbmVyLm51bXB5Cm5kYXJyYXkKcQFLAIVxAkMBYnEDh3EEUnEFfXEGKFgFAAAAbnVtcHlxByhLAUsDhXEIY251bXB5CmR0eXBlCnEJWAIAAABpOHEKSwBLAYdxC1JxDChLA1gBAAAAPHENTk5OSv////9K/////0sAdHEOYolDGAEAAAAAAAAAAgAAAAAAAAADAAAAAAAAAHEPdHEQWAgAAABtZXRhZGF0YXERY2QzbS5tZXRhZGF0YS5iYXNlCkRhdGFNZXRhZGF0YQpxEimBcRN9cRQoWBEAAABfY3VycmVudF9tZXRhZGF0YXEVY2QzbS5tZXRhZGF0YS5iYXNlCk1ldGFkYXRhRW50cnkKcRYpgXEXTn1xGChYCAAAAGVsZW1lbnRzcRljZDNtLnV0aWxzCnBtYXAKcRp9cRuFcRxScR1YDAAAAGFsbF9lbGVtZW50c3EeaBYpgXEfTn1xIChoGWgdaB5OaBFjZnJvemVuZGljdApGcm96ZW5PcmRlcmVkRGljdApxISmBcSJ9cSMoWAUAAABfZGljdHEkY2NvbGxlY3Rpb25zCk9yZGVyZWREaWN0CnElKVJxJlgPAAAAc3RydWN0dXJhbF90eXBlcSdjbnVtcHkKaW50NjQKcShzWAUAAABfaGFzaHEpTnViWAgAAABpc19lbXB0eXEqiVgRAAAAaXNfZWxlbWVudHNfZW1wdHlxK4h1hnEsYmgRaCEpgXEtfXEuKGgkaCUpUnEvKFgGAAAAc2NoZW1hcTBYQgAAAGh0dHBzOi8vbWV0YWRhdGEuZGF0YWRyaXZlbmRpc2NvdmVyeS5vcmcvc2NoZW1hcy92MC9jb250YWluZXIuanNvbnExaCdoAVgJAAAAZGltZW5zaW9ucTJoISmBcTN9cTQoaCRoJSlScTVYBgAAAGxlbmd0aHE2SwNzaClOdWJ1aClOdWJoKoloK4h1hnE3YmgpTnVidWIu'}})
        self.assertTrue(numpy.array_equal(TestHyperparams.values_from_json_structure(values.values_to_json_structure())['value'], values['value']))

    def test_set(self):
        set_hyperparameter = hyperparams.Set(hyperparams.Hyperparameter[int](1), [])
        with self.assertLogs(hyperparams.logger) as cm:
            self.assertEqual(set(set_hyperparameter.sample_multiple(min_samples=2, max_samples=2)), {(1,), ()})
        self.assertEqual(len(cm.records), 1)
        elements = hyperparams.Enumeration(['a', 'b', 1, 2, None], None)
        set_hyperparameter = hyperparams.Set(elements, ('a', 'b', 1, 2, None), 5, 5)

        self.assertEqual(set_hyperparameter.get_default(), ('a', 'b', 1, 2, None))
        self.assertEqual(set_hyperparameter.sample(45), ('b', None, 'a', 1, 2))
        self.assertEqual(set_hyperparameter.get_max_samples(), 1)
        self.assertEqual(set_hyperparameter.sample_multiple(1, 1, 42), (('b', None, 1, 'a', 2),))
        self.assertEqual(set_hyperparameter.sample_multiple(0, 1, 42), ())

        self.maxDiff = None

        self.assertEqual(set_hyperparameter.to_simple_structure(), {
            'default': ('a', 'b', 1, 2, None),
            'semantic_types': [],
            'structural_type': typing.Sequence[typing.Union[str, int, type(None)]],
            'type': hyperparams.Set,
            'min_size': 5,
            'max_size': 5,
            'elements': {
                'default': None,
                'semantic_types': [],
                'structural_type': typing.Union[str, int, type(None)],
                'type': hyperparams.Enumeration,
                'values': ['a', 'b', 1, 2, None],
            },
            'is_configuration': False,
        })

        self.assertEqual(set_hyperparameter.value_to_json_structure(set_hyperparameter.get_default()), ['a', 'b', 1, 2, None])
        self.assertEqual(set_hyperparameter.value_to_json_structure(set_hyperparameter.sample(45)), ['b', None, 'a', 1, 2])

        self.assertEqual(set_hyperparameter.value_from_json_structure(set_hyperparameter.value_to_json_structure(set_hyperparameter.get_default())), set_hyperparameter.get_default())
        self.assertEqual(set_hyperparameter.value_from_json_structure(set_hyperparameter.value_to_json_structure(set_hyperparameter.sample(45))), set_hyperparameter.sample(45))

        with self.assertRaisesRegex(ValueError, 'Value \'.*\' has less than 5 elements'):
            elements = hyperparams.Enumeration(['a', 'b', 1, 2, None], None)
            hyperparams.Set(elements, (), 5, 5)

        with self.assertRaisesRegex(ValueError, 'Value \'.*\' is not among values'):
            elements = hyperparams.Enumeration(['a', 'b', 1, 2, None], None)
            hyperparams.Set(elements, ('a', 'b', 1, 2, 3), 5, 5)

        with self.assertRaisesRegex(ValueError, 'Value \'.*\' has duplicate elements'):
            elements = hyperparams.Enumeration(['a', 'b', 1, 2, None], None)
            hyperparams.Set(elements, ('a', 'b', 1, 2, 2), 5, 5)

        set_hyperparameter.contribute_to_class('foo')

        with self.assertRaises(KeyError):
            set_hyperparameter.get_default('foo')

        list_of_supported_metafeatures = ['f1', 'f2', 'f3']
        metafeature = hyperparams.Enumeration(list_of_supported_metafeatures, list_of_supported_metafeatures[0], semantic_types=['https://metadata.datadrivendiscovery.org/types/MetafeatureParameter'])
        set_hyperparameter = hyperparams.Set(metafeature, (), 0, 3)

        self.assertEqual(set_hyperparameter.get_default(), ())
        self.assertEqual(set_hyperparameter.sample(42), ('f2', 'f3'))
        self.assertEqual(set_hyperparameter.get_max_samples(), 8)
        self.assertEqual(set_hyperparameter.sample_multiple(0, 3, 42), (('f2', 'f3', 'f1'), ('f2', 'f3')))

        self.assertEqual(set_hyperparameter.value_to_json_structure(set_hyperparameter.get_default()), [])
        self.assertEqual(set_hyperparameter.value_to_json_structure(set_hyperparameter.sample(42)), ['f2', 'f3'])

        self.assertEqual(set_hyperparameter.value_from_json_structure(set_hyperparameter.value_to_json_structure(set_hyperparameter.get_default())), set_hyperparameter.get_default())
        self.assertEqual(set_hyperparameter.value_from_json_structure(set_hyperparameter.value_to_json_structure(set_hyperparameter.sample(42))), set_hyperparameter.sample(42))

        set_hyperparameter = hyperparams.Set(metafeature, (), 0, None)

        self.assertEqual(set_hyperparameter.get_default(), ())
        self.assertEqual(set_hyperparameter.sample(42), ('f2', 'f3'))
        self.assertEqual(set_hyperparameter.get_max_samples(), 8)
        self.assertEqual(set_hyperparameter.sample_multiple(0, 3, 42), (('f2', 'f3', 'f1'), ('f2', 'f3')))

    def test_set_with_hyperparams(self):
        elements = hyperparams.Hyperparams.define(OrderedDict(
            range=hyperparams.UniformInt(1, 10, 2),
            enum=hyperparams.Enumeration(['a', 'b', 1, 2, None], None),
        ))
        set_hyperparameter = hyperparams.Set(elements, (elements(range=2, enum='a'),), 0, 5)

        self.assertEqual(set_hyperparameter.get_default(), ({'range': 2, 'enum': 'a'},))
        self.assertEqual(set_hyperparameter.sample(45), ({'range': 4, 'enum': None}, {'range': 1, 'enum': 2}, {'range': 5, 'enum': 'b'}))
        self.assertEqual(set_hyperparameter.get_max_samples(), 1385980)
        self.assertEqual(set_hyperparameter.sample_multiple(1, 1, 42), (({'range': 8, 'enum': None}, {'range': 5, 'enum': 'b'}, {'range': 3, 'enum': 1}),))
        self.assertEqual(set_hyperparameter.sample_multiple(0, 1, 42), ())
        self.maxDiff = None

        self.assertEqual(set_hyperparameter.to_simple_structure(), {
            'default': ({'range': 2, 'enum': 'a'},),
            'elements': {
                'enum': {
                    'default': None,
                    'semantic_types': [],
                    'structural_type': typing.Union[str, int, type(None)],
                    'type': hyperparams.Enumeration,
                    'values': ['a', 'b', 1, 2, None],
                },
                'range': {
                    'default': 2,
                    'lower': 1,
                    'semantic_types': [],
                    'structural_type': int,
                    'type': hyperparams.UniformInt,
                    'upper': 10,
                    'lower_inclusive': True,
                    'upper_inclusive': False,
                },
            },
            'is_configuration': True,
            'max_size': 5,
            'min_size': 0,
            'semantic_types': [],
            'structural_type': typing.Sequence[elements],
            'type': hyperparams.Set,
        })

        self.assertEqual(set_hyperparameter.value_to_json_structure(set_hyperparameter.get_default()), [{'range': 2, 'enum': 'a'}])
        self.assertEqual(set_hyperparameter.value_to_json_structure(set_hyperparameter.sample(45)), [{'range': 4, 'enum': None}, {'range': 1, 'enum': 2}, {'range': 5, 'enum': 'b'}])

        self.assertEqual(set_hyperparameter.value_from_json_structure(set_hyperparameter.value_to_json_structure(set_hyperparameter.get_default())), set_hyperparameter.get_default())
        self.assertEqual(set_hyperparameter.value_from_json_structure(set_hyperparameter.value_to_json_structure(set_hyperparameter.sample(45))), set_hyperparameter.sample(45))

        # We have to explicitly disable setting names if we want to use it for "Set" hyper-parameter.
        class SetHyperparams(hyperparams.Hyperparams, set_names=False):
            choice = hyperparams.Choice({
                'none': hyperparams.Hyperparams,
                'range': hyperparams.Hyperparams.define(OrderedDict(
                    value=hyperparams.UniformInt(1, 10, 2),
                )),
            }, 'none')

        class TestHyperparams(hyperparams.Hyperparams):
            a = set_hyperparameter
            b = hyperparams.Set(SetHyperparams, (SetHyperparams({'choice': {'choice': 'none'}}),), 0, 3)

        self.assertEqual(TestHyperparams.to_simple_structure(), {
            'a': {
                'type': hyperparams.Set,
                'default': ({'range': 2, 'enum': 'a'},),
                'structural_type': typing.Sequence[elements],
                'semantic_types': [],
                'elements': {
                    'range': {
                        'type': hyperparams.UniformInt,
                        'default': 2,
                        'structural_type': int,
                        'semantic_types': [],
                        'lower': 1,
                        'upper': 10,
                        'lower_inclusive': True,
                        'upper_inclusive': False,
                    },
                    'enum': {
                        'type': hyperparams.Enumeration,
                        'default': None,
                        'structural_type': typing.Union[str, int, type(None)],
                        'semantic_types': [],
                        'values': ['a', 'b', 1, 2, None],
                    },
                },
                'is_configuration': True,
                'min_size': 0,
                'max_size': 5,
            },
            'b': {
                'type': hyperparams.Set,
                'default': ({'choice': {'choice': 'none'}},),
                'structural_type': typing.Sequence[SetHyperparams],
                'semantic_types': [],
                'elements': {
                    'choice': {
                        'type': hyperparams.Choice,
                        'default': {'choice': 'none'},
                        'structural_type': typing.Dict,
                        'semantic_types': [],
                        'choices': {
                            'none': {
                                'choice': {
                                    'type': hyperparams.Hyperparameter,
                                    'default': 'none',
                                    'structural_type': str,
                                    'semantic_types': ['https://metadata.datadrivendiscovery.org/types/ChoiceParameter'],
                                },
                            },
                            'range': {
                                'value': {
                                    'type': hyperparams.UniformInt,
                                    'default': 2,
                                    'structural_type': int,
                                    'semantic_types': [],
                                    'lower': 1,
                                    'upper': 10,
                                    'lower_inclusive': True,
                                    'upper_inclusive': False,
                                },
                                'choice': {
                                    'type': hyperparams.Hyperparameter,
                                    'default': 'range',
                                    'structural_type': str,
                                    'semantic_types': ['https://metadata.datadrivendiscovery.org/types/ChoiceParameter'],
                                },
                            },
                        },
                    },
                },
                'is_configuration': True,
                'min_size': 0,
                'max_size': 3,
            },
        })

        self.assertEqual(TestHyperparams.configuration['b'].elements.configuration['choice'].choices['range'].configuration['value'].name, 'b.choice.range.value')

        self.assertEqual(TestHyperparams.defaults(), {
            'a': ({'range': 2, 'enum': 'a'},),
            'b': ({'choice': {'choice': 'none'}},),
        })
        self.assertTrue(utils.is_instance(TestHyperparams.defaults()['a'], typing.Sequence[elements]))
        self.assertTrue(utils.is_instance(TestHyperparams.defaults()['b'], typing.Sequence[SetHyperparams]))

        with self.assertLogs(hyperparams.logger) as cm:
            self.assertEqual(TestHyperparams.sample(42), {
                'a': ({'range': 8, 'enum': None}, {'range': 5, 'enum': 'b'}, {'range': 3, 'enum': 1}),
                'b': (
                    {
                        'choice': {'value': 5, 'choice': 'range'},
                    }, {
                        'choice': {'value': 8, 'choice': 'range'},
                    },
                ),
            })
        self.assertEqual(len(cm.records), 1)

        with self.assertLogs(hyperparams.logger) as cm:
            self.assertEqual(TestHyperparams.sample(42).values_to_json_structure(), {
                'a': [{'range': 8, 'enum': None}, {'range': 5, 'enum': 'b'}, {'range': 3, 'enum': 1}],
                'b': [
                    {
                        'choice': {'value': 5, 'choice': 'range'},
                    }, {
                        'choice': {'value': 8, 'choice': 'range'},
                    },
                ],
            })
        self.assertEqual(len(cm.records), 1)
        with self.assertLogs(hyperparams.logger) as cm:
            self.assertEqual(TestHyperparams.values_from_json_structure(TestHyperparams.sample(42).values_to_json_structure()), TestHyperparams.sample(42))
        self.assertEqual(len(cm.records), 1)

        self.assertEqual(len(list(TestHyperparams.traverse())), 8)

        self.assertEqual(TestHyperparams.defaults('a'), ({'range': 2, 'enum': 'a'},))
        self.assertEqual(TestHyperparams.defaults('a.range'), 2)
        # Default of a whole "Set" hyper-parameter can be different than of nested hyper-parameters.
        self.assertEqual(TestHyperparams.defaults('a.enum'), None)
        self.assertEqual(TestHyperparams.defaults('b'), ({'choice': {'choice': 'none'}},))
        self.assertEqual(TestHyperparams.defaults('b.choice'), {'choice': 'none'})
        self.assertEqual(TestHyperparams.defaults('b.choice.none'), {'choice': 'none'})
        self.assertEqual(TestHyperparams.defaults('b.choice.none.choice'), 'none')
        self.assertEqual(TestHyperparams.defaults('b.choice.range'), {'choice': 'range', 'value': 2})
        self.assertEqual(TestHyperparams.defaults('b.choice.range.value'), 2)
        self.assertEqual(TestHyperparams.defaults('b.choice.range.choice'), 'range')

        self.assertEqual(TestHyperparams(TestHyperparams.defaults(), b=(
            SetHyperparams({
                'choice': {'value': 5, 'choice': 'range'},
            }),
            SetHyperparams({
                'choice': {'value': 8, 'choice': 'range'},
            }),
        )), {
            'a': ({'range': 2, 'enum': 'a'},),
            'b': (
                {
                    'choice': {'value': 5, 'choice': 'range'},
                },
                {
                    'choice': {'value': 8, 'choice': 'range'},
                },
            ),
        })
        self.assertEqual(TestHyperparams(TestHyperparams.defaults(), **{'a': (
            elements({'range': 8, 'enum': None}),
            elements({'range': 5, 'enum': 'b'}),
            elements({'range': 3, 'enum': 1}),
        )}), {
            'a': (
                {'range': 8, 'enum': None},
                {'range': 5, 'enum': 'b'},
                {'range': 3, 'enum': 1},
            ),
            'b': ({'choice': {'choice': 'none'}},)
        })

        self.assertEqual(TestHyperparams.defaults().replace({'a': (
            elements({'range': 8, 'enum': None}),
            elements({'range': 5, 'enum': 'b'}),
            elements({'range': 3, 'enum': 1}),
        )}), {
            'a': (
                {'range': 8, 'enum': None},
                {'range': 5, 'enum': 'b'},
                {'range': 3, 'enum': 1},
            ),
            'b': ({'choice': {'choice': 'none'}},),
        })

    def test_choice(self):
        choices_hyperparameter = hyperparams.Choice({
            'none': hyperparams.Hyperparams,
            'range': hyperparams.Hyperparams.define(OrderedDict(
                # To test that we can use this name.
                configuration=hyperparams.UniformInt(1, 10, 2),
            )),
        }, 'none')

        # Class should not be changed directly (when adding "choice").
        self.assertEqual(hyperparams.Hyperparams.configuration, {})

        self.assertEqual(choices_hyperparameter.get_default(), {'choice': 'none'})
        with self.assertLogs(hyperparams.logger) as cm:
            self.assertEqual(choices_hyperparameter.sample(45), {'choice': 'range', 'configuration': 4})
        self.assertEqual(len(cm.records), 1)
        self.assertEqual(choices_hyperparameter.get_max_samples(), 10)
        with self.assertLogs(hyperparams.logger) as cm:
            self.assertEqual(choices_hyperparameter.sample_multiple(0, 3, 42), (frozendict.frozendict({'choice': 'range', 'configuration': 8}), frozendict.frozendict({'choice': 'none'})))
        self.assertEqual(len(cm.records), 1)

        self.maxDiff = None

        self.assertEqual(choices_hyperparameter.to_simple_structure(), {
            'default': {'choice': 'none'},
            'semantic_types': [],
            'structural_type': typing.Dict,
            'type': hyperparams.Choice,
            'choices': {
                'none': {
                    'choice': {
                        'default': 'none',
                        'semantic_types': ['https://metadata.datadrivendiscovery.org/types/ChoiceParameter'],
                        'structural_type': str,
                        'type': hyperparams.Hyperparameter,
                    },
                },
                'range': {
                    'choice': {
                        'default': 'range',
                        'semantic_types': ['https://metadata.datadrivendiscovery.org/types/ChoiceParameter'],
                        'structural_type': str,
                        'type': hyperparams.Hyperparameter,
                    },
                    'configuration': {
                        'default': 2,
                        'lower': 1,
                        'lower_inclusive': True,
                        'upper': 10,
                        'upper_inclusive': False,
                        'semantic_types': [],
                        'structural_type': int,
                        'type': hyperparams.UniformInt,
                    },
                },
            },
        })

        self.assertEqual(choices_hyperparameter.value_to_json_structure(choices_hyperparameter.get_default()), {'choice': 'none'})
        with self.assertLogs(hyperparams.logger) as cm:
            self.assertEqual(choices_hyperparameter.value_to_json_structure(choices_hyperparameter.sample(45)), {'configuration': 4, 'choice': 'range'})
        self.assertEqual(len(cm.records), 1)

        self.assertEqual(choices_hyperparameter.value_from_json_structure(choices_hyperparameter.value_to_json_structure(choices_hyperparameter.get_default())), choices_hyperparameter.get_default())
        with self.assertLogs(hyperparams.logger) as cm:
            self.assertEqual(choices_hyperparameter.value_from_json_structure(choices_hyperparameter.value_to_json_structure(choices_hyperparameter.sample(45))), choices_hyperparameter.sample(45))
        self.assertEqual(len(cm.records), 1)

        # We have to explicitly disable setting names if we want to use it for "Choice" hyper-parameter.
        class ChoicesHyperparams(hyperparams.Hyperparams, set_names=False):
            foo = hyperparams.UniformInt(5, 20, 10)

        class TestHyperparams(hyperparams.Hyperparams):
            a = choices_hyperparameter
            b = hyperparams.Choice({
                'nochoice': ChoicesHyperparams,
            }, 'nochoice')

        self.assertEqual(TestHyperparams.configuration['a'].choices['range'].configuration['configuration'].name, 'a.range.configuration')

        self.assertEqual(TestHyperparams.defaults(), {'a': {'choice': 'none'}, 'b': {'choice': 'nochoice', 'foo': 10}})
        self.assertIsInstance(TestHyperparams.defaults()['a'], hyperparams.Hyperparams)
        self.assertIsInstance(TestHyperparams.defaults()['b'], ChoicesHyperparams)

        with self.assertLogs(hyperparams.logger) as cm:
            self.assertEqual(TestHyperparams.sample(42), {'a': {'choice': 'none'}, 'b': {'choice': 'nochoice', 'foo': 8}})
        self.assertEqual(len(cm.records), 1)

        with self.assertLogs(hyperparams.logger) as cm:
            self.assertEqual(TestHyperparams.sample(42).values_to_json_structure(), {'a': {'choice': 'none'}, 'b': {'choice': 'nochoice', 'foo': 8}})
        self.assertEqual(len(cm.records), 1)
        with self.assertLogs(hyperparams.logger) as cm:
            self.assertEqual(TestHyperparams.values_from_json_structure(TestHyperparams.sample(42).values_to_json_structure()), TestHyperparams.sample(42))
        self.assertEqual(len(cm.records), 1)

        self.assertEqual(len(list(TestHyperparams.traverse())), 7)

        self.assertEqual(TestHyperparams.defaults('a'), {'choice': 'none'})
        self.assertEqual(TestHyperparams.defaults('a.none'), {'choice': 'none'})
        self.assertEqual(TestHyperparams.defaults('a.none.choice'), 'none')
        self.assertEqual(TestHyperparams.defaults('a.range'), {'choice': 'range', 'configuration': 2})
        self.assertEqual(TestHyperparams.defaults('a.range.configuration'), 2)
        self.assertEqual(TestHyperparams.defaults('a.range.choice'), 'range')
        self.assertEqual(TestHyperparams.defaults('b'), {'choice': 'nochoice', 'foo': 10})
        self.assertEqual(TestHyperparams.defaults('b.nochoice'), {'choice': 'nochoice', 'foo': 10})
        self.assertEqual(TestHyperparams.defaults('b.nochoice.foo'), 10)
        self.assertEqual(TestHyperparams.defaults('b.nochoice.choice'), 'nochoice')

    def test_primitive(self):
        # To hide any logging or stdout output.
        with utils.silence():
            index.register_primitive('d3m.primitives.regression.monomial.Test', MonomialPrimitive)
            index.register_primitive('d3m.primitives.data_generation.random.Test', RandomPrimitive)
            index.register_primitive('d3m.primitives.operator.sum.Test', SumPrimitive)
            index.register_primitive('d3m.primitives.operator.increment.Test', IncrementPrimitive)

        hyperparameter = hyperparams.Primitive(MonomialPrimitive)

        self.assertEqual(hyperparameter.structural_type, MonomialPrimitive)
        self.assertEqual(hyperparameter.get_default(), MonomialPrimitive)
        # To hide any logging or stdout output.
        with utils.silence():
            self.assertEqual(hyperparameter.sample(42), MonomialPrimitive)

        hyperparams_class = MonomialPrimitive.metadata.get_hyperparams()
        primitive = MonomialPrimitive(hyperparams=hyperparams_class.defaults())

        hyperparameter = hyperparams.Enumeration([MonomialPrimitive, RandomPrimitive, SumPrimitive, IncrementPrimitive, None], None)

        self.assertEqual(hyperparameter.structural_type, typing.Union[MonomialPrimitive, RandomPrimitive, SumPrimitive, IncrementPrimitive, type(None)])
        self.assertEqual(hyperparameter.get_default(), None)
        self.assertEqual(hyperparameter.sample(42), IncrementPrimitive)

        hyperparameter = hyperparams.Enumeration[typing.Optional[base.PrimitiveBase]]([MonomialPrimitive, RandomPrimitive, SumPrimitive, IncrementPrimitive, None], None)

        self.assertEqual(hyperparameter.structural_type, typing.Optional[base.PrimitiveBase])
        self.assertEqual(hyperparameter.get_default(), None)
        self.assertEqual(hyperparameter.sample(42), IncrementPrimitive)

        set_hyperparameter = hyperparams.Set(hyperparameter, (MonomialPrimitive, RandomPrimitive), 2, 4)

        self.assertEqual(set_hyperparameter.get_default(), (MonomialPrimitive, RandomPrimitive))
        self.assertEqual(set_hyperparameter.sample(42), (RandomPrimitive, None, SumPrimitive, MonomialPrimitive))

        union_hyperparameter = hyperparams.Union(OrderedDict(
            none=hyperparams.Hyperparameter(None),
            primitive=hyperparams.Enumeration[base.PrimitiveBase]([MonomialPrimitive, RandomPrimitive, SumPrimitive, IncrementPrimitive], MonomialPrimitive),
        ), 'none')

        self.assertEqual(union_hyperparameter.get_default(), None)
        self.assertEqual(union_hyperparameter.sample(45), SumPrimitive)

        hyperparameter = hyperparams.Enumeration([primitive, RandomPrimitive, SumPrimitive, IncrementPrimitive, None], None)

        self.assertEqual(hyperparameter.structural_type, typing.Union[MonomialPrimitive, RandomPrimitive, SumPrimitive, IncrementPrimitive, type(None)])
        self.assertEqual(hyperparameter.get_default(), None)
        self.assertEqual(hyperparameter.sample(42), IncrementPrimitive)

        hyperparameter = hyperparams.Enumeration[typing.Optional[base.PrimitiveBase]]([primitive, RandomPrimitive, SumPrimitive, IncrementPrimitive, None], None)

        self.assertEqual(hyperparameter.structural_type, typing.Optional[base.PrimitiveBase])
        self.assertEqual(hyperparameter.get_default(), None)
        self.assertEqual(hyperparameter.sample(42), IncrementPrimitive)

        set_hyperparameter = hyperparams.Set(hyperparameter, (primitive, RandomPrimitive), 2, 4)

        self.assertEqual(set_hyperparameter.get_default(), (primitive, RandomPrimitive))
        self.assertEqual(set_hyperparameter.sample(42), (RandomPrimitive, None, SumPrimitive, primitive))

        union_hyperparameter = hyperparams.Union(OrderedDict(
            none=hyperparams.Hyperparameter(None),
            primitive=hyperparams.Enumeration[base.PrimitiveBase]([primitive, RandomPrimitive, SumPrimitive, IncrementPrimitive], primitive),
        ), 'none')

        self.assertEqual(union_hyperparameter.get_default(), None)
        self.assertEqual(union_hyperparameter.sample(45), SumPrimitive)

        hyperparameter = hyperparams.Primitive(primitive)

        self.assertEqual(hyperparameter.structural_type, MonomialPrimitive)
        self.assertEqual(hyperparameter.get_default(), primitive)
        # To hide any logging or stdout output.
        with utils.silence():
            self.assertEqual(hyperparameter.sample(42), primitive)

        hyperparameter = hyperparams.Primitive[base.PrimitiveBase](MonomialPrimitive)

        self.assertEqual(hyperparameter.get_default(), MonomialPrimitive)
        # To hide any logging or stdout output.
        with utils.silence():
            # There might be additional primitives available in the system,
            # so we cannot know which one will really be returned.
            self.assertTrue(hyperparameter.sample(42), hyperparameter.matching_primitives)

        self.maxDiff = None

        self.assertEqual(hyperparameter.to_simple_structure(), {
            'default': MonomialPrimitive,
            'semantic_types': [],
            'structural_type': base.PrimitiveBase,
            'type': hyperparams.Primitive,
            'primitive_families': [],
            'algorithm_types': [],
            'produce_methods': [],
        })

        self.assertEqual(hyperparameter.value_to_json_structure(hyperparameter.get_default()), {'class': 'd3m.primitives.regression.monomial.Test'})
        self.assertEqual(hyperparameter.value_from_json_structure(hyperparameter.value_to_json_structure(hyperparameter.get_default())), hyperparameter.get_default())

        self.assertTrue(hyperparameter.get_max_samples() >= 4, hyperparameter.get_max_samples())

        hyperparameter = hyperparams.Primitive[base.PrimitiveBase](primitive)

        self.assertEqual(hyperparameter.get_default(), primitive)

        self.assertEqual(hyperparameter.to_simple_structure(), {
            'default': primitive,
            'semantic_types': [],
            'structural_type': base.PrimitiveBase,
            'type': hyperparams.Primitive,
            'primitive_families': [],
            'algorithm_types': [],
            'produce_methods': [],
        })

        self.assertEqual(hyperparameter.value_to_json_structure(hyperparameter.get_default()), {'instance': 'gANjdGVzdF9wcmltaXRpdmVzLm1vbm9taWFsCk1vbm9taWFsUHJpbWl0aXZlCnEAKYFxAX1xAihYCwAAAGNvbnN0cnVjdG9ycQN9cQQoWAsAAABoeXBlcnBhcmFtc3EFY3Rlc3RfcHJpbWl0aXZlcy5tb25vbWlhbApIeXBlcnBhcmFtcwpxBimBcQd9cQhYBAAAAGJpYXNxCUcAAAAAAAAAAHNiWAsAAAByYW5kb21fc2VlZHEKSwB1WAYAAABwYXJhbXNxC2N0ZXN0X3ByaW1pdGl2ZXMubW9ub21pYWwKUGFyYW1zCnEMKYFxDVgBAAAAYXEOSwBzdWIu'})

        set_hyperparameter = hyperparams.Set(hyperparameter, (MonomialPrimitive, RandomPrimitive), 2, 4)

        self.assertEqual(set_hyperparameter.get_default(), (MonomialPrimitive, RandomPrimitive))

        union_hyperparameter = hyperparams.Union(OrderedDict(
            none=hyperparams.Hyperparameter(None),
            primitive=hyperparameter,
        ), 'none')

        self.assertEqual(union_hyperparameter.get_default(), None)

    def test_invalid_name(self):
        with self.assertRaisesRegex(ValueError, 'Hyper-parameter name \'.*\' contains invalid characters.'):
            hyperparams.Hyperparams.define({
                'foo.bar': hyperparams.Uniform(1.0, 10.0, 2.0),
            })

    def test_class_as_default(self):
        class Foo:
            pass

        foo = Foo()

        hyperparameter = hyperparams.Enumeration(['a', 'b', 1, 2, foo], foo)

        self.assertEqual(hyperparameter.value_to_json_structure(1), {'encoding': 'pickle', 'value': 'gANLAS4='})

        hyperparameter = hyperparams.Enumeration(['a', 'b', 1, 2], 2)

        self.assertEqual(hyperparameter.value_to_json_structure(1), 1)

    def test_configuration_immutability(self):
        class TestHyperparams(hyperparams.Hyperparams):
            a = hyperparams.Union(OrderedDict(
                range=hyperparams.UniformInt(1, 10, 2),
                none=hyperparams.Hyperparameter(None),
            ), 'range')
            b = hyperparams.Uniform(1.0, 10.0, 2.0)

        with self.assertRaisesRegex(TypeError, '\'FrozenOrderedDict\' object does not support item assignment'):
            TestHyperparams.configuration['c'] = hyperparams.Enumeration(['a', 'b', 1, 2, None], None)

        with self.assertRaisesRegex(AttributeError, 'Hyper-parameters configuration is immutable'):
            TestHyperparams.configuration = OrderedDict(
                range=hyperparams.UniformInt(1, 10, 2),
                none=hyperparams.Hyperparameter(None),
            )

    def test_dict_as_default(self):
        Inputs = container.DataFrame
        Outputs = container.DataFrame

        class Hyperparams(hyperparams.Hyperparams):
            value = hyperparams.Hyperparameter({}, semantic_types=['https://metadata.datadrivendiscovery.org/types/ControlParameter'])

        # Silence any validation warnings.
        with utils.silence():
            class Primitive(transformer.TransformerPrimitiveBase[Inputs, Outputs, Hyperparams]):
                metadata = metadata_base.PrimitiveMetadata({
                    'id': '152ea984-d8a4-4a37-87a0-29829b082e54',
                    'version': '0.1.0',
                    'name': "Test Primitive",
                    'python_path': 'd3m.primitives.test.dict_as_default',
                    'algorithm_types': [
                        metadata_base.PrimitiveAlgorithmType.PRINCIPAL_COMPONENT_ANALYSIS,
                    ],
                    'primitive_family': metadata_base.PrimitiveFamily.FEATURE_SELECTION,
                })

                def produce(self, *, inputs: Inputs, timeout: float = None, iterations: int = None) -> base.CallResult[Outputs]:
                    pass

        self.assertEqual(Primitive.metadata.query()['primitive_code']['hyperparams']['value']['default'], {})

    def test_comma_warning(self):
        logger = logging.getLogger('d3m.metadata.hyperparams')

        with self.assertLogs(logger=logger, level=logging.DEBUG) as cm:
            class Hyperparams(hyperparams.Hyperparams):
                value = hyperparams.Hyperparameter({}, semantic_types=['https://metadata.datadrivendiscovery.org/types/ControlParameter']),

        self.assertEqual(len(cm.records), 1)
        self.assertEqual(cm.records[0].message, 'Probably invalid definition of a hyper-parameter. Hyper-parameter should be defined as class attribute without a trailing comma.')

    def test_json_schema(self):
        Inputs = container.DataFrame
        Outputs = container.DataFrame

        # Silence any validation warnings.
        with utils.silence():
            # Defining primitive triggers checking against JSON schema.
            class TestJsonPrimitive(transformer.TransformerPrimitiveBase[Inputs, Outputs, TestPicklingHyperparams]):
                metadata = metadata_base.PrimitiveMetadata({
                    'id': 'cdfada09-5161-4f2e-bc7f-223d843d59c1',
                    'version': '0.1.0',
                    'name': "Test Primitive",
                    'python_path': 'd3m.primitives.test.json_schema',
                    'algorithm_types': [
                        metadata_base.PrimitiveAlgorithmType.PRINCIPAL_COMPONENT_ANALYSIS,
                    ],
                    'primitive_family': metadata_base.PrimitiveFamily.FEATURE_SELECTION,
                })

                def produce(self, *, inputs: Inputs, timeout: float = None, iterations: int = None) -> base.CallResult[Outputs]:
                    pass

    def test_pickling(self):
        pickle.loads(pickle.dumps(TestPicklingHyperparams))

        unpickled = pickle.loads(pickle.dumps(TestPicklingHyperparams.defaults()))

        self.assertEqual(unpickled['choice'].configuration['value'].structural_type, typing.Union[float, int])

    def test_sorted_set(self):
        set_hyperparameter = hyperparams.SortedSet(hyperparams.Hyperparameter[int](1), [])
        with self.assertLogs(hyperparams.logger) as cm:
            self.assertEqual(set(set_hyperparameter.sample_multiple(min_samples=2, max_samples=2)), {(1,), ()})
        self.assertEqual(len(cm.records), 1)

        elements = hyperparams.Enumeration(['a', 'b', 'c', 'd', 'e'], 'e')
        set_hyperparameter = hyperparams.SortedSet(elements, ('a', 'b', 'c', 'd', 'e'), 5, 5)

        self.assertEqual(set_hyperparameter.get_default(), ('a', 'b', 'c', 'd', 'e'))
        self.assertEqual(set_hyperparameter.sample(45), ('a', 'b', 'c', 'd', 'e'))
        self.assertEqual(set_hyperparameter.get_max_samples(), 1)
        self.assertEqual(set_hyperparameter.sample_multiple(1, 1, 42), (('a', 'b', 'c', 'd', 'e'),))
        self.assertEqual(set_hyperparameter.sample_multiple(0, 1, 42), ())

        self.maxDiff = None

        self.assertEqual(set_hyperparameter.to_simple_structure(), {
            'default': ('a', 'b', 'c', 'd', 'e'),
            'semantic_types': [],
            'structural_type': typing.Sequence[str],
            'type': hyperparams.SortedSet,
            'min_size': 5,
            'max_size': 5,
            'elements': {
                'default': 'e',
                'semantic_types': [],
                'structural_type': str,
                'type': hyperparams.Enumeration,
                'values': ['a', 'b', 'c', 'd', 'e'],
            },
            'ascending': True,
        })

        self.assertEqual(set_hyperparameter.value_to_json_structure(set_hyperparameter.get_default()), ['a', 'b', 'c', 'd', 'e'])
        self.assertEqual(set_hyperparameter.value_to_json_structure(set_hyperparameter.sample(45)), ['a', 'b', 'c', 'd', 'e'])

        self.assertEqual(set_hyperparameter.value_from_json_structure(set_hyperparameter.value_to_json_structure(set_hyperparameter.get_default())), set_hyperparameter.get_default())
        self.assertEqual(set_hyperparameter.value_from_json_structure(set_hyperparameter.value_to_json_structure(set_hyperparameter.sample(45))), set_hyperparameter.sample(45))

        with self.assertRaisesRegex(ValueError, 'Value \'.*\' has less than 5 elements'):
            elements = hyperparams.Enumeration(['a', 'b', 'c', 'd', 'e'], 'e')
            hyperparams.SortedSet(elements, (), 5, 5)

        with self.assertRaisesRegex(ValueError, 'Value \'.*\' is not among values'):
            elements = hyperparams.Enumeration(['a', 'b', 'c', 'd', 'e'], 'e')
            hyperparams.SortedSet(elements, ('a', 'b', 'c', 'd', 'f'), 5, 5)

        with self.assertRaisesRegex(ValueError, 'Value \'.*\' has duplicate elements'):
            elements = hyperparams.Enumeration(['a', 'b', 'c', 'd', 'e'], 'e')
            hyperparams.SortedSet(elements, ('a', 'b', 'c', 'd', 'd'), 5, 5)

        set_hyperparameter.contribute_to_class('foo')

        with self.assertRaises(KeyError):
            set_hyperparameter.get_default('foo')

        list_of_supported_metafeatures = ['f1', 'f2', 'f3']
        metafeature = hyperparams.Enumeration(list_of_supported_metafeatures, list_of_supported_metafeatures[0], semantic_types=['https://metadata.datadrivendiscovery.org/types/MetafeatureParameter'])
        set_hyperparameter = hyperparams.SortedSet(metafeature, (), 0, 3)

        self.assertEqual(set_hyperparameter.get_default(), ())
        self.assertEqual(set_hyperparameter.sample(42), ('f2', 'f3'))
        self.assertEqual(set_hyperparameter.get_max_samples(), 8)
        self.assertEqual(set_hyperparameter.sample_multiple(0, 3, 42), (('f1', 'f2', 'f3'), ('f2', 'f3')))

        self.assertEqual(set_hyperparameter.value_to_json_structure(set_hyperparameter.get_default()), [])
        self.assertEqual(set_hyperparameter.value_to_json_structure(set_hyperparameter.sample(42)), ['f2', 'f3'])

        self.assertEqual(set_hyperparameter.value_from_json_structure(set_hyperparameter.value_to_json_structure(set_hyperparameter.get_default())), set_hyperparameter.get_default())
        self.assertEqual(set_hyperparameter.value_from_json_structure(set_hyperparameter.value_to_json_structure(set_hyperparameter.sample(42))), set_hyperparameter.sample(42))

        set_hyperparameter = hyperparams.SortedSet(metafeature, (), 0, None)

        self.assertEqual(set_hyperparameter.get_default(), ())
        self.assertEqual(set_hyperparameter.sample(42), ('f2', 'f3'))
        self.assertEqual(set_hyperparameter.get_max_samples(), 8)
        self.assertEqual(set_hyperparameter.sample_multiple(0, 3, 42), (('f1', 'f2', 'f3'), ('f2', 'f3')))

        set_hyperparameter = hyperparams.SortedSet(hyperparams.Hyperparameter[int](0), (0, 1), min_size=2, max_size=2)

        with self.assertLogs(hyperparams.logger) as cm:
            self.assertEqual(set_hyperparameter.sample_multiple(1, 1, 42), ((0, 1),))
        self.assertEqual(len(cm.records), 1)

        with self.assertLogs(hyperparams.logger) as cm:
            self.assertEqual(set_hyperparameter.sample(42), (0, 1))
        self.assertEqual(len(cm.records), 1)

        set_hyperparameter = hyperparams.SortedSet(hyperparams.Hyperparameter[int](0), (0,), min_size=1, max_size=1)

        with self.assertLogs(hyperparams.logger) as cm:
            set_hyperparameter.sample(42)
        self.assertEqual(len(cm.records), 1)

        set_hyperparameter = hyperparams.SortedSet(hyperparams.Uniform(0.0, 100.0, 50.0, lower_inclusive=False, upper_inclusive=False), (25.0, 75.0), min_size=2, max_size=2)

        self.assertEqual(set_hyperparameter.sample(42), (37.454011884736246, 95.07143064099162))

    def test_sorted_set_with_hyperparams(self):
        elements = hyperparams.Hyperparams.define(OrderedDict(
            range=hyperparams.UniformInt(1, 10, 2),
            enum=hyperparams.Enumeration(['a', 'b', 'c', 'd', 'e'], 'e'),
        ))

        with self.assertRaises(exceptions.NotSupportedError):
            hyperparams.SortedSet(elements, (elements(range=2, enum='a'),), 0, 5)

    def test_list(self):
        list_hyperparameter = hyperparams.List(hyperparams.Hyperparameter[int](1), [], 0, 1)
        with self.assertLogs(hyperparams.logger) as cm:
            self.assertEqual(set(list_hyperparameter.sample_multiple(min_samples=2, max_samples=2)), {(1,), ()})
        self.assertEqual(len(cm.records), 1)

        elements = hyperparams.Enumeration(['a', 'b', 1, 2, None], None)
        list_hyperparameter = hyperparams.List(elements, ('a', 'b', 1, 2, None), 5, 5)

        self.assertEqual(list_hyperparameter.get_default(), ('a', 'b', 1, 2, None))
        self.assertEqual(list_hyperparameter.sample(45), (2, 2, None, 'a', 2))
        self.assertEqual(list_hyperparameter.get_max_samples(), 3125)
        self.assertEqual(list_hyperparameter.sample_multiple(1, 1, 42), ((2, None, 1, None, None),))
        self.assertEqual(list_hyperparameter.sample_multiple(0, 1, 42), ())

        self.maxDiff = None

        self.assertEqual(list_hyperparameter.to_simple_structure(), {
            'default': ('a', 'b', 1, 2, None),
            'semantic_types': [],
            'structural_type': typing.Sequence[typing.Union[str, int, type(None)]],
            'type': hyperparams.List,
            'min_size': 5,
            'max_size': 5,
            'elements': {
                'default': None,
                'semantic_types': [],
                'structural_type': typing.Union[str, int, type(None)],
                'type': hyperparams.Enumeration,
                'values': ['a', 'b', 1, 2, None],
            },
            'is_configuration': False,
        })

        self.assertEqual(list_hyperparameter.value_to_json_structure(list_hyperparameter.get_default()), ['a', 'b', 1, 2, None])
        self.assertEqual(list_hyperparameter.value_to_json_structure(list_hyperparameter.sample(45)), [2, 2, None, 'a', 2])

        self.assertEqual(list_hyperparameter.value_from_json_structure(list_hyperparameter.value_to_json_structure(list_hyperparameter.get_default())), list_hyperparameter.get_default())
        self.assertEqual(list_hyperparameter.value_from_json_structure(list_hyperparameter.value_to_json_structure(list_hyperparameter.sample(45))), list_hyperparameter.sample(45))

        with self.assertRaisesRegex(ValueError, 'Value \'.*\' has less than 5 elements'):
            elements = hyperparams.Enumeration(['a', 'b', 1, 2, None], None)
            hyperparams.List(elements, (), 5, 5)

        with self.assertRaisesRegex(ValueError, 'Value \'.*\' is not among values'):
            elements = hyperparams.Enumeration(['a', 'b', 1, 2, None], None)
            hyperparams.List(elements, ('a', 'b', 1, 2, 3), 5, 5)

        list_hyperparameter.contribute_to_class('foo')

        with self.assertRaises(KeyError):
            list_hyperparameter.get_default('foo')

        list_of_supported_metafeatures = ['f1', 'f2', 'f3']
        metafeature = hyperparams.Enumeration(list_of_supported_metafeatures, list_of_supported_metafeatures[0], semantic_types=['https://metadata.datadrivendiscovery.org/types/MetafeatureParameter'])
        list_hyperparameter = hyperparams.List(metafeature, (), 0, 3)

        self.assertEqual(list_hyperparameter.get_default(), ())
        self.assertEqual(list_hyperparameter.sample(42), ('f1', 'f3'))
        self.assertEqual(list_hyperparameter.get_max_samples(), 40)
        self.assertEqual(list_hyperparameter.sample_multiple(0, 3, 42), (('f1', 'f3', 'f3'), ('f1', 'f1', 'f3')))

        self.assertEqual(list_hyperparameter.value_to_json_structure(list_hyperparameter.get_default()), [])
        self.assertEqual(list_hyperparameter.value_to_json_structure(list_hyperparameter.sample(42)), ['f1', 'f3'])

        self.assertEqual(list_hyperparameter.value_from_json_structure(list_hyperparameter.value_to_json_structure(list_hyperparameter.get_default())), list_hyperparameter.get_default())
        self.assertEqual(list_hyperparameter.value_from_json_structure(list_hyperparameter.value_to_json_structure(list_hyperparameter.sample(42))), list_hyperparameter.sample(42))

        list_hyperparameter = hyperparams.List(metafeature, (), 0, 10)

        self.assertEqual(list_hyperparameter.get_default(), ())
        self.assertEqual(list_hyperparameter.sample(42), ('f1', 'f3', 'f3', 'f1', 'f1', 'f3'))
        self.assertEqual(list_hyperparameter.get_max_samples(), 88573)
        self.assertEqual(list_hyperparameter.sample_multiple(0, 3, 42), (('f1', 'f3', 'f3'), ('f1', 'f1', 'f3', 'f2', 'f3', 'f3', 'f3')))

        list_hyperparameter = hyperparams.List(hyperparams.Bounded(1, None, 100), (100,), min_size=1, max_size=None)

        with self.assertLogs(hyperparams.logger) as cm:
            self.assertEqual(list_hyperparameter.sample(42), (100,))
        self.assertEqual(len(cm.records), 1)

    def test_list_with_hyperparams(self):
        elements = hyperparams.Hyperparams.define(OrderedDict(
            range=hyperparams.UniformInt(1, 10, 2),
            enum=hyperparams.Enumeration(['a', 'b', 1, 2, None], None),
        ))
        list_hyperparameter = hyperparams.List(elements, (elements(range=2, enum='a'),), 0, 5)

        self.assertEqual(list_hyperparameter.get_default(), ({'range': 2, 'enum': 'a'},))
        self.assertEqual(list_hyperparameter.sample(45), ({'range': 4, 'enum': None}, {'range': 1, 'enum': 2}, {'range': 5, 'enum': 'b'}))
        self.assertEqual(list_hyperparameter.get_max_samples(), 188721946)
        self.assertEqual(list_hyperparameter.sample_multiple(1, 1, 42), (({'range': 8, 'enum': None}, {'range': 5, 'enum': 'b'}, {'range': 3, 'enum': 1}),))
        self.assertEqual(list_hyperparameter.sample_multiple(0, 1, 42), ())
        self.maxDiff = None

        self.assertEqual(list_hyperparameter.to_simple_structure(), {
            'default': ({'range': 2, 'enum': 'a'},),
            'elements': {
                'enum': {
                    'default': None,
                    'semantic_types': [],
                    'structural_type': typing.Union[str, int, type(None)],
                    'type': hyperparams.Enumeration,
                    'values': ['a', 'b', 1, 2, None],
                },
                'range': {
                    'default': 2,
                    'lower': 1,
                    'semantic_types': [],
                    'structural_type': int,
                    'type': hyperparams.UniformInt,
                    'upper': 10,
                    'lower_inclusive': True,
                    'upper_inclusive': False,
                },
            },
            'is_configuration': True,
            'max_size': 5,
            'min_size': 0,
            'semantic_types': [],
            'structural_type': typing.Sequence[elements],
            'type': hyperparams.List,
        })

        self.assertEqual(list_hyperparameter.value_to_json_structure(list_hyperparameter.get_default()), [{'range': 2, 'enum': 'a'}])
        self.assertEqual(list_hyperparameter.value_to_json_structure(list_hyperparameter.sample(45)), [{'range': 4, 'enum': None}, {'range': 1, 'enum': 2}, {'range': 5, 'enum': 'b'}])

        self.assertEqual(list_hyperparameter.value_from_json_structure(list_hyperparameter.value_to_json_structure(list_hyperparameter.get_default())), list_hyperparameter.get_default())
        self.assertEqual(list_hyperparameter.value_from_json_structure(list_hyperparameter.value_to_json_structure(list_hyperparameter.sample(45))), list_hyperparameter.sample(45))

        # We have to explicitly disable setting names if we want to use it for "List" hyper-parameter.
        class ListHyperparams(hyperparams.Hyperparams, set_names=False):
            choice = hyperparams.Choice({
                'none': hyperparams.Hyperparams,
                'range': hyperparams.Hyperparams.define(OrderedDict(
                    value=hyperparams.UniformInt(1, 10, 2),
                )),
            }, 'none')

        class TestHyperparams(hyperparams.Hyperparams):
            a = list_hyperparameter
            b = hyperparams.List(ListHyperparams, (ListHyperparams({'choice': {'choice': 'none'}}),), 0, 3)

        self.assertEqual(TestHyperparams.to_simple_structure(), {
            'a': {
                'type': hyperparams.List,
                'default': ({'range': 2, 'enum': 'a'},),
                'structural_type': typing.Sequence[elements],
                'semantic_types': [],
                'elements': {
                    'range': {
                        'type': hyperparams.UniformInt,
                        'default': 2,
                        'structural_type': int,
                        'semantic_types': [],
                        'lower': 1,
                        'upper': 10,
                        'lower_inclusive': True,
                        'upper_inclusive': False,
                    },
                    'enum': {
                        'type': hyperparams.Enumeration,
                        'default': None,
                        'structural_type': typing.Union[str, int, type(None)],
                        'semantic_types': [],
                        'values': ['a', 'b', 1, 2, None],
                    },
                },
                'is_configuration': True,
                'min_size': 0,
                'max_size': 5,
            },
            'b': {
                'type': hyperparams.List,
                'default': ({'choice': {'choice': 'none'}},),
                'structural_type': typing.Sequence[ListHyperparams],
                'semantic_types': [],
                'elements': {
                    'choice': {
                        'type': hyperparams.Choice,
                        'default': {'choice': 'none'},
                        'structural_type': typing.Dict,
                        'semantic_types': [],
                        'choices': {
                            'none': {
                                'choice': {
                                    'type': hyperparams.Hyperparameter,
                                    'default': 'none',
                                    'structural_type': str,
                                    'semantic_types': ['https://metadata.datadrivendiscovery.org/types/ChoiceParameter'],
                                },
                            },
                            'range': {
                                'value': {
                                    'type': hyperparams.UniformInt,
                                    'default': 2,
                                    'structural_type': int,
                                    'semantic_types': [],
                                    'lower': 1,
                                    'upper': 10,
                                    'lower_inclusive': True,
                                    'upper_inclusive': False,
                                },
                                'choice': {
                                    'type': hyperparams.Hyperparameter,
                                    'default': 'range',
                                    'structural_type': str,
                                    'semantic_types': ['https://metadata.datadrivendiscovery.org/types/ChoiceParameter'],
                                },
                            },
                        },
                    },
                },
                'is_configuration': True,
                'min_size': 0,
                'max_size': 3,
            },
        })

        self.assertEqual(TestHyperparams.configuration['b'].elements.configuration['choice'].choices['range'].configuration['value'].name, 'b.choice.range.value')

        self.assertEqual(TestHyperparams.defaults(), {
            'a': ({'range': 2, 'enum': 'a'},),
            'b': ({'choice': {'choice': 'none'}},),
        })
        self.assertTrue(utils.is_instance(TestHyperparams.defaults()['a'], typing.Sequence[elements]))
        self.assertTrue(utils.is_instance(TestHyperparams.defaults()['b'], typing.Sequence[ListHyperparams]))

        with self.assertLogs(hyperparams.logger) as cm:
            self.assertEqual(TestHyperparams.sample(42), {
                'a': ({'range': 8, 'enum': None}, {'range': 5, 'enum': 'b'}, {'range': 3, 'enum': 1}),
                'b': (
                    {
                        'choice': {'value': 5, 'choice': 'range'},
                    }, {
                        'choice': {'value': 8, 'choice': 'range'},
                    },
                ),
            })
        self.assertEqual(len(cm.records), 1)

        with self.assertLogs(hyperparams.logger) as cm:
            self.assertEqual(TestHyperparams.sample(42).values_to_json_structure(), {
                'a': [{'range': 8, 'enum': None}, {'range': 5, 'enum': 'b'}, {'range': 3, 'enum': 1}],
                'b': [
                    {
                        'choice': {'value': 5, 'choice': 'range'},
                    }, {
                        'choice': {'value': 8, 'choice': 'range'},
                    },
                ],
            })
        self.assertEqual(len(cm.records), 1)
        with self.assertLogs(hyperparams.logger) as cm:
            self.assertEqual(TestHyperparams.values_from_json_structure(TestHyperparams.sample(42).values_to_json_structure()), TestHyperparams.sample(42))
        self.assertEqual(len(cm.records), 1)

        self.assertEqual(len(list(TestHyperparams.traverse())), 8)

        self.assertEqual(TestHyperparams.defaults('a'), ({'range': 2, 'enum': 'a'},))
        self.assertEqual(TestHyperparams.defaults('a.range'), 2)
        # Default of a whole "List" hyper-parameter can be different than of nested hyper-parameters.
        self.assertEqual(TestHyperparams.defaults('a.enum'), None)
        self.assertEqual(TestHyperparams.defaults('b'), ({'choice': {'choice': 'none'}},))
        self.assertEqual(TestHyperparams.defaults('b.choice'), {'choice': 'none'})
        self.assertEqual(TestHyperparams.defaults('b.choice.none'), {'choice': 'none'})
        self.assertEqual(TestHyperparams.defaults('b.choice.none.choice'), 'none')
        self.assertEqual(TestHyperparams.defaults('b.choice.range'), {'choice': 'range', 'value': 2})
        self.assertEqual(TestHyperparams.defaults('b.choice.range.value'), 2)
        self.assertEqual(TestHyperparams.defaults('b.choice.range.choice'), 'range')

        self.assertEqual(TestHyperparams(TestHyperparams.defaults(), b=(
            ListHyperparams({
                'choice': {'value': 5, 'choice': 'range'},
            }),
            ListHyperparams({
                'choice': {'value': 8, 'choice': 'range'},
            }),
        )), {
            'a': ({'range': 2, 'enum': 'a'},),
            'b': (
                {
                    'choice': {'value': 5, 'choice': 'range'},
                },
                {
                    'choice': {'value': 8, 'choice': 'range'},
                },
            ),
        })
        self.assertEqual(TestHyperparams(TestHyperparams.defaults(), **{'a': (
            elements({'range': 8, 'enum': None}),
            elements({'range': 5, 'enum': 'b'}),
            elements({'range': 3, 'enum': 1}),
        )}), {
            'a': (
                {'range': 8, 'enum': None},
                {'range': 5, 'enum': 'b'},
                {'range': 3, 'enum': 1},
            ),
            'b': ({'choice': {'choice': 'none'}},)
        })

        self.assertEqual(TestHyperparams.defaults().replace({'a': (
            elements({'range': 8, 'enum': None}),
            elements({'range': 5, 'enum': 'b'}),
            elements({'range': 3, 'enum': 1}),
        )}), {
            'a': (
                {'range': 8, 'enum': None},
                {'range': 5, 'enum': 'b'},
                {'range': 3, 'enum': 1},
            ),
            'b': ({'choice': {'choice': 'none'}},),
        })

    def test_sorted_list(self):
        list_hyperparameter = hyperparams.SortedList(hyperparams.Hyperparameter[int](1), [], 0, 1)
        with self.assertLogs(hyperparams.logger) as cm:
            self.assertEqual(set(list_hyperparameter.sample_multiple(min_samples=2, max_samples=2)), {(1,), ()})
        self.assertEqual(len(cm.records), 1)

        elements = hyperparams.Enumeration(['a', 'b', 'c', 'd', 'e'], 'e')
        list_hyperparameter = hyperparams.SortedList(elements, ('a', 'b', 'c', 'd', 'e'), 5, 5)

        self.assertEqual(list_hyperparameter.get_default(), ('a', 'b', 'c', 'd', 'e'))
        self.assertEqual(list_hyperparameter.sample(45), ('a', 'd', 'd', 'd', 'e'))
        self.assertEqual(list_hyperparameter.get_max_samples(), 126)
        self.assertEqual(list_hyperparameter.sample_multiple(1, 1, 42), (('c', 'd', 'e', 'e', 'e'),))
        self.assertEqual(list_hyperparameter.sample_multiple(0, 1, 42), ())

        self.maxDiff = None

        self.assertEqual(list_hyperparameter.to_simple_structure(), {
            'default': ('a', 'b', 'c', 'd', 'e'),
            'semantic_types': [],
            'structural_type': typing.Sequence[str],
            'type': hyperparams.SortedList,
            'min_size': 5,
            'max_size': 5,
            'elements': {
                'default': 'e',
                'semantic_types': [],
                'structural_type': str,
                'type': hyperparams.Enumeration,
                'values': ['a', 'b', 'c', 'd', 'e'],
            },
            'ascending': True,
        })

        self.assertEqual(list_hyperparameter.value_to_json_structure(list_hyperparameter.get_default()), ['a', 'b', 'c', 'd', 'e'])
        self.assertEqual(list_hyperparameter.value_to_json_structure(list_hyperparameter.sample(45)), ['a', 'd', 'd', 'd', 'e'])

        self.assertEqual(list_hyperparameter.value_from_json_structure(list_hyperparameter.value_to_json_structure(list_hyperparameter.get_default())), list_hyperparameter.get_default())
        self.assertEqual(list_hyperparameter.value_from_json_structure(list_hyperparameter.value_to_json_structure(list_hyperparameter.sample(45))), list_hyperparameter.sample(45))

        with self.assertRaisesRegex(ValueError, 'Value \'.*\' has less than 5 elements'):
            elements = hyperparams.Enumeration(['a', 'b', 1, 2, None], None)
            hyperparams.SortedList(elements, (), 5, 5)

        with self.assertRaisesRegex(ValueError, 'Value \'.*\' is not among values'):
            elements = hyperparams.Enumeration(['a', 'b', 1, 2, None], None)
            hyperparams.SortedList(elements, ('a', 'b', 1, 2, 3), 5, 5)

        list_hyperparameter.contribute_to_class('foo')

        with self.assertRaises(KeyError):
            list_hyperparameter.get_default('foo')

        list_of_supported_metafeatures = ['f1', 'f2', 'f3']
        metafeature = hyperparams.Enumeration(list_of_supported_metafeatures, list_of_supported_metafeatures[0], semantic_types=['https://metadata.datadrivendiscovery.org/types/MetafeatureParameter'])
        list_hyperparameter = hyperparams.SortedList(metafeature, (), 0, 3)

        self.assertEqual(list_hyperparameter.get_default(), ())
        self.assertEqual(list_hyperparameter.sample(42), ('f1', 'f3'))
        self.assertEqual(list_hyperparameter.get_max_samples(), 20)
        self.assertEqual(list_hyperparameter.sample_multiple(0, 3, 42), (('f1', 'f3', 'f3'), ('f1', 'f1', 'f3')))

        self.assertEqual(list_hyperparameter.value_to_json_structure(list_hyperparameter.get_default()), [])
        self.assertEqual(list_hyperparameter.value_to_json_structure(list_hyperparameter.sample(42)), ['f1', 'f3'])

        self.assertEqual(list_hyperparameter.value_from_json_structure(list_hyperparameter.value_to_json_structure(list_hyperparameter.get_default())), list_hyperparameter.get_default())
        self.assertEqual(list_hyperparameter.value_from_json_structure(list_hyperparameter.value_to_json_structure(list_hyperparameter.sample(42))), list_hyperparameter.sample(42))

        list_hyperparameter = hyperparams.SortedList(metafeature, (), 0, 10)

        self.assertEqual(list_hyperparameter.get_default(), ())
        self.assertEqual(list_hyperparameter.sample(42), ('f1', 'f1', 'f1', 'f3', 'f3', 'f3'))
        self.assertEqual(list_hyperparameter.get_max_samples(), 286)
        self.assertEqual(list_hyperparameter.sample_multiple(0, 3, 42), (('f1', 'f3', 'f3'), ('f1', 'f1', 'f2', 'f3', 'f3', 'f3', 'f3')))

        list_hyperparameter = hyperparams.SortedList(hyperparams.Bounded[int](1, None, 1), (1, 1), min_size=2, max_size=2)

        with self.assertLogs(hyperparams.logger) as cm:
            self.assertEqual(list_hyperparameter.sample(42), (1, 1))
        self.assertEqual(len(cm.records), 1)

    def test_sorted_list_with_hyperparams(self):
        elements = hyperparams.Hyperparams.define(OrderedDict(
            range=hyperparams.UniformInt(1, 10, 2),
            enum=hyperparams.Enumeration(['a', 'b', 'c', 'd', 'e'], 'e'),
        ))

        with self.assertRaises(exceptions.NotSupportedError):
            hyperparams.SortedList(elements, (elements(range=2, enum='a'),), 0, 5)

    def test_import_cycle(self):
        # All references to "hyperparams_module" in "d3m.metadata.base" should be lazy:
        # for example, as a string in the typing signature, because we have an import cycle.
        subprocess.run([sys.executable, '-c', 'import d3m.metadata.base'], check=True)
        subprocess.run([sys.executable, '-c', 'import d3m.metadata.hyperparams'], check=True)

    def test_union_float_int(self):
        float_hp = hyperparams.Uniform(1, 10, 2)
        int_hp = hyperparams.UniformInt(1, 10, 2)

        x = float_hp.value_from_json_structure(2.0)
        self.assertEqual(x, 2.0)
        self.assertIs(type(x), float)

        x = float_hp.value_from_json_structure(2)
        self.assertEqual(x, 2.0)
        self.assertIs(type(x), float)

        x = float_hp.value_from_json_structure(2.1)
        self.assertEqual(x, 2.1)
        self.assertIs(type(x), float)

        x = int_hp.value_from_json_structure(2.0)
        self.assertEqual(x, 2)
        self.assertIs(type(x), int)

        x = int_hp.value_from_json_structure(2)
        self.assertEqual(x, 2)
        self.assertIs(type(x), int)

        with self.assertRaises(exceptions.InvalidArgumentTypeError):
            int_hp.value_from_json_structure(2.1)

        hyperparameter = hyperparams.Union(
            OrderedDict(
                float=hyperparams.Uniform(1, 5, 2),
                int=hyperparams.UniformInt(6, 10, 7),
            ),
            'float',
        )

        self.assertEqual(hyperparameter.value_to_json_structure(2.0), {'case': 'float', 'value': 2.0})
        self.assertEqual(hyperparameter.value_to_json_structure(7), {'case': 'int', 'value': 7})

        x = hyperparameter.value_from_json_structure({'case': 'float', 'value': 2.0})
        self.assertEqual(x, 2.0)
        self.assertIs(type(x), float)

        x = hyperparameter.value_from_json_structure({'case': 'float', 'value': 2.1})
        self.assertEqual(x, 2.1)
        self.assertIs(type(x), float)

        x = hyperparameter.value_from_json_structure({'case': 'float', 'value': 2})
        self.assertEqual(x, 2.0)
        self.assertIs(type(x), float)

        x = hyperparameter.value_from_json_structure({'case': 'int', 'value': 7})
        self.assertEqual(x, 7)
        self.assertIs(type(x), int)

        x = hyperparameter.value_from_json_structure({'case': 'int', 'value': 7.0})
        self.assertEqual(x, 7)
        self.assertIs(type(x), int)

    def test_can_serialize_to_json(self):
        # See: https://gitlab.com/datadrivendiscovery/d3m/-/issues/440
        # This is enumeration internally so it tests also that enumeration values are kept as-is when sampled.
        hyperparameter = hyperparams.UniformBool(True)
        sample = hyperparameter.sample()
        self.assertIsInstance(sample, bool)
        x = hyperparameter.value_to_json_structure(sample)
        json.dumps(x)

    def test_sampling_type(self):
        sample = hyperparams.Uniform(0, 10, 5).sample()
        self.assertIsInstance(sample, float)

    def test_numpy_sampling(self):
        class UniformInt64(hyperparams.Bounded[numpy.int64]):
            def __init__(
                self, lower: numpy.int64, upper: numpy.int64, default: numpy.int64, *, lower_inclusive: bool = True, upper_inclusive: bool = False,
                semantic_types: typing.Sequence[str] = None, description: str = None,
            ) -> None:
                if lower is None or upper is None:
                    raise exceptions.InvalidArgumentValueError("Bounds cannot be None.")

                super().__init__(lower, upper, default, lower_inclusive=lower_inclusive, upper_inclusive=upper_inclusive, semantic_types=semantic_types, description=description)

            def _initialize_effective_bounds(self) -> None:
                self._initialize_effective_bounds_int()

                super()._initialize_effective_bounds()

            def sample(self, random_state: numpy.random.RandomState = None) -> int:
                random_state = sklearn_validation.check_random_state(random_state)

                return self.structural_type(random_state.randint(self._effective_lower, self._effective_upper))

            def get_max_samples(self) -> typing.Optional[int]:
                return self._effective_upper - self._effective_lower

        with self.assertRaises(exceptions.InvalidArgumentTypeError):
            UniformInt64(0, 10, 5)

        hyperparameter = UniformInt64(numpy.int64(0), numpy.int64(10), numpy.int64(5))
        sample = hyperparameter.sample()
        self.assertIsInstance(sample, numpy.int64)
        x = hyperparameter.value_to_json_structure(sample)
        json.dumps(x)


if __name__ == '__main__':
    unittest.main()