From 0d962845b5e6fc55865e265df835f41fcd33f4b9 Mon Sep 17 00:00:00 2001 From: lhenry15 Date: Mon, 9 Nov 2020 23:26:52 -0600 Subject: [PATCH 1/3] fix the file name spelling Former-commit-id: 89bd39b2bf949768351d4caf1ee5da9487066be9 [formerly d2d67a7a8fba78459bf2db61504265a3130d0296] [formerly 4f97b7e2a9b8032117cfdf5fb817f1623d5a9cc8 [formerly 0b3ad60f0632707f8ff05c060bde5075ec126c53]] [formerly 90b6af304634415c94d558459cb158fb0a049b38 [formerly eb7a882fb80507e51f77f7fa35de311ca7c060e8] [formerly e90d2eb0855e8f6856087942bbf0dc8e4e91ab15 [formerly 8b75ca84dc533715a7467313cde52a48f217fa13]]] [formerly 75eaf5df27c2569c12d37118c210ea2574a2db3d [formerly a31eab992e589368c07ce945fdb2f03249d27d4f] [formerly a2eca3adffc5bbf8be3b57ea34b3a734b6261445 [formerly d8a265360f49b81ef7ad72bacafcf6b6654170c3]] [formerly c1f31f4fa228b58fb8d2886f8c6e0d11c318b6cf [formerly d3b6d793f0d7ec5d74025138d97bf37ab77f2168] [formerly 2ff18e01531732f95da273962e3fe57ca1c631cf [formerly e3cb047523ad70a77dd00c8daa1fb256472a98bf]]]] [formerly 3d9448676f29aa8bef883d3b9bdd9befb23a67b5 [formerly 01bc098ba5a8313f2da371bf0d91a0b59d1dbf19] [formerly d8d2bfa27e6021f09d2df00192a4e8e46218ec3a [formerly 9d8c10748a903b6d60b17cddf7dc52ce4b770c62]] [formerly 7f514d5caa9d66f874078eb8ca100ebc81337a87 [formerly 323bcdf75ae74db7fd1dfa403c35d0e5995d0388] [formerly 88000d934bff90dccd73c44d962cd61c6f6f0d21 [formerly 85819d7f9633f494f0969ed6cd73a5c3915dfe27]]] [formerly 7f0371611230a64b757bab1504dd9f692ec40fb3 [formerly aba35685faf94358a83c99ace9870a174687973f] [formerly ee5d065d8b07a2e45077967e3347e3f8222360a7 [formerly c2fa01a8d9fc1c67eb6ddb0fdd5cdc94849513ca]] [formerly cf6c97ff0eb88dfcb003b7f2907861c9dafd4966 [formerly 81e0ae5f69a1a36c86a9c576870f3b845741a138] [formerly 80735c237b86c4a86dc0422036260d81d97f70e0 [formerly 9cb940cf10955697bb32bbf14e9a7499e0dfddeb]]]]] [formerly 7c4157b29f34f95c49858fef4fba687f1c764f3c [formerly 0d336cae7c88bbea71afe668a6ca993278241651] [formerly caddc5edaf684322eb5d5940a2fd93c24b57b6ba [formerly 1d16809335a8deabc5f005165a6da65f4bafb309]] [formerly 36de01097f8c2d77ffdb58dda0d85707f435e7b5 [formerly 998a68bd73c4065bbd7d5dd8ec26691c3ada8203] [formerly d60075800251b7f0c40034bf922ba9d5b0936e75 [formerly 0f885146a2e5057d44ed179b731d2bd48984bc8d]]] [formerly 3a74384dfbd034a2f78a25ff716c5b602956c12f [formerly 398236eb3924ec59d4711154698e213fa1b9a6d9] [formerly 0fba0e86acbb827f4d81e3b072e88183667af092 [formerly 833c4353d8bbced1ce571fd8fc19e2575e4b6241]] [formerly 49bf49887efb1fa7ed60c098d4d9b96881ba3b46 [formerly e3dd355c54d716e40d166211c9872fbf876ad202] [formerly 2657c760ffe0d773a5b8ecafb11975984a54e528 [formerly 51ed389d2f6cba20ba27349d658c5debf87fa799]]]] [formerly d056eb315953e7b173a685ce8aba7ff7e4afe657 [formerly 17cfc6b22c4a21ffdd48722344a9a07ef0675b95] [formerly 88870237c3403becdbc3ea1367cbbf5a09b6a059 [formerly f2b4b09bcfb3ac4828dfa17b1be07b139a03420d]] [formerly 9be089bdb983788a33253ce6f812dbb7f2ff08da [formerly bc57e821b2cc389abf7d8aec30fb221db817ced3] [formerly bb9ffc04561c05a444584e5c41038f30e87df993 [formerly 9222d20b46d7ead0df2a6d1841269f9c4860c0f1]]] [formerly 4cec912606a47deb4f1d23b1fccbcf069d500b7d [formerly 90c1b2d875b462a2ff2a1d997d51872d5ea50045] [formerly 9e2b30853b756700b7a128204716c5d60655e315 [formerly 5c3ba414f480af72dccff08f79d668702feb1ad9]] [formerly 5068c85747802fd93320fdcd5ba291cfdcb366db [formerly 887aa721aa4287bbaf4eb6a8949852af11c9a3b8] [formerly 1be79c7bec07fb960289c0c8ca8cc3c1768dcbce [formerly 601a8002a67c5ae7062aa90ce12ca3bb28ec3564]]]]]] Former-commit-id: 806b1917ac5efa4d62efe4db153713a8eb9a4507 [formerly 9e1c9afa8c339392063688f7032c352857545163] [formerly 5a16e9bff88d8ba9590d9c876faf9560e2f74199 [formerly d63e58d7d70753d8b1bc81b478fc6fab59fa5f2c]] [formerly 7ed60881d23bf63720031c4fa1e8af7e0af6fabf [formerly f2c7cf0dbe2f234dba0f4cdeacaf7739571d86da] [formerly ba1c949fdd4fe0e5084469f8fb821ef4bdb8847f [formerly 316939c4e0c9790007573329ab08fcad60616465]]] [formerly 7d39a7d56781c35d2da47b0892702bb8f01e6aa2 [formerly 1ab7bbc52db86d30ea6919bf00c9a7f99bb7ea08] [formerly ed9faf4f77a4e3af7dedf0e594644c1ef31b6cae [formerly 3db7cc05e6e197035737cc0fc741bb24b8b3689e]] [formerly e6055b726da49356af54b7320306c860138c7cc9 [formerly c28e09b7a73e969890d7db921ae446f927a90cbe] [formerly c12c2fc98520fb13e628579e6a130865041a9689 [formerly 8883770c295305d2d7ec34f2e0561379ee9d1ffb]]]] [formerly 14bdc0277d9bfb5ddee844b0da56f5bb272f3585 [formerly 72f77425b61c9d68b484c1dfb80d3b928821cc45] [formerly 17f0eb65ef79547970490f7e084adbe92ce9a2f4 [formerly 4d7d110103aa6a81272fe80f8797a9aef4d13b2e]] [formerly 64eed0dd23d71300dc99b5d32263ac89b954026e [formerly cc716c396b301fee51d194efbcd78d157ac578db] [formerly e97e4445c5c840afa05d35cb7fd854fc3b3d04b7 [formerly 51a71dde983dd904fc290250a88c20539dae5543]]] [formerly dbacc561e26669ce6dee43cffcbb69d9a03b6ea1 [formerly ecfd28dd009f68e277a0c0676669ccf20a98d49b] [formerly bb9dfc8c49d962d36398b0bc076ac1d4393ecda4 [formerly 2336fffa6efb62f2c463f482a2955bec0f065c4e]] [formerly d7ddb8c8a011a84d793100dac8f8beaf31bd142b [formerly d2af0f7c4075c741216108c0d8a959e967ee5def] [formerly 1be79c7bec07fb960289c0c8ca8cc3c1768dcbce]]]] Former-commit-id: 43a4b7e87dbd8cb827b6525fefd4aef80446cb64 [formerly afb8d0479f7e2c89b4ee22603a425cd1a8731b2c] [formerly b3d1bdf6d3ad225bc7ac0d84a7443fecf6dbfc73 [formerly c51df118401db5236349f398df7276cb0111f8c1]] [formerly bf6bd4478c1c94c5eba51be4513214655603f9d1 [formerly 6db7031d8836ff3d0c83a0a2673171b2c3ac0ed4] [formerly 388372c71db7e577c75f4bacf85abebe84db01e7 [formerly 43e8f31ec1858cf967d3084154848cf25555f354]]] [formerly ab2dbfa42b40ea465040e4c3008db76df4e228f3 [formerly 40e825431355ba937f28e4032b89be895d016544] [formerly 01b30b89f7a34feac436d2ebc52da388377bbde0 [formerly f76c98b0ef56d48051fcc78e2aa2ca3c7a93133a]] [formerly e734d9c94f2b5556176e0d79e01cdb6180d67472 [formerly 2bc8d77aa4ddbfa2b128d258ae871c68759967d1] [formerly 90f6961e6b35c6c090c7ae5d66265c20ec2603ae [formerly 08acb5034ad22b41763e6f193efcc0076367bc02]]]] Former-commit-id: 15d94b31b9e09f62543ce8d20f8f38f10ad5b1a2 [formerly 321ac7cc3fb4b2109a49da23c203634e0191de4d] [formerly 572f6aa23ae5d0d90bb8fb0f9e464ef9e5b7f686 [formerly f96bedb57beda0b14e7ce5d0db7f516f75ce2de2]] [formerly c0534c1974ca00e75c0501c8d36530f973cbd6bf [formerly 8b1945233d55d3ad9ceab3288ad8fbf956df433f] [formerly 3d71717ab1ab7d8e45f5128f6895f981b8261bce [formerly 8c029d23f604885b6450c8b5a454677416e59b4f]]] Former-commit-id: 69918bc962457095993c7ee363f61b15424efaab [formerly b03bb371a1945972f30e5c4218e4c7c034798909] [formerly 13a0eb2d4ec1de0d5deb39be8ac584f9b20f17e9 [formerly d8ede2bc43e291dee42ab6e26671fd94436c050c]] Former-commit-id: f2a32a2b42c424b4d834d90efe2b76765fbbc663 [formerly ee29e4877a112c520a5baae384fdc90b30db6961] Former-commit-id: 906cb000fc4941753f039b27de427662a338267a --- tods/data_processing/ContructPredictions.py | 261 -------------------- 1 file changed, 261 deletions(-) delete mode 100644 tods/data_processing/ContructPredictions.py diff --git a/tods/data_processing/ContructPredictions.py b/tods/data_processing/ContructPredictions.py deleted file mode 100644 index ecc89cf..0000000 --- a/tods/data_processing/ContructPredictions.py +++ /dev/null @@ -1,261 +0,0 @@ -import os -import typing - -from d3m import container, utils as d3m_utils -from d3m.metadata import base as metadata_base, hyperparams -from d3m.primitive_interfaces import base, transformer -from d3m.contrib.primitives import compute_scores - -import common_primitives - -__all__ = ('ConstructPredictionsPrimitive',) - -Inputs = container.DataFrame -Outputs = container.DataFrame - - -class Hyperparams(hyperparams.Hyperparams): - use_columns = hyperparams.Set( - elements=hyperparams.Hyperparameter[int](-1), - default=(), - semantic_types=['https://metadata.datadrivendiscovery.org/types/ControlParameter'], - description="A set of column indices to force primitive to operate on. If metadata reconstruction happens, this is used for reference columns." - " If any specified column is not a primary index or a predicted target, it is skipped.", - ) - exclude_columns = hyperparams.Set( - elements=hyperparams.Hyperparameter[int](-1), - default=(), - semantic_types=['https://metadata.datadrivendiscovery.org/types/ControlParameter'], - description="A set of column indices to not operate on. If metadata reconstruction happens, this is used for reference columns. Applicable only if \"use_columns\" is not provided.", - ) - - -class ConstructPredictionsPrimitive(transformer.TransformerPrimitiveBase[Inputs, Outputs, Hyperparams]): - """ - A primitive which takes as input a DataFrame and outputs a DataFrame in Lincoln Labs predictions - format: first column is a d3mIndex column (and other primary index columns, e.g., for object detection - problem), and then predicted targets, each in its column, followed by optional confidence column(s). - - It supports both input columns annotated with semantic types (``https://metadata.datadrivendiscovery.org/types/PrimaryKey``, - ``https://metadata.datadrivendiscovery.org/types/PrimaryMultiKey``, ``https://metadata.datadrivendiscovery.org/types/PredictedTarget``, - ``https://metadata.datadrivendiscovery.org/types/Confidence``), or trying to reconstruct metadata. - This is why the primitive takes also additional input of a reference DataFrame which should - have metadata to help reconstruct missing metadata. If metadata is missing, the primitive - assumes that all ``inputs`` columns are predicted targets, without confidence column(s). - """ - - metadata = metadata_base.PrimitiveMetadata( - { - 'id': '8d38b340-f83f-4877-baaa-162f8e551736', - 'version': '0.3.0', - 'name': "Construct pipeline predictions output", - 'python_path': 'd3m.primitives.tods.data_processing.construct_predictions', - 'source': { - 'name': common_primitives.__author__, - 'contact': 'mailto:mitar.commonprimitives@tnode.com', - 'uris': [ - 'https://gitlab.com/datadrivendiscovery/common-primitives/blob/master/common_primitives/construct_predictions.py', - 'https://gitlab.com/datadrivendiscovery/common-primitives.git', - ], - }, - 'installation': [{ - 'type': metadata_base.PrimitiveInstallationType.PIP, - 'package_uri': 'git+https://gitlab.com/datadrivendiscovery/common-primitives.git@{git_commit}#egg=common_primitives'.format( - git_commit=d3m_utils.current_git_commit(os.path.dirname(__file__)), - ), - }], - 'algorithm_types': [ - metadata_base.PrimitiveAlgorithmType.DATA_CONVERSION, - ], - 'primitive_family': metadata_base.PrimitiveFamily.DATA_TRANSFORMATION, - }, - ) - - def produce(self, *, inputs: Inputs, reference: Inputs, timeout: float = None, iterations: int = None) -> base.CallResult[Outputs]: # type: ignore - index_columns = inputs.metadata.get_index_columns() - target_columns = inputs.metadata.list_columns_with_semantic_types(('https://metadata.datadrivendiscovery.org/types/PredictedTarget',)) - - # Target columns cannot be also index columns. This should not really happen, - # but it could happen with buggy primitives. - target_columns = [target_column for target_column in target_columns if target_column not in index_columns] - - if index_columns and target_columns: - outputs = self._produce_using_semantic_types(inputs, index_columns, target_columns) - else: - outputs = self._produce_reconstruct(inputs, reference, index_columns, target_columns) - - outputs = compute_scores.ComputeScoresPrimitive._encode_columns(outputs) - - # Generally we do not care about column names in DataFrame itself (but use names of columns from metadata), - # but in this case setting column names makes it easier to assure that "to_csv" call produces correct output. - # See: https://gitlab.com/datadrivendiscovery/d3m/issues/147 - column_names = [] - for column_index in range(len(outputs.columns)): - column_names.append(outputs.metadata.query_column(column_index).get('name', outputs.columns[column_index])) - outputs.columns = column_names - - return base.CallResult(outputs) - - def _filter_index_columns(self, inputs_metadata: metadata_base.DataMetadata, index_columns: typing.Sequence[int]) -> typing.Sequence[int]: - if self.hyperparams['use_columns']: - index_columns = [index_column_index for index_column_index in index_columns if index_column_index in self.hyperparams['use_columns']] - if not index_columns: - raise ValueError("No index columns listed in \"use_columns\" hyper-parameter, but index columns are required.") - - else: - index_columns = [index_column_index for index_column_index in index_columns if index_column_index not in self.hyperparams['exclude_columns']] - if not index_columns: - raise ValueError("All index columns listed in \"exclude_columns\" hyper-parameter, but index columns are required.") - - names = [] - for index_column in index_columns: - index_metadata = inputs_metadata.query_column(index_column) - # We do not care about empty strings for names either. - if index_metadata.get('name', None): - names.append(index_metadata['name']) - - if 'd3mIndex' not in names: - raise ValueError("\"d3mIndex\" index column is missing.") - - names_set = set(names) - if len(names) != len(names_set): - duplicate_names = names - for name in names_set: - # Removes just the first occurrence. - duplicate_names.remove(name) - - self.logger.warning("Duplicate names for index columns: %(duplicate_names)s", { - 'duplicate_names': list(set(duplicate_names)), - }) - - return index_columns - - def _get_columns(self, inputs_metadata: metadata_base.DataMetadata, index_columns: typing.Sequence[int], target_columns: typing.Sequence[int]) -> typing.List[int]: - assert index_columns - assert target_columns - - index_columns = self._filter_index_columns(inputs_metadata, index_columns) - - if self.hyperparams['use_columns']: - target_columns = [target_column_index for target_column_index in target_columns if target_column_index in self.hyperparams['use_columns']] - if not target_columns: - raise ValueError("No target columns listed in \"use_columns\" hyper-parameter, but target columns are required.") - - else: - target_columns = [target_column_index for target_column_index in target_columns if target_column_index not in self.hyperparams['exclude_columns']] - if not target_columns: - raise ValueError("All target columns listed in \"exclude_columns\" hyper-parameter, but target columns are required.") - - assert index_columns - assert target_columns - - return list(index_columns) + list(target_columns) - - def _get_confidence_columns(self, inputs_metadata: metadata_base.DataMetadata) -> typing.List[int]: - confidence_columns = inputs_metadata.list_columns_with_semantic_types(('https://metadata.datadrivendiscovery.org/types/Confidence',)) - - if self.hyperparams['use_columns']: - confidence_columns = [confidence_column_index for confidence_column_index in confidence_columns if confidence_column_index in self.hyperparams['use_columns']] - else: - confidence_columns = [confidence_column_index for confidence_column_index in confidence_columns if confidence_column_index not in self.hyperparams['exclude_columns']] - - return confidence_columns - - def _produce_using_semantic_types(self, inputs: Inputs, index_columns: typing.Sequence[int], - target_columns: typing.Sequence[int]) -> Outputs: - confidence_columns = self._get_confidence_columns(inputs.metadata) - - output_columns = self._get_columns(inputs.metadata, index_columns, target_columns) + confidence_columns - - # "get_index_columns" makes sure that "d3mIndex" is always listed first. - # And "select_columns" selects columns in order listed, which then - # always puts "d3mIndex" first. - outputs = inputs.select_columns(output_columns) - - if confidence_columns: - outputs.metadata = self._update_confidence_columns(outputs.metadata, confidence_columns) - - return outputs - - def _update_confidence_columns(self, inputs_metadata: metadata_base.DataMetadata, confidence_columns: typing.Sequence[int]) -> metadata_base.DataMetadata: - output_columns_length = inputs_metadata.query((metadata_base.ALL_ELEMENTS,))['dimension']['length'] - - outputs_metadata = inputs_metadata - - # All confidence columns have to be named "confidence". - for column_index in range(output_columns_length - len(confidence_columns), output_columns_length): - outputs_metadata = outputs_metadata.update((metadata_base.ALL_ELEMENTS, column_index), { - 'name': 'confidence', - }) - - return outputs_metadata - - def _produce_reconstruct(self, inputs: Inputs, reference: Inputs, index_columns: typing.Sequence[int], target_columns: typing.Sequence[int]) -> Outputs: - if not index_columns: - reference_index_columns = reference.metadata.get_index_columns() - - if not reference_index_columns: - raise ValueError("Cannot find an index column in reference data, but index column is required.") - - filtered_index_columns = self._filter_index_columns(reference.metadata, reference_index_columns) - index = reference.select_columns(filtered_index_columns) - else: - filtered_index_columns = self._filter_index_columns(inputs.metadata, index_columns) - index = inputs.select_columns(filtered_index_columns) - - if not target_columns: - if index_columns: - raise ValueError("No target columns in input data, but index column(s) present.") - - # We assume all inputs are targets. - targets = inputs - - # We make sure at least basic metadata is generated correctly, so we regenerate metadata. - targets.metadata = targets.metadata.generate(targets) - - # We set target column names from the reference. We set semantic types. - targets.metadata = self._update_targets_metadata(targets.metadata, self._get_target_names(reference.metadata)) - - else: - targets = inputs.select_columns(target_columns) - - return index.append_columns(targets) - - def multi_produce(self, *, produce_methods: typing.Sequence[str], inputs: Inputs, reference: Inputs, timeout: float = None, iterations: int = None) -> base.MultiCallResult: # type: ignore - return self._multi_produce(produce_methods=produce_methods, timeout=timeout, iterations=iterations, inputs=inputs, reference=reference) - - def fit_multi_produce(self, *, produce_methods: typing.Sequence[str], inputs: Inputs, reference: Inputs, timeout: float = None, iterations: int = None) -> base.MultiCallResult: # type: ignore - return self._fit_multi_produce(produce_methods=produce_methods, timeout=timeout, iterations=iterations, inputs=inputs, reference=reference) - - def _get_target_names(self, metadata: metadata_base.DataMetadata) -> typing.List[typing.Union[str, None]]: - target_names = [] - - for column_index in metadata.list_columns_with_semantic_types(('https://metadata.datadrivendiscovery.org/types/TrueTarget',)): - column_metadata = metadata.query((metadata_base.ALL_ELEMENTS, column_index)) - - target_names.append(column_metadata.get('name', None)) - - return target_names - - def _update_targets_metadata(self, metadata: metadata_base.DataMetadata, target_names: typing.Sequence[typing.Union[str, None]]) -> metadata_base.DataMetadata: - targets_length = metadata.query((metadata_base.ALL_ELEMENTS,))['dimension']['length'] - - if targets_length != len(target_names): - raise ValueError("Not an expected number of target columns to apply names for. Expected {target_names}, provided {targets_length}.".format( - target_names=len(target_names), - targets_length=targets_length, - )) - - for column_index, target_name in enumerate(target_names): - metadata = metadata.add_semantic_type((metadata_base.ALL_ELEMENTS, column_index), 'https://metadata.datadrivendiscovery.org/types/Target') - metadata = metadata.add_semantic_type((metadata_base.ALL_ELEMENTS, column_index), 'https://metadata.datadrivendiscovery.org/types/PredictedTarget') - - # We do not have it, let's skip it and hope for the best. - if target_name is None: - continue - - metadata = metadata.update_column(column_index, { - 'name': target_name, - }) - - return metadata From 154cdfea8e2eb438a4236cc0ac864cbada9a6205 Mon Sep 17 00:00:00 2001 From: lhenry15 Date: Tue, 10 Nov 2020 00:38:26 -0600 Subject: [PATCH 2/3] remove common-primitive dependency, revise primitive_tests Former-commit-id: 160ba3a41db331406a549fa56bae1f9ce026dc81 [formerly 07125c718a9285a8b576fbe8e38a519e2605cda5] [formerly 0f81003685ab707dff628baa9f4b473dcbb7aaa5 [formerly a146528d67cf08c42345bdd49470120508732b54]] [formerly 6162a89a5a16851631ef4ccceb4df8a0d04b01ff [formerly 68cbb6a42e00c1eb23e1941e82e303735de90e7e] [formerly 10cc4b89ce52f1faedb740a8e5da77640ec77849 [formerly e619c2260cf46637a054d722f0459fa0917bbe78]]] [formerly 972f0e1432914bd0939a6d6929aa6fd925b2ad7a [formerly 830ea86dae2fb412fd033f2743914e51c653fb86] [formerly 106d9a0f66a7899fcf928be39906491bcf434dd1 [formerly 3962e5ff748b4ee28ebc18228b3eabd8aa3d61b4]] [formerly bee878ccbfd01caf912c7297bf60649924bb319d [formerly 5adab1c5966aca29f87cba20c1962b243a7ca6f3] [formerly c54a7b24ef34889051807ebeda118d5c444211ac [formerly 0837d0e4462c25f9f98595ee660f8f5061bc331d]]]] [formerly 9fa473030915bccfafce1882df47879e1a76e5cd [formerly d7c7dcd3862fd4b96983a2eac23958506139c154] [formerly f86f25ed8c63375a43a74efc37c01b826ac18ddb [formerly a733830dd8701a9f673e0567a77b45fc88150711]] [formerly e97b53c22a4bfc3aec107c5e4b98967214f057fd [formerly cbf53cf9e6d84c09f10868f4b6a32bd7973c7c52] [formerly f8d34b54a25c9dbdd0b8555dcf8d597ca7174063 [formerly dd5f2e6564402849a28a0f7a2591e8640be1e1db]]] [formerly 03b22869c2832ed2767109236bd60e24539af057 [formerly 58598aa338d335b3b9b452d1f3a1ce27a2378689] [formerly 681ff31fb0566edce835b4df2608e0b66613672d [formerly b7377674fed40cf9416f37a4ff9a0fa6000d57c1]] [formerly 184f35644302785caed1e6fd6ec358794a6d5ed8 [formerly 83d86648d4d7292640dd6931d82e366e6fe2b958] [formerly dace51781aa9a356f5215e06da100a9feb3af641 [formerly 1905e1df459d9383d8e2c637da4165cfb7015f04]]]]] [formerly 4ab9ecea870d9e6ed048fc3a4c67331e8e86724c [formerly b9f81966d516a1e29f9941e2c523b6deb5a402ca] [formerly e84c3e3f44b2e9dac9a1bd8c5f2b110e99a82468 [formerly 8fc402120e852317b93dafe4e8d64b2d411703c7]] [formerly 9764ae63eb6568988be07152457f6a58128a700d [formerly 9966f5263fae5dcca3e2aa035d35537d9dda0f73] [formerly 6583b27757bee26b153704d37f991bea238b4e3b [formerly 169394eb28ae528971ba8efa5b9b1786f6d832f6]]] [formerly 49995d4a88ff269060edca50f81875b83275d9cb [formerly 8e7979866bd05674b5dff98a8ed66c42841acc6d] [formerly 58bea2ac95a2537115d6c589391cb777312cf5cd [formerly 7bcb79d10d5168966dc485ce94cdf4cae5ac4616]] [formerly 3acd90de39338004fe4f853243183abba93bfc30 [formerly 7a92bba6fb042518c2717b0506ce1127a95b66f4] [formerly 252dbf61998179f3ffc6d84a6505f019e56c3f14 [formerly f2edef88147ae669253c6ece6245314a08e61c0f]]]] [formerly a8432b11884152912259e1875bc86790960f9a69 [formerly 7163030193b56d88a613eb1d13866cedd2f0e1b4] [formerly d5dbcc81582b06f7caa0e5a02a9966a04aca43bd [formerly 2c5f43bb9350460644f649a6118ed86d1639a322]] [formerly 525582cafa2181329bbffe0bd7031e2d4696c3c6 [formerly 859c0edef35ab6ddd06cc397c5184b5142b0dd04] [formerly 1141577189f18930d72242ccaf4a88629dae7262 [formerly 96f62655fade6ff11bd7026daf88238d3e1dc159]]] [formerly 714f425f4066c7c6991e5d25427b77dd9db2c99f [formerly f0110252685cecb4b0c38b360bab314e6167182e] [formerly 7b8c741ed1fa11e7846ce34d394878dee83048f6 [formerly 1e1e679c0abf6b9f262c35e335cde4e718f4d6c2]] [formerly 3eb17e347f9ddab025caf79c23b6aa95a5af0b7b [formerly a84f30a47734b9eb2b42309bf09bf5300aead657] [formerly 674df774fc078f6a9059fb113611a4de8fe6b27d [formerly fae7de48dd298648d399d3217ddf25a3116b5065]]]]]] Former-commit-id: a2a07c13835a35a336954fa380cd990092d7956e [formerly 416ec93396dcfb6e45fc1177056aed458f238aa1] [formerly 06fb3d76be784d65b4eaa748a239e22a82a823ef [formerly e76996cbe822540c2a27216f1d42f62cc3b276f8]] [formerly e9ff0e36a3eefe24b0d094e33b17437bfd937d48 [formerly 868c786b3f4240077151555f5866118e8df12b25] [formerly 82b60165d58dd64799f8eab1b9e55c9870c1c206 [formerly 034747c213db980275d99bcc7da5805e86cf510e]]] [formerly babc12a7123c71a4e36e266bc31103bc59209533 [formerly 0a146b28ef80680914e4741cf8d4dfed8c291c54] [formerly 72314677fb739c63f577edea60b2342651062f32 [formerly ff42b8e151b337232baf02ae1ec17ec794f680ac]] [formerly 88b97d74c756e1b4dac38ae2adb054cb98bb9ee2 [formerly 504b50e5478cb5e2741fb781e3bc9fc3a410af9e] [formerly 4b2f6111cecebf6edcb37240b6aa3fb80eb74739 [formerly 802b1a62704bf185026fbefb7f46e735f88848f9]]]] [formerly 0214cb7a3242ba4dce2491aa9f8965e34f75b10d [formerly 5c770977953b7d12c0f34577294b883ae190766a] [formerly b7dd35aac3a2956fd1a210e845981d702f4fbc40 [formerly 11a359eac696c14c84829cd239ba6f523750475f]] [formerly 385645172db92b2b3f4a3bf701b1e693d161eca3 [formerly a793ce79c66f92ac87e3ffe033741cd4429ed0d6] [formerly aa3a910f82a25a9637f570dd322ebd401077736a [formerly d7ba434edcb67dfe944b4cedba65bb56f9361a49]]] [formerly 3c67ce9dcf2394be2387cc5dbb199ebf3c0d33cd [formerly 3291186b5f6ccde9d7ec28d6e7f93bc5f2c23f48] [formerly 8f5992dff364505da7cf18c3ca379caf3e91478f [formerly 9c8a2f8cb60e519fcd1679430a536b3189298cf3]] [formerly cdfbac45be26fc21e54a6b09cf18d3629ef7f6a3 [formerly b4308298d38d8a5ba76b3705970b85c1178fffa7] [formerly 674df774fc078f6a9059fb113611a4de8fe6b27d]]]] Former-commit-id: cee65f78f49406dc34dfcaa58550a97705ad5fb7 [formerly 1332ce886f17a22af4f30b11d7e6119cf99f9a55] [formerly 19fc724aa5d306a37ede95d5daa2564c62026b73 [formerly f03559e414348b7d506d5dcb64b9b8a050f479fa]] [formerly 3f91aa84949831fcb4e8293db4f38d57208c5fea [formerly d8866a78a0ba9231e11d5d3679dccb74da88af30] [formerly 9966d727db4fb2b5d8b172a85440ed6c6bc5a794 [formerly 070aa61aec6f48f9b030c660967b9f2695111d68]]] [formerly a766a963a7f46e8347e52f71bc74dd18e028497d [formerly a244707ec8780d5b338f5285eb3bcf9bf4b7e4fd] [formerly a6beaca84ee9b8300a2c2617e83722f28b864fb3 [formerly 1e24b63e11f72384ad9273e4392beff516336daa]] [formerly 617c1586242f6ac86186817b80d83d8350309131 [formerly 8e22ff2a52663c3ca599223deffbdfa018a4e411] [formerly 83e7b575e7dd9338e53a01e8731a25d75c8491dc [formerly 2516625a718a86a1b3ad70bd6339cb2b7efb9183]]]] Former-commit-id: 7a068f1f72c7515ff90d35453dc5dccd0e40cf71 [formerly 48570c29d118a6b361ccbc2a96ebe5dd6bceffe9] [formerly c86777b7c29bc2662b16a3d5c110acb0ee85e706 [formerly 563d9825b00c32742f4569876c0948e515e87fe1]] [formerly 9a85ff3517a4f4cbbfc3c8ec862ca4212bf62a72 [formerly 7ede1c1e5eeb02ffb9139b082b2c3ac06ac9e184] [formerly f087239b16153e398ea8e5193fd24632d666b1ca [formerly 0aa3080e5fd96cb624af2fc813585f423d5a697e]]] Former-commit-id: d1c39434d1be5a388c55a729d9fd19cfffb4643f [formerly 67be9733bc521fe75593b33d153d2c28622270f5] [formerly ebecc0cbc29aea061a60cd8960e953aeac15724b [formerly 28f45cdda9d5ee7904fc24a8abda74ac8fe2fd6b]] Former-commit-id: 895aa362470ee392ab385050b5b6c997c92d424d [formerly fddc7bee0d52827b009e94d39b21f4db008275b8] Former-commit-id: 5c4c810268965d016fd5b1dd486bceb2843978be --- new_tests/build_ABOD_pipline.py | 70 ----- new_tests/build_AutoEncoder.py | 67 ----- new_tests/build_AutoRegODetect_pipeline.py | 71 ----- new_tests/build_AxiswiseScale_pipline.py | 50 ---- new_tests/build_BKFilter_pipline.py | 44 --- new_tests/build_CBLOF_pipline.py | 51 ---- new_tests/build_CategoricalToBinary.py | 48 ---- new_tests/build_ColumnFilter_pipeline.py | 49 ---- .../build_ContinuityValidation_pipline.py | 43 --- new_tests/build_DeepLog_pipeline.py | 49 ---- new_tests/build_DiscreteCosineTransform.py | 50 ---- .../build_DuplicationValidation_pipline.py | 42 --- new_tests/build_FastFourierTransform.py | 48 ---- new_tests/build_HBOS_pipline.py | 68 ----- new_tests/build_HBOS_score_pipline.py | 71 ----- new_tests/build_HPFilter_pipline.py | 46 ---- new_tests/build_HoltSmoothing_pipline.py | 76 ----- ...HoltWintersExponentialSmoothing_pipline.py | 76 ----- new_tests/build_IsolationForest_pipline.py | 59 ---- new_tests/build_KDiscord_pipeline.py | 71 ----- new_tests/build_KNN_pipline.py | 51 ---- new_tests/build_LODA_pipline.py | 51 ---- new_tests/build_LOF_pipline.py | 51 ---- new_tests/build_LSTMOD_pipline.py | 70 ----- new_tests/build_MatrixProfile_pipeline.py | 49 ---- .../build_MeanAverageTransform_pipline.py | 77 ------ .../build_NonNegativeMatrixFactorization.py | 50 ---- new_tests/build_OCSVM_pipline.py | 51 ---- new_tests/build_PCAODetect_pipeline.py | 71 ----- new_tests/build_PowerTransform_pipline.py | 49 ---- new_tests/build_PyodCOF.py | 51 ---- new_tests/build_QuantileTransform_pipline.py | 49 ---- new_tests/build_RuleBasedFilter_pipline.py | 54 ---- new_tests/build_SOD_pipeline.py | 49 ---- ...uild_SimpleExponentialSmoothing_pipline.py | 76 ----- new_tests/build_Standardize_pipline.py | 49 ---- new_tests/build_TRMF_pipline.py | 44 --- new_tests/build_Telemanom.py | 48 ---- .../build_TimeIntervalTransform_pipeline.py | 86 ------ new_tests/build_TruncatedSVD_pipline.py | 44 --- new_tests/build_VariationalAutoEncoder.py | 67 ----- new_tests/build_WaveletTransform_pipline.py | 64 ----- ...ild_test_detection_algorithm_PyodMoGaal.py | 50 ---- ...ild_test_detection_algorithm_PyodSoGaal.py | 50 ---- ...is_spectral_residual_transform_pipeline.py | 61 ---- ...feature_analysis_statistical_abs_energy.py | 62 ----- ...st_feature_analysis_statistical_abs_sum.py | 62 ----- ...test_feature_analysis_statistical_gmean.py | 62 ----- ...test_feature_analysis_statistical_hmean.py | 62 ----- ...t_feature_analysis_statistical_kurtosis.py | 62 ----- ...st_feature_analysis_statistical_maximum.py | 62 ----- ..._test_feature_analysis_statistical_mean.py | 62 ----- ...t_feature_analysis_statistical_mean_abs.py | 62 ----- ...tatistical_mean_abs_temporal_derivative.py | 62 ----- ...is_statistical_mean_temporal_derivative.py | 62 ----- ...est_feature_analysis_statistical_median.py | 62 ----- ...s_statistical_median_absolute_deviation.py | 63 ----- ...st_feature_analysis_statistical_minimum.py | 62 ----- ..._test_feature_analysis_statistical_skew.py | 62 ----- ...d_test_feature_analysis_statistical_std.py | 62 ----- ...d_test_feature_analysis_statistical_var.py | 62 ----- ..._feature_analysis_statistical_variation.py | 62 ----- ...st_feature_analysis_statistical_vec_sum.py | 62 ----- ...analysis_statistical_willison_amplitude.py | 62 ----- ...ture_analysis_statistical_zero_crossing.py | 62 ----- ..._series_seasonality_trend_decomposition.py | 61 ---- primitive_tests/build_ABOD_pipline.py | 6 +- primitive_tests/build_AutoEncoder.py | 6 +- .../build_AutoRegODetect_pipeline.py | 4 +- .../build_AxiswiseScale_pipline.py | 2 +- primitive_tests/build_BKFilter_pipline.py | 2 +- primitive_tests/build_CBLOF_pipline.py | 2 +- primitive_tests/build_CategoricalToBinary.py | 2 +- .../build_ColumnFilter_pipeline.py | 4 +- .../build_ContinuityValidation_pipline.py | 2 +- primitive_tests/build_DeepLog_pipeline.py | 4 +- .../build_DiscreteCosineTransform.py | 2 +- .../build_DuplicationValidation_pipline.py | 2 +- primitive_tests/build_FastFourierTransform.py | 2 +- primitive_tests/build_HBOS_pipline.py | 6 +- primitive_tests/build_HBOS_score_pipline.py | 6 +- primitive_tests/build_HPFilter_pipline.py | 4 +- .../build_HoltSmoothing_pipline.py | 6 +- ...HoltWintersExponentialSmoothing_pipline.py | 6 +- .../build_IsolationForest_pipline.py | 4 +- primitive_tests/build_KDiscord_pipeline.py | 6 +- primitive_tests/build_KNN_pipline.py | 2 +- primitive_tests/build_LODA_pipline.py | 2 +- primitive_tests/build_LOF_pipline.py | 2 +- primitive_tests/build_LSTMOD_pipline.py | 4 +- .../build_MatrixProfile_pipeline.py | 2 +- .../build_MeanAverageTransform_pipline.py | 6 +- .../build_NonNegativeMatrixFactorization.py | 2 +- primitive_tests/build_OCSVM_pipline.py | 2 +- primitive_tests/build_PCAODetect_pipeline.py | 6 +- .../build_PowerTransform_pipline.py | 2 +- primitive_tests/build_PyodCOF.py | 2 +- .../build_QuantileTransform_pipline.py | 2 +- .../build_RuleBasedFilter_pipline.py | 4 +- primitive_tests/build_SOD_pipeline.py | 2 +- ...uild_SimpleExponentialSmoothing_pipline.py | 6 +- primitive_tests/build_Standardize_pipline.py | 2 +- primitive_tests/build_TRMF_pipline.py | 2 +- primitive_tests/build_Telemanom.py | 2 +- .../build_TimeIntervalTransform_pipeline.py | 8 +- primitive_tests/build_TruncatedSVD_pipline.py | 2 +- .../build_VariationalAutoEncoder.py | 6 +- .../build_WaveletTransform_pipline.py | 2 +- ...ild_test_detection_algorithm_PyodMoGaal.py | 2 +- ...ild_test_detection_algorithm_PyodSoGaal.py | 2 +- ...is_spectral_residual_transform_pipeline.py | 4 +- ...feature_analysis_statistical_abs_energy.py | 2 +- ...st_feature_analysis_statistical_abs_sum.py | 2 +- ...test_feature_analysis_statistical_gmean.py | 2 +- ...test_feature_analysis_statistical_hmean.py | 4 +- ...t_feature_analysis_statistical_kurtosis.py | 4 +- ...st_feature_analysis_statistical_maximum.py | 2 +- ..._test_feature_analysis_statistical_mean.py | 2 +- ...t_feature_analysis_statistical_mean_abs.py | 2 +- ...tatistical_mean_abs_temporal_derivative.py | 2 +- ...is_statistical_mean_temporal_derivative.py | 2 +- ...est_feature_analysis_statistical_median.py | 2 +- ...s_statistical_median_absolute_deviation.py | 2 +- ...st_feature_analysis_statistical_minimum.py | 2 +- ..._test_feature_analysis_statistical_skew.py | 2 +- ...d_test_feature_analysis_statistical_std.py | 2 +- ...d_test_feature_analysis_statistical_var.py | 2 +- ..._feature_analysis_statistical_variation.py | 2 +- ...st_feature_analysis_statistical_vec_sum.py | 2 +- ...analysis_statistical_willison_amplitude.py | 2 +- ...ture_analysis_statistical_zero_crossing.py | 2 +- ..._series_seasonality_trend_decomposition.py | 2 +- replace.sh | 9 + test.sh | 2 +- tested_file.txt | 1 - tods/data_processing/CategoricalToBinary.py | 1 - tods/data_processing/ColumnFilter.py | 3 - tods/data_processing/ColumnParser.py | 5 +- tods/data_processing/ConstructPredictions.py | 260 ++++++++++++++++++ .../ExtractColumnsBySemanticTypes.py | 4 +- tods/data_processing/TimeIntervalTransform.py | 4 +- tods/data_processing/utils.py | 192 +++++++++++++ tods/feature_analysis/AutoCorrelation.py | 1 - .../DiscreteCosineTransform.py | 1 - tods/feature_analysis/FastFourierTransform.py | 1 - tods/feature_analysis/WaveletTransform.py | 3 +- tods/resources/.requirements.txt | 1 - .../timeseries_processing/SKAxiswiseScaler.py | 1 - 148 files changed, 568 insertions(+), 3986 deletions(-) delete mode 100644 new_tests/build_ABOD_pipline.py delete mode 100644 new_tests/build_AutoEncoder.py delete mode 100644 new_tests/build_AutoRegODetect_pipeline.py delete mode 100644 new_tests/build_AxiswiseScale_pipline.py delete mode 100644 new_tests/build_BKFilter_pipline.py delete mode 100644 new_tests/build_CBLOF_pipline.py delete mode 100644 new_tests/build_CategoricalToBinary.py delete mode 100644 new_tests/build_ColumnFilter_pipeline.py delete mode 100644 new_tests/build_ContinuityValidation_pipline.py delete mode 100644 new_tests/build_DeepLog_pipeline.py delete mode 100644 new_tests/build_DiscreteCosineTransform.py delete mode 100644 new_tests/build_DuplicationValidation_pipline.py delete mode 100644 new_tests/build_FastFourierTransform.py delete mode 100644 new_tests/build_HBOS_pipline.py delete mode 100644 new_tests/build_HBOS_score_pipline.py delete mode 100644 new_tests/build_HPFilter_pipline.py delete mode 100644 new_tests/build_HoltSmoothing_pipline.py delete mode 100644 new_tests/build_HoltWintersExponentialSmoothing_pipline.py delete mode 100644 new_tests/build_IsolationForest_pipline.py delete mode 100644 new_tests/build_KDiscord_pipeline.py delete mode 100644 new_tests/build_KNN_pipline.py delete mode 100644 new_tests/build_LODA_pipline.py delete mode 100644 new_tests/build_LOF_pipline.py delete mode 100644 new_tests/build_LSTMOD_pipline.py delete mode 100644 new_tests/build_MatrixProfile_pipeline.py delete mode 100644 new_tests/build_MeanAverageTransform_pipline.py delete mode 100644 new_tests/build_NonNegativeMatrixFactorization.py delete mode 100644 new_tests/build_OCSVM_pipline.py delete mode 100644 new_tests/build_PCAODetect_pipeline.py delete mode 100644 new_tests/build_PowerTransform_pipline.py delete mode 100644 new_tests/build_PyodCOF.py delete mode 100644 new_tests/build_QuantileTransform_pipline.py delete mode 100644 new_tests/build_RuleBasedFilter_pipline.py delete mode 100644 new_tests/build_SOD_pipeline.py delete mode 100644 new_tests/build_SimpleExponentialSmoothing_pipline.py delete mode 100644 new_tests/build_Standardize_pipline.py delete mode 100644 new_tests/build_TRMF_pipline.py delete mode 100644 new_tests/build_Telemanom.py delete mode 100644 new_tests/build_TimeIntervalTransform_pipeline.py delete mode 100644 new_tests/build_TruncatedSVD_pipline.py delete mode 100644 new_tests/build_VariationalAutoEncoder.py delete mode 100644 new_tests/build_WaveletTransform_pipline.py delete mode 100644 new_tests/build_test_detection_algorithm_PyodMoGaal.py delete mode 100644 new_tests/build_test_detection_algorithm_PyodSoGaal.py delete mode 100644 new_tests/build_test_feature_analysis_spectral_residual_transform_pipeline.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_abs_energy.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_abs_sum.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_gmean.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_hmean.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_kurtosis.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_maximum.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_mean.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_mean_abs.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_mean_abs_temporal_derivative.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_mean_temporal_derivative.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_median.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_median_absolute_deviation.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_minimum.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_skew.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_std.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_var.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_variation.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_vec_sum.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_willison_amplitude.py delete mode 100644 new_tests/build_test_feature_analysis_statistical_zero_crossing.py delete mode 100644 new_tests/build_test_time_series_seasonality_trend_decomposition.py create mode 100644 replace.sh delete mode 100644 tested_file.txt create mode 100644 tods/data_processing/ConstructPredictions.py create mode 100644 tods/data_processing/utils.py diff --git a/new_tests/build_ABOD_pipline.py b/new_tests/build_ABOD_pipline.py deleted file mode 100644 index 5faccc2..0000000 --- a/new_tests/build_ABOD_pipline.py +++ /dev/null @@ -1,70 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe')) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/Attribute']) -pipeline_description.add_step(step_2) - -# Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_3.add_output('produce') -step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/TrueTarget']) -pipeline_description.add_step(step_3) - -attributes = 'steps.2.produce' -targets = 'steps.3.produce' - -# Step 4: imputer -step_4 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.impute_missing')) -step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) -step_4.add_output('produce') -pipeline_description.add_step(step_4) - -# Step 5: ABOD -step_5 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.detection_algorithm.pyod_abod')) -step_5.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.4.produce') - -step_5.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_5.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -step_5.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2, 4,)) -step_5.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='replace') - -step_5.add_output('produce') -pipeline_description.add_step(step_5) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.5.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_AutoEncoder.py b/new_tests/build_AutoEncoder.py deleted file mode 100644 index 7482be5..0000000 --- a/new_tests/build_AutoEncoder.py +++ /dev/null @@ -1,67 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe')) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/Attribute']) -pipeline_description.add_step(step_2) - -# Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_3.add_output('produce') -step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/TrueTarget']) -pipeline_description.add_step(step_3) - -attributes = 'steps.2.produce' -targets = 'steps.3.produce' - -# Step 4: imputer -step_4 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.impute_missing')) -step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) -step_4.add_output('produce') -pipeline_description.add_step(step_4) - -# Step 5: auto encoder -step_5 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.detection_algorithm.pyod_ae')) -step_5.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) -step_5.add_output('produce') -pipeline_description.add_step(step_5) - - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.5.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_AutoRegODetect_pipeline.py b/new_tests/build_AutoRegODetect_pipeline.py deleted file mode 100644 index e6debfa..0000000 --- a/new_tests/build_AutoRegODetect_pipeline.py +++ /dev/null @@ -1,71 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import numpy as np - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, data=['https://metadata.datadrivendiscovery.org/types/Attribute']) -pipeline_description.add_step(step_2) - -# # Step 3: Standardization -primitive_3 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(1,2,3,4,5,)) -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='new') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - -# # Step 4: test primitive -primitive_4 = index.get_primitive('d3m.primitives.tods.detection_algorithm.AutoRegODetector') -step_4 = PrimitiveStep(primitive=primitive_4) -step_4.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -step_4.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=10) -# step_4.add_hyperparameter(name='weights', argument_type=ArgumentType.VALUE, data=weights_ndarray) -step_4.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=False) -# step_4.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) # There is sth wrong with multi-dimensional -step_4.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_4.add_hyperparameter(name='return_subseq_inds', argument_type=ArgumentType.VALUE, data=True) -step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.3.produce') -step_4.add_output('produce') -step_4.add_output('produce_score') -pipeline_description.add_step(step_4) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.4.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_AxiswiseScale_pipline.py b/new_tests/build_AxiswiseScale_pipline.py deleted file mode 100644 index 3352f48..0000000 --- a/new_tests/build_AxiswiseScale_pipline.py +++ /dev/null @@ -1,50 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import copy - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: test primitive -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.axiswise_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_BKFilter_pipline.py b/new_tests/build_BKFilter_pipline.py deleted file mode 100644 index c2b306f..0000000 --- a/new_tests/build_BKFilter_pipline.py +++ /dev/null @@ -1,44 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe')) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - - -# Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - - -# Step 2: BKFilter -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.feature_analysis.bk_filter')) -# step_2.add_hyperparameter(name = 'columns_using_method', argument_type=ArgumentType.VALUE, data = 'name') -step_2.add_hyperparameter(name = 'use_semantic_types', argument_type=ArgumentType.VALUE, data = True) -step_2.add_hyperparameter(name = 'use_columns', argument_type=ArgumentType.VALUE, data = (2,3)) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_CBLOF_pipline.py b/new_tests/build_CBLOF_pipline.py deleted file mode 100644 index 2180b6d..0000000 --- a/new_tests/build_CBLOF_pipline.py +++ /dev/null @@ -1,51 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import copy - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: test primitive -primitive_2 = index.get_primitive('d3m.primitives.tods.detection_algorithm.pyod_cblof') - -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,)) # There is sth wrong with multi-dimensional -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_CategoricalToBinary.py b/new_tests/build_CategoricalToBinary.py deleted file mode 100644 index 9f9782e..0000000 --- a/new_tests/build_CategoricalToBinary.py +++ /dev/null @@ -1,48 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: Column Parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: Categorical to Binary -primitive_2 = index.get_primitive('d3m.primitives.tods.data_processing.categorical_to_binary') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(3,)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() diff --git a/new_tests/build_ColumnFilter_pipeline.py b/new_tests/build_ColumnFilter_pipeline.py deleted file mode 100644 index 3dd3be3..0000000 --- a/new_tests/build_ColumnFilter_pipeline.py +++ /dev/null @@ -1,49 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -#Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -primitive_2 = index.get_primitive('d3m.primitives.tods.feature_analysis.auto_correlation') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name="use_semantic_types", argument_type=ArgumentType.VALUE, data = True) -step_2.add_hyperparameter(name="use_columns", argument_type=ArgumentType.VALUE, data = (2, 3)) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -primitive_3 = index.get_primitive('d3m.primitives.tods.data_processing.column_filter') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() diff --git a/new_tests/build_ContinuityValidation_pipline.py b/new_tests/build_ContinuityValidation_pipline.py deleted file mode 100644 index 3b76d84..0000000 --- a/new_tests/build_ContinuityValidation_pipline.py +++ /dev/null @@ -1,43 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe')) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - - -# Step 2: ContinuityValidation -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.continuity_validation')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name = 'continuity_option', argument_type=ArgumentType.VALUE, data = 'imputation') -step_2.add_hyperparameter(name = 'interval', argument_type=ArgumentType.VALUE, data = 0.3) -# Or: -# step_2.add_hyperparameter(name = 'continuity_option', argument_type=ArgumentType.VALUE, data = 'ablation') -pipeline_description.add_step(step_2) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_DeepLog_pipeline.py b/new_tests/build_DeepLog_pipeline.py deleted file mode 100644 index 110c6d3..0000000 --- a/new_tests/build_DeepLog_pipeline.py +++ /dev/null @@ -1,49 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: test primitive -primitive_2 = index.get_primitive('d3m.primitives.tods.detection_algorithm.deeplog') - -step_2 = PrimitiveStep(primitive=primitive_2) -#step_2.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,)) # There is sth wrong with multi-dimensional -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# # Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() diff --git a/new_tests/build_DiscreteCosineTransform.py b/new_tests/build_DiscreteCosineTransform.py deleted file mode 100644 index c052207..0000000 --- a/new_tests/build_DiscreteCosineTransform.py +++ /dev/null @@ -1,50 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: Column Parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - - -# Step 2: Discrete Cosine Transform -primitive_2 = index.get_primitive('d3m.primitives.tods.feature_analysis.discrete_cosine_transform') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_DuplicationValidation_pipline.py b/new_tests/build_DuplicationValidation_pipline.py deleted file mode 100644 index 57673d2..0000000 --- a/new_tests/build_DuplicationValidation_pipline.py +++ /dev/null @@ -1,42 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe')) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - - -# Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - - -# Step 2: DuplicationValidation -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.duplication_validation')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name = 'keep_option', argument_type=ArgumentType.VALUE, data = 'average') # Or: 'first' -pipeline_description.add_step(step_2) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_FastFourierTransform.py b/new_tests/build_FastFourierTransform.py deleted file mode 100644 index 5c7f083..0000000 --- a/new_tests/build_FastFourierTransform.py +++ /dev/null @@ -1,48 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: Column Parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: Fast Fourier Transform -primitive_2 = index.get_primitive('d3m.primitives.tods.feature_analysis.fast_fourier_transform') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() diff --git a/new_tests/build_HBOS_pipline.py b/new_tests/build_HBOS_pipline.py deleted file mode 100644 index b281ba0..0000000 --- a/new_tests/build_HBOS_pipline.py +++ /dev/null @@ -1,68 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe')) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/Attribute']) -pipeline_description.add_step(step_2) - -# Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_3.add_output('produce') -step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/TrueTarget']) -pipeline_description.add_step(step_3) - -attributes = 'steps.2.produce' -targets = 'steps.3.produce' - -# Step 4: imputer -step_4 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.impute_missing')) -step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) -step_4.add_output('produce') -pipeline_description.add_step(step_4) - -# Step 5: HBOS -step_5 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.detection_algorithm.pyod_hbos')) -step_5.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.4.produce') - -step_5.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -# step_5.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') - -step_5.add_output('produce') -pipeline_description.add_step(step_5) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.5.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_HBOS_score_pipline.py b/new_tests/build_HBOS_score_pipline.py deleted file mode 100644 index b389a1e..0000000 --- a/new_tests/build_HBOS_score_pipline.py +++ /dev/null @@ -1,71 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe')) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/Attribute']) -pipeline_description.add_step(step_2) - -# Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_3.add_output('produce') -step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/TrueTarget']) -pipeline_description.add_step(step_3) - -attributes = 'steps.2.produce' -targets = 'steps.3.produce' - -# Step 4: imputer -step_4 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.impute_missing')) -step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) -step_4.add_output('produce') -pipeline_description.add_step(step_4) - -# Step 5: HBOS -step_5 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.detection_algorithm.pyod_hbos')) -step_5.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.4.produce') - -step_5.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -step_5.add_hyperparameter(name='return_subseq_inds', argument_type=ArgumentType.VALUE, data=True) -# step_5.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') - -step_5.add_output('produce_score') -step_5.add_output('produce') -pipeline_description.add_step(step_5) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.5.produce') -# pipeline_description.add_output(name='output score', data_reference='steps.5.produce_score') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_HPFilter_pipline.py b/new_tests/build_HPFilter_pipline.py deleted file mode 100644 index 355c076..0000000 --- a/new_tests/build_HPFilter_pipline.py +++ /dev/null @@ -1,46 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common')) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - - -# Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - - -# Step 2: HPFilter -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.feature_analysis.hp_filter')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') - -step_2.add_hyperparameter(name = 'use_columns', argument_type=ArgumentType.VALUE, data = [2,3,6]) - -step_2.add_hyperparameter(name = 'use_semantic_types', argument_type=ArgumentType.VALUE, data = True) -step_2.add_hyperparameter(name = 'return_result', argument_type=ArgumentType.VALUE, data = 'append') -pipeline_description.add_step(step_2) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_HoltSmoothing_pipline.py b/new_tests/build_HoltSmoothing_pipline.py deleted file mode 100644 index 8f8a31e..0000000 --- a/new_tests/build_HoltSmoothing_pipline.py +++ /dev/null @@ -1,76 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/Attribute']) -pipeline_description.add_step(step_2) - -# Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_3.add_output('produce') -step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/TrueTarget']) -pipeline_description.add_step(step_3) - -attributes = 'steps.2.produce' -targets = 'steps.3.produce' - -# Step 4: imputer -step_4 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.impute_missing')) -step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) -step_4.add_output('produce') -pipeline_description.add_step(step_4) - -# Step 5: holt smoothing -step_5 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.holt_smoothing')) -step_5.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) -step_5.add_hyperparameter(name="exclude_columns", argument_type=ArgumentType.VALUE, data = (2, 3)) -step_5.add_hyperparameter(name="use_semantic_types", argument_type=ArgumentType.VALUE, data = True) -step_5.add_output('produce') -pipeline_description.add_step(step_5) - -# Step 6: isolation forest -#step_6 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.anomaly_detection.isolation_forest.Algorithm')) -#step_6.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.5.produce') -#step_6.add_argument(name='outputs', argument_type=ArgumentType.CONTAINER, data_reference=targets) -#step_6.add_output('produce') -#pipeline_description.add_step(step_6) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.5.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_HoltWintersExponentialSmoothing_pipline.py b/new_tests/build_HoltWintersExponentialSmoothing_pipline.py deleted file mode 100644 index 6ede370..0000000 --- a/new_tests/build_HoltWintersExponentialSmoothing_pipline.py +++ /dev/null @@ -1,76 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/Attribute']) -pipeline_description.add_step(step_2) - -# Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_3.add_output('produce') -step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/TrueTarget']) -pipeline_description.add_step(step_3) - -attributes = 'steps.2.produce' -targets = 'steps.3.produce' - -# Step 4: imputer -step_4 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.impute_missing')) -step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) -step_4.add_output('produce') -pipeline_description.add_step(step_4) - -# Step 5: holt winters exponential smoothing -step_5 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.holt_winters_exponential_smoothing')) -step_5.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) -step_5.add_hyperparameter(name="use_columns", argument_type=ArgumentType.VALUE, data = (2, 3)) -step_5.add_hyperparameter(name="use_semantic_types", argument_type=ArgumentType.VALUE, data = True) -step_5.add_output('produce') -pipeline_description.add_step(step_5) - -# Step 6: isolation forest -#step_6 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.anomaly_detection.isolation_forest.Algorithm')) -#step_6.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.5.produce') -#step_6.add_argument(name='outputs', argument_type=ArgumentType.CONTAINER, data_reference=targets) -#step_6.add_output('produce') -#pipeline_description.add_step(step_6) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.5.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_IsolationForest_pipline.py b/new_tests/build_IsolationForest_pipline.py deleted file mode 100644 index 80923c9..0000000 --- a/new_tests/build_IsolationForest_pipline.py +++ /dev/null @@ -1,59 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import copy - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, data=['https://metadata.datadrivendiscovery.org/types/Attribute']) -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -primitive_3 = index.get_primitive('d3m.primitives.tods.detection_algorithm.pyod_iforest') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -# step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -# step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_hyperparameter(name='return_subseq_inds', argument_type=ArgumentType.VALUE, data=True) -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce_score') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.3.produce_score') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_KDiscord_pipeline.py b/new_tests/build_KDiscord_pipeline.py deleted file mode 100644 index 09d6a7c..0000000 --- a/new_tests/build_KDiscord_pipeline.py +++ /dev/null @@ -1,71 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import numpy as np - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, data=['https://metadata.datadrivendiscovery.org/types/Attribute']) -pipeline_description.add_step(step_2) - -# # Step 3: Standardization -primitive_3 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(1,2,3,4,5,)) -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='new') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - -# # Step 4: test primitive -primitive_4 = index.get_primitive('d3m.primitives.tods.detection_algorithm.KDiscordODetector') -step_4 = PrimitiveStep(primitive=primitive_4) -step_4.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -step_4.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=10) -# step_4.add_hyperparameter(name='weights', argument_type=ArgumentType.VALUE, data=weights_ndarray) -step_4.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=False) -# step_4.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) # There is sth wrong with multi-dimensional -step_4.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_4.add_hyperparameter(name='return_subseq_inds', argument_type=ArgumentType.VALUE, data=True) -step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.3.produce') -step_4.add_output('produce') -step_4.add_output('produce_score') -pipeline_description.add_step(step_4) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.4.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_KNN_pipline.py b/new_tests/build_KNN_pipline.py deleted file mode 100644 index 8b31557..0000000 --- a/new_tests/build_KNN_pipline.py +++ /dev/null @@ -1,51 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import copy - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: test primitive -primitive_2 = index.get_primitive('d3m.primitives.tods.detection_algorithm.pyod_knn') - -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,)) # There is sth wrong with multi-dimensional -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_LODA_pipline.py b/new_tests/build_LODA_pipline.py deleted file mode 100644 index 05b022d..0000000 --- a/new_tests/build_LODA_pipline.py +++ /dev/null @@ -1,51 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import copy - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: test primitive -primitive_2 = index.get_primitive('d3m.primitives.tods.detection_algorithm.pyod_loda') - -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,)) # There is sth wrong with multi-dimensional -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_LOF_pipline.py b/new_tests/build_LOF_pipline.py deleted file mode 100644 index ec444cf..0000000 --- a/new_tests/build_LOF_pipline.py +++ /dev/null @@ -1,51 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import copy - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: test primitive -primitive_2 = index.get_primitive('d3m.primitives.tods.detection_algorithm.pyod_lof') - -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,)) # There is sth wrong with multi-dimensional -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_LSTMOD_pipline.py b/new_tests/build_LSTMOD_pipline.py deleted file mode 100644 index 3575904..0000000 --- a/new_tests/build_LSTMOD_pipline.py +++ /dev/null @@ -1,70 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import numpy as np - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, data=['https://metadata.datadrivendiscovery.org/types/Attribute']) -pipeline_description.add_step(step_2) - -# # Step 2: Standardization -primitive_3 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(1,2,3,4,5,)) -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='new') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - -# # Step 3: test primitive -primitive_4 = index.get_primitive('d3m.primitives.tods.detection_algorithm.LSTMODetector') -step_4 = PrimitiveStep(primitive=primitive_4) -step_4.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -step_4.add_hyperparameter(name='diff_group_method', argument_type=ArgumentType.VALUE, data='average') -step_4.add_hyperparameter(name='feature_dim', argument_type=ArgumentType.VALUE, data=5) -step_4.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=False) -# step_4.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) # There is sth wrong with multi-dimensional -step_4.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_4.add_hyperparameter(name='return_subseq_inds', argument_type=ArgumentType.VALUE, data=True) -step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.3.produce') -step_4.add_output('produce') -pipeline_description.add_step(step_4) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.4.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_MatrixProfile_pipeline.py b/new_tests/build_MatrixProfile_pipeline.py deleted file mode 100644 index 458823e..0000000 --- a/new_tests/build_MatrixProfile_pipeline.py +++ /dev/null @@ -1,49 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: test primitive -primitive_2 = index.get_primitive('d3m.primitives.tods.detection_algorithm.matrix_profile') - -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4)) # There is sth wrong with multi-dimensional -step_2.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=3) # There is sth wrong with multi-dimensional -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# # Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() diff --git a/new_tests/build_MeanAverageTransform_pipline.py b/new_tests/build_MeanAverageTransform_pipline.py deleted file mode 100644 index 43bf392..0000000 --- a/new_tests/build_MeanAverageTransform_pipline.py +++ /dev/null @@ -1,77 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - - -# Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/Attribute']) -pipeline_description.add_step(step_2) - -# Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_3.add_output('produce') -step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/TrueTarget']) -pipeline_description.add_step(step_3) - -attributes = 'steps.2.produce' -targets = 'steps.3.produce' - -# Step 4: imputer -step_4 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.impute_missing')) -step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) -step_4.add_output('produce') -pipeline_description.add_step(step_4) - -# Step 5: mean average transform -step_5 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.moving_average_transform')) -step_5.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) -step_5.add_hyperparameter(name="use_columns", argument_type=ArgumentType.VALUE, data = (2, 3)) -step_5.add_hyperparameter(name="use_semantic_types", argument_type=ArgumentType.VALUE, data = True) -step_5.add_output('produce') -pipeline_description.add_step(step_5) - -# Step 6: isolation forest -#step_6 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.anomaly_detection.isolation_forest.Algorithm')) -#step_6.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.5.produce') -#step_6.add_argument(name='outputs', argument_type=ArgumentType.CONTAINER, data_reference=targets) -#step_6.add_output('produce') -#pipeline_description.add_step(step_6) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.5.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_NonNegativeMatrixFactorization.py b/new_tests/build_NonNegativeMatrixFactorization.py deleted file mode 100644 index 787013c..0000000 --- a/new_tests/build_NonNegativeMatrixFactorization.py +++ /dev/null @@ -1,50 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: Column Parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: Non Negative Matrix Factorization -primitive_2 = index.get_primitive('d3m.primitives.tods.feature_analysis.non_negative_matrix_factorization') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_hyperparameter(name='rank', argument_type=ArgumentType.VALUE, data=5) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_OCSVM_pipline.py b/new_tests/build_OCSVM_pipline.py deleted file mode 100644 index d8cd8c9..0000000 --- a/new_tests/build_OCSVM_pipline.py +++ /dev/null @@ -1,51 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import copy - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: test primitive -primitive_2 = index.get_primitive('d3m.primitives.tods.detection_algorithm.pyod_ocsvm') - -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,)) # There is sth wrong with multi-dimensional -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_PCAODetect_pipeline.py b/new_tests/build_PCAODetect_pipeline.py deleted file mode 100644 index 327cacd..0000000 --- a/new_tests/build_PCAODetect_pipeline.py +++ /dev/null @@ -1,71 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import numpy as np - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, data=['https://metadata.datadrivendiscovery.org/types/Attribute']) -pipeline_description.add_step(step_2) - -# # Step 3: Standardization -primitive_3 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(1,2,3,4,5,)) -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='new') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - -# # Step 4: test primitive -primitive_4 = index.get_primitive('d3m.primitives.tods.detection_algorithm.PCAODetector') -step_4 = PrimitiveStep(primitive=primitive_4) -step_4.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -step_4.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=10) -# step_4.add_hyperparameter(name='weights', argument_type=ArgumentType.VALUE, data=weights_ndarray) -step_4.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=False) -# step_4.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) # There is sth wrong with multi-dimensional -step_4.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_4.add_hyperparameter(name='return_subseq_inds', argument_type=ArgumentType.VALUE, data=True) -step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.3.produce') -step_4.add_output('produce') -step_4.add_output('produce_score') -pipeline_description.add_step(step_4) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.4.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_PowerTransform_pipline.py b/new_tests/build_PowerTransform_pipline.py deleted file mode 100644 index b855dc7..0000000 --- a/new_tests/build_PowerTransform_pipline.py +++ /dev/null @@ -1,49 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import copy - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: test primitive -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.power_transformer') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_PyodCOF.py b/new_tests/build_PyodCOF.py deleted file mode 100644 index fcd0d2b..0000000 --- a/new_tests/build_PyodCOF.py +++ /dev/null @@ -1,51 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import copy - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: test primitive -primitive_2 = index.get_primitive('d3m.primitives.tods.detection_algorithm.pyod_cof') - -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4)) # There is sth wrong with multi-dimensional -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_QuantileTransform_pipline.py b/new_tests/build_QuantileTransform_pipline.py deleted file mode 100644 index f6c4868..0000000 --- a/new_tests/build_QuantileTransform_pipline.py +++ /dev/null @@ -1,49 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import copy - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: test primitive -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.quantile_transformer') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_RuleBasedFilter_pipline.py b/new_tests/build_RuleBasedFilter_pipline.py deleted file mode 100644 index 87a74b9..0000000 --- a/new_tests/build_RuleBasedFilter_pipline.py +++ /dev/null @@ -1,54 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe')) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, data=['https://metadata.datadrivendiscovery.org/types/Attribute']) -pipeline_description.add_step(step_2) - - -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.reinforcement.rule_filter')) -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') - -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2, 4,)) -step_3.add_hyperparameter(name='rule', argument_type=ArgumentType.VALUE, data='#4# % 2 == 0 and #2# <= 0.3') - -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -pipeline_description.add_step(step_3) - - - - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_SOD_pipeline.py b/new_tests/build_SOD_pipeline.py deleted file mode 100644 index e4ed1b3..0000000 --- a/new_tests/build_SOD_pipeline.py +++ /dev/null @@ -1,49 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: test primitive -primitive_2 = index.get_primitive('d3m.primitives.tods.detection_algorithm.pyod_sod') - -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4)) # There is sth wrong with multi-dimensional -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# # Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() diff --git a/new_tests/build_SimpleExponentialSmoothing_pipline.py b/new_tests/build_SimpleExponentialSmoothing_pipline.py deleted file mode 100644 index b33db22..0000000 --- a/new_tests/build_SimpleExponentialSmoothing_pipline.py +++ /dev/null @@ -1,76 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/Attribute']) -pipeline_description.add_step(step_2) - -# Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_3.add_output('produce') -step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/TrueTarget']) -pipeline_description.add_step(step_3) - -attributes = 'steps.2.produce' -targets = 'steps.3.produce' - -# Step 4: imputer -step_4 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.impute_missing')) -step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) -step_4.add_output('produce') -pipeline_description.add_step(step_4) - -# Step 5: simple exponential smoothing -step_5 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.simple_exponential_smoothing')) -step_5.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) -step_5.add_hyperparameter(name="use_columns", argument_type=ArgumentType.VALUE, data = (1,)) -step_5.add_hyperparameter(name="use_semantic_types", argument_type=ArgumentType.VALUE, data = True) -step_5.add_output('produce') -pipeline_description.add_step(step_5) - -# Step 6: isolation forest -#step_6 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.anomaly_detection.isolation_forest.Algorithm')) -#step_6.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.5.produce') -#step_6.add_argument(name='outputs', argument_type=ArgumentType.CONTAINER, data_reference=targets) -#step_6.add_output('produce') -#pipeline_description.add_step(step_6) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.5.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_Standardize_pipline.py b/new_tests/build_Standardize_pipline.py deleted file mode 100644 index 8300d7c..0000000 --- a/new_tests/build_Standardize_pipline.py +++ /dev/null @@ -1,49 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import copy - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: test primitive -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_TRMF_pipline.py b/new_tests/build_TRMF_pipline.py deleted file mode 100644 index 7d7c407..0000000 --- a/new_tests/build_TRMF_pipline.py +++ /dev/null @@ -1,44 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe')) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - - -# Step 2: TRMF -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.feature_analysis.trmf')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') - -step_2.add_hyperparameter(name = 'lags', argument_type=ArgumentType.VALUE, data = [1,2,10,100]) -# step_2.add_hyperparameter(name = 'K', argument_type=ArgumentType.VALUE, data = 3) -# step_2.add_hyperparameter(name = 'use_columns', argument_type=ArgumentType.VALUE, data = (2, 3, 4, 5, 6)) - -pipeline_description.add_step(step_2) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() diff --git a/new_tests/build_Telemanom.py b/new_tests/build_Telemanom.py deleted file mode 100644 index 06a192c..0000000 --- a/new_tests/build_Telemanom.py +++ /dev/null @@ -1,48 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: Column Parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: Fast Fourier Transform -primitive_2 = index.get_primitive('d3m.primitives.tods.detection_algorithm.telemanom') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() diff --git a/new_tests/build_TimeIntervalTransform_pipeline.py b/new_tests/build_TimeIntervalTransform_pipeline.py deleted file mode 100644 index be7990f..0000000 --- a/new_tests/build_TimeIntervalTransform_pipeline.py +++ /dev/null @@ -1,86 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: dataframe transformation -# primitive_1 = index.get_primitive('d3m.primitives.data_transformation.SKPowerTransformer') -# primitive_1 = index.get_primitive('d3m.primitives.data_transformation.SKStandardization') -# primitive_1 = index.get_primitive('d3m.primitives.data_transformation.SKQuantileTransformer') - -#Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -primitive_2 = index.get_primitive('d3m.primitives.tods.data_processing.time_interval_transform') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name="time_interval", argument_type=ArgumentType.VALUE, data = '5T') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) -# -# # Step 2: column_parser -# step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -# step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -# step_2.add_output('produce') -# pipeline_description.add_step(step_2) -# -# -# # Step 3: extract_columns_by_semantic_types(attributes) -# step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -# step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -# step_3.add_output('produce') -# step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, -# data=['https://metadata.datadrivendiscovery.org/types/Attribute']) -# pipeline_description.add_step(step_3) -# -# # Step 4: extract_columns_by_semantic_types(targets) -# step_4 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -# step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -# step_4.add_output('produce') -# step_4.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, -# data=['https://metadata.datadrivendiscovery.org/types/TrueTarget']) -# pipeline_description.add_step(step_4) -# -# attributes = 'steps.3.produce' -# targets = 'steps.4.produce' -# -# # Step 5: imputer -# step_5 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_cleaning.imputer.SKlearn')) -# step_5.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) -# step_5.add_output('produce') -# pipeline_description.add_step(step_5) -# -# # Step 6: random_forest -# step_6 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.regression.random_forest.SKlearn')) -# step_6.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.5.produce') -# step_6.add_argument(name='outputs', argument_type=ArgumentType.CONTAINER, data_reference=targets) -# step_6.add_output('produce') -# pipeline_description.add_step(step_6) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.1.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() diff --git a/new_tests/build_TruncatedSVD_pipline.py b/new_tests/build_TruncatedSVD_pipline.py deleted file mode 100644 index 290f181..0000000 --- a/new_tests/build_TruncatedSVD_pipline.py +++ /dev/null @@ -1,44 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe')) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - - -# Step 2: TruncatedSVD -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.feature_analysis.truncated_svd')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name = 'n_components', argument_type=ArgumentType.VALUE, data = 3) -step_2.add_hyperparameter(name = 'use_columns', argument_type=ArgumentType.VALUE, data = (2, 3, 4, 5, 6)) -step_2.add_hyperparameter(name = 'return_result', argument_type=ArgumentType.VALUE, data = 'append') -step_2.add_hyperparameter(name = 'use_semantic_types', argument_type=ArgumentType.VALUE, data = True) -pipeline_description.add_step(step_2) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_VariationalAutoEncoder.py b/new_tests/build_VariationalAutoEncoder.py deleted file mode 100644 index e585a0a..0000000 --- a/new_tests/build_VariationalAutoEncoder.py +++ /dev/null @@ -1,67 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe')) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/Attribute']) -pipeline_description.add_step(step_2) - -# Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_3.add_output('produce') -step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, - data=['https://metadata.datadrivendiscovery.org/types/TrueTarget']) -pipeline_description.add_step(step_3) - -attributes = 'steps.2.produce' -targets = 'steps.3.produce' - -# Step 4: imputer -step_4 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.impute_missing')) -step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) -step_4.add_output('produce') -pipeline_description.add_step(step_4) - -# Step 5: variatinal auto encoder -step_5 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.detection_algorithm.pyod_vae')) -step_5.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference=attributes) -step_5.add_output('produce') -pipeline_description.add_step(step_5) - - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.5.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_WaveletTransform_pipline.py b/new_tests/build_WaveletTransform_pipline.py deleted file mode 100644 index ee6c766..0000000 --- a/new_tests/build_WaveletTransform_pipline.py +++ /dev/null @@ -1,64 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import copy - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: test WaveletTransform -primitive_2 = index.get_primitive('d3m.primitives.tods.feature_analysis.wavelet_transform') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='wavelet', argument_type=ArgumentType.VALUE, data='db8') -step_2.add_hyperparameter(name='level', argument_type=ArgumentType.VALUE, data=2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='new') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 2: test inverse WaveletTransform -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.wavelet_transform') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='wavelet', argument_type=ArgumentType.VALUE, data='db8') -step_3.add_hyperparameter(name='level', argument_type=ArgumentType.VALUE, data=2) -step_3.add_hyperparameter(name='inverse', argument_type=ArgumentType.VALUE, data=1) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=False) -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='new') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_detection_algorithm_PyodMoGaal.py b/new_tests/build_test_detection_algorithm_PyodMoGaal.py deleted file mode 100644 index 713a2cd..0000000 --- a/new_tests/build_test_detection_algorithm_PyodMoGaal.py +++ /dev/null @@ -1,50 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import copy - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: test primitive -primitive_2 = index.get_primitive('d3m.primitives.tods.detection_algorithm.pyod_mogaal') - -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,)) # There is sth wrong with multi-dimensional -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() diff --git a/new_tests/build_test_detection_algorithm_PyodSoGaal.py b/new_tests/build_test_detection_algorithm_PyodSoGaal.py deleted file mode 100644 index 4caa752..0000000 --- a/new_tests/build_test_detection_algorithm_PyodSoGaal.py +++ /dev/null @@ -1,50 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import copy - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: test primitive -primitive_2 = index.get_primitive('d3m.primitives.tods.detection_algorithm.pyod_sogaal') - -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='contamination', argument_type=ArgumentType.VALUE, data=0.1) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,)) # There is sth wrong with multi-dimensional -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# Final Output -pipeline_description.add_output(name='output predictions', data_reference='steps.2.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) -print(yaml) - -# Or you can output json -#data = pipline_description.to_json() diff --git a/new_tests/build_test_feature_analysis_spectral_residual_transform_pipeline.py b/new_tests/build_test_feature_analysis_spectral_residual_transform_pipeline.py deleted file mode 100644 index 5fbd61e..0000000 --- a/new_tests/build_test_feature_analysis_spectral_residual_transform_pipeline.py +++ /dev/null @@ -1,61 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.spectral_residual_transform') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='avg_filter_dimension', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(8,9,10,11,12)) # There is sth wrong with multi-dimensional -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_abs_energy.py b/new_tests/build_test_feature_analysis_statistical_abs_energy.py deleted file mode 100644 index cb28366..0000000 --- a/new_tests/build_test_feature_analysis_statistical_abs_energy.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_abs_energy') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(8,9,10,11,12)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_abs_sum.py b/new_tests/build_test_feature_analysis_statistical_abs_sum.py deleted file mode 100644 index 91b3d42..0000000 --- a/new_tests/build_test_feature_analysis_statistical_abs_sum.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_abs_sum') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(8,9,10,11,12)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_gmean.py b/new_tests/build_test_feature_analysis_statistical_gmean.py deleted file mode 100644 index 5d54b3c..0000000 --- a/new_tests/build_test_feature_analysis_statistical_gmean.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_g_mean') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(5,6)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_hmean.py b/new_tests/build_test_feature_analysis_statistical_hmean.py deleted file mode 100644 index 01f9dbb..0000000 --- a/new_tests/build_test_feature_analysis_statistical_hmean.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_h_mean') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(5,6)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_kurtosis.py b/new_tests/build_test_feature_analysis_statistical_kurtosis.py deleted file mode 100644 index 3276152..0000000 --- a/new_tests/build_test_feature_analysis_statistical_kurtosis.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_kurtosis') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(5,6)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_maximum.py b/new_tests/build_test_feature_analysis_statistical_maximum.py deleted file mode 100644 index 900a5c1..0000000 --- a/new_tests/build_test_feature_analysis_statistical_maximum.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_maximum') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(5,6)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_mean.py b/new_tests/build_test_feature_analysis_statistical_mean.py deleted file mode 100644 index 29c7bb0..0000000 --- a/new_tests/build_test_feature_analysis_statistical_mean.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_mean') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(5,6)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_mean_abs.py b/new_tests/build_test_feature_analysis_statistical_mean_abs.py deleted file mode 100644 index 6be3c45..0000000 --- a/new_tests/build_test_feature_analysis_statistical_mean_abs.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_mean_abs') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(5,6)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_mean_abs_temporal_derivative.py b/new_tests/build_test_feature_analysis_statistical_mean_abs_temporal_derivative.py deleted file mode 100644 index 15c12aa..0000000 --- a/new_tests/build_test_feature_analysis_statistical_mean_abs_temporal_derivative.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_mean_abs_temporal_derivative') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(5,6)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_mean_temporal_derivative.py b/new_tests/build_test_feature_analysis_statistical_mean_temporal_derivative.py deleted file mode 100644 index d63dddb..0000000 --- a/new_tests/build_test_feature_analysis_statistical_mean_temporal_derivative.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_mean_temporal_derivative') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(5,6)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_median.py b/new_tests/build_test_feature_analysis_statistical_median.py deleted file mode 100644 index cefe002..0000000 --- a/new_tests/build_test_feature_analysis_statistical_median.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_median') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(5,6)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_median_absolute_deviation.py b/new_tests/build_test_feature_analysis_statistical_median_absolute_deviation.py deleted file mode 100644 index 499a877..0000000 --- a/new_tests/build_test_feature_analysis_statistical_median_absolute_deviation.py +++ /dev/null @@ -1,63 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_median_abs_deviation') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(5,6)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_minimum.py b/new_tests/build_test_feature_analysis_statistical_minimum.py deleted file mode 100644 index 01c918d..0000000 --- a/new_tests/build_test_feature_analysis_statistical_minimum.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_minimum') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(5,6)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_skew.py b/new_tests/build_test_feature_analysis_statistical_skew.py deleted file mode 100644 index 7ca113c..0000000 --- a/new_tests/build_test_feature_analysis_statistical_skew.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_skew') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(5,6)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_std.py b/new_tests/build_test_feature_analysis_statistical_std.py deleted file mode 100644 index 66d3180..0000000 --- a/new_tests/build_test_feature_analysis_statistical_std.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_std') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(5,6)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_var.py b/new_tests/build_test_feature_analysis_statistical_var.py deleted file mode 100644 index bd13e96..0000000 --- a/new_tests/build_test_feature_analysis_statistical_var.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_var') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(5,6)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_variation.py b/new_tests/build_test_feature_analysis_statistical_variation.py deleted file mode 100644 index 5292e03..0000000 --- a/new_tests/build_test_feature_analysis_statistical_variation.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_variation') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(5,6)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_vec_sum.py b/new_tests/build_test_feature_analysis_statistical_vec_sum.py deleted file mode 100644 index fa8f99b..0000000 --- a/new_tests/build_test_feature_analysis_statistical_vec_sum.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_vec_sum') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(5,6)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_willison_amplitude.py b/new_tests/build_test_feature_analysis_statistical_willison_amplitude.py deleted file mode 100644 index f750dad..0000000 --- a/new_tests/build_test_feature_analysis_statistical_willison_amplitude.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_willison_amplitude') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=4) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(5,6)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_feature_analysis_statistical_zero_crossing.py b/new_tests/build_test_feature_analysis_statistical_zero_crossing.py deleted file mode 100644 index 1c4efa1..0000000 --- a/new_tests/build_test_feature_analysis_statistical_zero_crossing.py +++ /dev/null @@ -1,62 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.feature_analysis.statistical_zero_crossing') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(9,10)) # There is sth wrong with multi-dimensional -step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/new_tests/build_test_time_series_seasonality_trend_decomposition.py b/new_tests/build_test_time_series_seasonality_trend_decomposition.py deleted file mode 100644 index ab172bf..0000000 --- a/new_tests/build_test_time_series_seasonality_trend_decomposition.py +++ /dev/null @@ -1,61 +0,0 @@ -from d3m import index -from d3m.metadata.base import ArgumentType -from d3m.metadata.pipeline import Pipeline, PrimitiveStep -from d3m.metadata import hyperparams -import copy - -# -> dataset_to_dataframe -> column_parser -> extract_columns_by_semantic_types(attributes) -> imputer -> random_forest -# extract_columns_by_semantic_types(targets) -> ^ - -# Creating pipeline -pipeline_description = Pipeline() -pipeline_description.add_input(name='inputs') - -# Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') -step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') -step_0.add_output('produce') -pipeline_description.add_step(step_0) - -# # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') -step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') -step_1.add_output('produce') -pipeline_description.add_step(step_1) - -# # Step 2: Standardization -primitive_2 = index.get_primitive('d3m.primitives.tods.timeseries_processing.transformation.standard_scaler') -step_2 = PrimitiveStep(primitive=primitive_2) -step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4,5,6)) -step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') -step_2.add_output('produce') -pipeline_description.add_step(step_2) - -# # Step 3: test primitive -# primitive_3 = index.get_primitive('d3m.primitives.anomaly_detection.KNNPrimitive') -primitive_3 = index.get_primitive('d3m.primitives.tods.timeseries_processing.decomposition.time_series_seasonality_trend_decomposition') -step_3 = PrimitiveStep(primitive=primitive_3) -step_3.add_hyperparameter(name='period', argument_type=ArgumentType.VALUE, data=5) -step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) -step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(8,9,10,11,12)) # There is sth wrong with multi-dimensional -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') -step_3.add_output('produce') -pipeline_description.add_step(step_3) - - - -# Final Output -pipeline_description.add_output(name='output', data_reference='steps.3.produce') - -# Output to YAML -yaml = pipeline_description.to_yaml() -with open('pipeline.yml', 'w') as f: - f.write(yaml) - -# Or you can output json -#data = pipline_description.to_json() - diff --git a/primitive_tests/build_ABOD_pipline.py b/primitive_tests/build_ABOD_pipline.py index b48328b..5faccc2 100644 --- a/primitive_tests/build_ABOD_pipline.py +++ b/primitive_tests/build_ABOD_pipline.py @@ -14,13 +14,13 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) # Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') step_2.add_output('produce') step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, @@ -28,7 +28,7 @@ step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALU pipeline_description.add_step(step_2) # Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_3.add_output('produce') step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, diff --git a/primitive_tests/build_AutoEncoder.py b/primitive_tests/build_AutoEncoder.py index 7e05df1..7482be5 100644 --- a/primitive_tests/build_AutoEncoder.py +++ b/primitive_tests/build_AutoEncoder.py @@ -16,13 +16,13 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) # Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') step_2.add_output('produce') step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, @@ -30,7 +30,7 @@ step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALU pipeline_description.add_step(step_2) # Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_3.add_output('produce') step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, diff --git a/primitive_tests/build_AutoRegODetect_pipeline.py b/primitive_tests/build_AutoRegODetect_pipeline.py index 1034b97..e6debfa 100644 --- a/primitive_tests/build_AutoRegODetect_pipeline.py +++ b/primitive_tests/build_AutoRegODetect_pipeline.py @@ -19,14 +19,14 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) # Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') step_2.add_output('produce') step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, data=['https://metadata.datadrivendiscovery.org/types/Attribute']) diff --git a/primitive_tests/build_AxiswiseScale_pipline.py b/primitive_tests/build_AxiswiseScale_pipline.py index 7c51ab2..3352f48 100644 --- a/primitive_tests/build_AxiswiseScale_pipline.py +++ b/primitive_tests/build_AxiswiseScale_pipline.py @@ -19,7 +19,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_BKFilter_pipline.py b/primitive_tests/build_BKFilter_pipline.py index e68fb68..c2b306f 100644 --- a/primitive_tests/build_BKFilter_pipline.py +++ b/primitive_tests/build_BKFilter_pipline.py @@ -15,7 +15,7 @@ pipeline_description.add_step(step_0) # Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) diff --git a/primitive_tests/build_CBLOF_pipline.py b/primitive_tests/build_CBLOF_pipline.py index 302727d..2180b6d 100644 --- a/primitive_tests/build_CBLOF_pipline.py +++ b/primitive_tests/build_CBLOF_pipline.py @@ -19,7 +19,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_CategoricalToBinary.py b/primitive_tests/build_CategoricalToBinary.py index a29399c..9f9782e 100644 --- a/primitive_tests/build_CategoricalToBinary.py +++ b/primitive_tests/build_CategoricalToBinary.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: Column Parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_ColumnFilter_pipeline.py b/primitive_tests/build_ColumnFilter_pipeline.py index 9afa446..7c1aa55 100644 --- a/primitive_tests/build_ColumnFilter_pipeline.py +++ b/primitive_tests/build_ColumnFilter_pipeline.py @@ -10,14 +10,14 @@ pipeline_description = Pipeline() pipeline_description.add_input(name='inputs') # Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common') +primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') step_0 = PrimitiveStep(primitive=primitive_0) step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') step_0.add_output('produce') pipeline_description.add_step(step_0) #Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) diff --git a/primitive_tests/build_ContinuityValidation_pipline.py b/primitive_tests/build_ContinuityValidation_pipline.py index c42310b..3b76d84 100644 --- a/primitive_tests/build_ContinuityValidation_pipline.py +++ b/primitive_tests/build_ContinuityValidation_pipline.py @@ -13,7 +13,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) diff --git a/primitive_tests/build_DeepLog_pipeline.py b/primitive_tests/build_DeepLog_pipeline.py index 32c69d0..0ab8fa3 100644 --- a/primitive_tests/build_DeepLog_pipeline.py +++ b/primitive_tests/build_DeepLog_pipeline.py @@ -11,14 +11,14 @@ pipeline_description = Pipeline() pipeline_description.add_input(name='inputs') # Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common') +primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') step_0 = PrimitiveStep(primitive=primitive_0) step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_DiscreteCosineTransform.py b/primitive_tests/build_DiscreteCosineTransform.py index c3cc52f..c052207 100644 --- a/primitive_tests/build_DiscreteCosineTransform.py +++ b/primitive_tests/build_DiscreteCosineTransform.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: Column Parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_DuplicationValidation_pipline.py b/primitive_tests/build_DuplicationValidation_pipline.py index 6471ac9..57673d2 100644 --- a/primitive_tests/build_DuplicationValidation_pipline.py +++ b/primitive_tests/build_DuplicationValidation_pipline.py @@ -15,7 +15,7 @@ pipeline_description.add_step(step_0) # Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) diff --git a/primitive_tests/build_FastFourierTransform.py b/primitive_tests/build_FastFourierTransform.py index 10a8914..5c7f083 100644 --- a/primitive_tests/build_FastFourierTransform.py +++ b/primitive_tests/build_FastFourierTransform.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: Column Parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_HBOS_pipline.py b/primitive_tests/build_HBOS_pipline.py index 2ebe958..b281ba0 100644 --- a/primitive_tests/build_HBOS_pipline.py +++ b/primitive_tests/build_HBOS_pipline.py @@ -14,13 +14,13 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) # Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') step_2.add_output('produce') step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, @@ -28,7 +28,7 @@ step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALU pipeline_description.add_step(step_2) # Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_3.add_output('produce') step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, diff --git a/primitive_tests/build_HBOS_score_pipline.py b/primitive_tests/build_HBOS_score_pipline.py index 6e3cd4a..b389a1e 100644 --- a/primitive_tests/build_HBOS_score_pipline.py +++ b/primitive_tests/build_HBOS_score_pipline.py @@ -14,13 +14,13 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) # Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') step_2.add_output('produce') step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, @@ -28,7 +28,7 @@ step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALU pipeline_description.add_step(step_2) # Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_3.add_output('produce') step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, diff --git a/primitive_tests/build_HPFilter_pipline.py b/primitive_tests/build_HPFilter_pipline.py index 80754bf..fbf6941 100644 --- a/primitive_tests/build_HPFilter_pipline.py +++ b/primitive_tests/build_HPFilter_pipline.py @@ -8,14 +8,14 @@ pipeline_description = Pipeline() pipeline_description.add_input(name='inputs') # Step 0: dataset_to_dataframe -step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common')) +step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe')) step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) diff --git a/primitive_tests/build_HoltSmoothing_pipline.py b/primitive_tests/build_HoltSmoothing_pipline.py index 374be8b..8f8a31e 100644 --- a/primitive_tests/build_HoltSmoothing_pipline.py +++ b/primitive_tests/build_HoltSmoothing_pipline.py @@ -17,13 +17,13 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) # Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') step_2.add_output('produce') step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, @@ -31,7 +31,7 @@ step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALU pipeline_description.add_step(step_2) # Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_3.add_output('produce') step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, diff --git a/primitive_tests/build_HoltWintersExponentialSmoothing_pipline.py b/primitive_tests/build_HoltWintersExponentialSmoothing_pipline.py index b275408..6ede370 100644 --- a/primitive_tests/build_HoltWintersExponentialSmoothing_pipline.py +++ b/primitive_tests/build_HoltWintersExponentialSmoothing_pipline.py @@ -17,13 +17,13 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) # Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') step_2.add_output('produce') step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, @@ -31,7 +31,7 @@ step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALU pipeline_description.add_step(step_2) # Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_3.add_output('produce') step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, diff --git a/primitive_tests/build_IsolationForest_pipline.py b/primitive_tests/build_IsolationForest_pipline.py index 965343c..80923c9 100644 --- a/primitive_tests/build_IsolationForest_pipline.py +++ b/primitive_tests/build_IsolationForest_pipline.py @@ -19,14 +19,14 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) # Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') step_2.add_output('produce') step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, data=['https://metadata.datadrivendiscovery.org/types/Attribute']) diff --git a/primitive_tests/build_KDiscord_pipeline.py b/primitive_tests/build_KDiscord_pipeline.py index e66bcb1..fc12db9 100644 --- a/primitive_tests/build_KDiscord_pipeline.py +++ b/primitive_tests/build_KDiscord_pipeline.py @@ -12,21 +12,21 @@ pipeline_description = Pipeline() pipeline_description.add_input(name='inputs') # Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common') +primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') step_0 = PrimitiveStep(primitive=primitive_0) step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) # Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') step_2.add_output('produce') step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, data=['https://metadata.datadrivendiscovery.org/types/Attribute']) diff --git a/primitive_tests/build_KNN_pipline.py b/primitive_tests/build_KNN_pipline.py index d188c76..8b31557 100644 --- a/primitive_tests/build_KNN_pipline.py +++ b/primitive_tests/build_KNN_pipline.py @@ -19,7 +19,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_LODA_pipline.py b/primitive_tests/build_LODA_pipline.py index daac9f6..05b022d 100644 --- a/primitive_tests/build_LODA_pipline.py +++ b/primitive_tests/build_LODA_pipline.py @@ -19,7 +19,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_LOF_pipline.py b/primitive_tests/build_LOF_pipline.py index 0c82d13..ec444cf 100644 --- a/primitive_tests/build_LOF_pipline.py +++ b/primitive_tests/build_LOF_pipline.py @@ -19,7 +19,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_LSTMOD_pipline.py b/primitive_tests/build_LSTMOD_pipline.py index 0e9a54b..3575904 100644 --- a/primitive_tests/build_LSTMOD_pipline.py +++ b/primitive_tests/build_LSTMOD_pipline.py @@ -19,14 +19,14 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) # Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') step_2.add_output('produce') step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, data=['https://metadata.datadrivendiscovery.org/types/Attribute']) diff --git a/primitive_tests/build_MatrixProfile_pipeline.py b/primitive_tests/build_MatrixProfile_pipeline.py index 41ceeaa..458823e 100644 --- a/primitive_tests/build_MatrixProfile_pipeline.py +++ b/primitive_tests/build_MatrixProfile_pipeline.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_MeanAverageTransform_pipline.py b/primitive_tests/build_MeanAverageTransform_pipline.py index 223c631..43bf392 100644 --- a/primitive_tests/build_MeanAverageTransform_pipline.py +++ b/primitive_tests/build_MeanAverageTransform_pipline.py @@ -17,14 +17,14 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) # Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') step_2.add_output('produce') step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, @@ -32,7 +32,7 @@ step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALU pipeline_description.add_step(step_2) # Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_3.add_output('produce') step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, diff --git a/primitive_tests/build_NonNegativeMatrixFactorization.py b/primitive_tests/build_NonNegativeMatrixFactorization.py index bdfcd2f..787013c 100644 --- a/primitive_tests/build_NonNegativeMatrixFactorization.py +++ b/primitive_tests/build_NonNegativeMatrixFactorization.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: Column Parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_OCSVM_pipline.py b/primitive_tests/build_OCSVM_pipline.py index 640fb9b..d8cd8c9 100644 --- a/primitive_tests/build_OCSVM_pipline.py +++ b/primitive_tests/build_OCSVM_pipline.py @@ -19,7 +19,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_PCAODetect_pipeline.py b/primitive_tests/build_PCAODetect_pipeline.py index 1e93027..1c7da0e 100644 --- a/primitive_tests/build_PCAODetect_pipeline.py +++ b/primitive_tests/build_PCAODetect_pipeline.py @@ -12,21 +12,21 @@ pipeline_description = Pipeline() pipeline_description.add_input(name='inputs') # Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common') +primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') step_0 = PrimitiveStep(primitive=primitive_0) step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) # Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') step_2.add_output('produce') step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, data=['https://metadata.datadrivendiscovery.org/types/Attribute']) diff --git a/primitive_tests/build_PowerTransform_pipline.py b/primitive_tests/build_PowerTransform_pipline.py index 94ecfbd..b855dc7 100644 --- a/primitive_tests/build_PowerTransform_pipline.py +++ b/primitive_tests/build_PowerTransform_pipline.py @@ -19,7 +19,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_PyodCOF.py b/primitive_tests/build_PyodCOF.py index 7128e6d..fcd0d2b 100644 --- a/primitive_tests/build_PyodCOF.py +++ b/primitive_tests/build_PyodCOF.py @@ -19,7 +19,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_QuantileTransform_pipline.py b/primitive_tests/build_QuantileTransform_pipline.py index 28c4911..f6c4868 100644 --- a/primitive_tests/build_QuantileTransform_pipline.py +++ b/primitive_tests/build_QuantileTransform_pipline.py @@ -19,7 +19,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_RuleBasedFilter_pipline.py b/primitive_tests/build_RuleBasedFilter_pipline.py index 013af40..87a74b9 100644 --- a/primitive_tests/build_RuleBasedFilter_pipline.py +++ b/primitive_tests/build_RuleBasedFilter_pipline.py @@ -14,13 +14,13 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) # Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') step_2.add_output('produce') step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, data=['https://metadata.datadrivendiscovery.org/types/Attribute']) diff --git a/primitive_tests/build_SOD_pipeline.py b/primitive_tests/build_SOD_pipeline.py index 8858cee..e4ed1b3 100644 --- a/primitive_tests/build_SOD_pipeline.py +++ b/primitive_tests/build_SOD_pipeline.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_SimpleExponentialSmoothing_pipline.py b/primitive_tests/build_SimpleExponentialSmoothing_pipline.py index 6f35df3..b33db22 100644 --- a/primitive_tests/build_SimpleExponentialSmoothing_pipline.py +++ b/primitive_tests/build_SimpleExponentialSmoothing_pipline.py @@ -17,13 +17,13 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) # Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') step_2.add_output('produce') step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, @@ -31,7 +31,7 @@ step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALU pipeline_description.add_step(step_2) # Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_3.add_output('produce') step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, diff --git a/primitive_tests/build_Standardize_pipline.py b/primitive_tests/build_Standardize_pipline.py index 50844ea..8300d7c 100644 --- a/primitive_tests/build_Standardize_pipline.py +++ b/primitive_tests/build_Standardize_pipline.py @@ -19,7 +19,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_TRMF_pipline.py b/primitive_tests/build_TRMF_pipline.py index b058cbc..7d7c407 100644 --- a/primitive_tests/build_TRMF_pipline.py +++ b/primitive_tests/build_TRMF_pipline.py @@ -14,7 +14,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) diff --git a/primitive_tests/build_Telemanom.py b/primitive_tests/build_Telemanom.py index afb4bb3..06a192c 100644 --- a/primitive_tests/build_Telemanom.py +++ b/primitive_tests/build_Telemanom.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: Column Parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_TimeIntervalTransform_pipeline.py b/primitive_tests/build_TimeIntervalTransform_pipeline.py index 289171c..be7990f 100644 --- a/primitive_tests/build_TimeIntervalTransform_pipeline.py +++ b/primitive_tests/build_TimeIntervalTransform_pipeline.py @@ -22,7 +22,7 @@ pipeline_description.add_step(step_0) # primitive_1 = index.get_primitive('d3m.primitives.data_transformation.SKQuantileTransformer') #Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) @@ -35,14 +35,14 @@ step_2.add_output('produce') pipeline_description.add_step(step_2) # # # Step 2: column_parser -# step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +# step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) # step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') # step_2.add_output('produce') # pipeline_description.add_step(step_2) # # # # Step 3: extract_columns_by_semantic_types(attributes) -# step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +# step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) # step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') # step_3.add_output('produce') # step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, @@ -50,7 +50,7 @@ pipeline_description.add_step(step_2) # pipeline_description.add_step(step_3) # # # Step 4: extract_columns_by_semantic_types(targets) -# step_4 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +# step_4 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) # step_4.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') # step_4.add_output('produce') # step_4.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, diff --git a/primitive_tests/build_TruncatedSVD_pipline.py b/primitive_tests/build_TruncatedSVD_pipline.py index af14404..290f181 100644 --- a/primitive_tests/build_TruncatedSVD_pipline.py +++ b/primitive_tests/build_TruncatedSVD_pipline.py @@ -14,7 +14,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) diff --git a/primitive_tests/build_VariationalAutoEncoder.py b/primitive_tests/build_VariationalAutoEncoder.py index 01031be..e585a0a 100644 --- a/primitive_tests/build_VariationalAutoEncoder.py +++ b/primitive_tests/build_VariationalAutoEncoder.py @@ -16,13 +16,13 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: column_parser -step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.column_parser.Common')) +step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) # Step 2: extract_columns_by_semantic_types(attributes) -step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') step_2.add_output('produce') step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, @@ -30,7 +30,7 @@ step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALU pipeline_description.add_step(step_2) # Step 3: extract_columns_by_semantic_types(targets) -step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common')) +step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_3.add_output('produce') step_3.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, diff --git a/primitive_tests/build_WaveletTransform_pipline.py b/primitive_tests/build_WaveletTransform_pipline.py index de3aa90..ee6c766 100644 --- a/primitive_tests/build_WaveletTransform_pipline.py +++ b/primitive_tests/build_WaveletTransform_pipline.py @@ -19,7 +19,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_detection_algorithm_PyodMoGaal.py b/primitive_tests/build_test_detection_algorithm_PyodMoGaal.py index a488900..713a2cd 100644 --- a/primitive_tests/build_test_detection_algorithm_PyodMoGaal.py +++ b/primitive_tests/build_test_detection_algorithm_PyodMoGaal.py @@ -19,7 +19,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_detection_algorithm_PyodSoGaal.py b/primitive_tests/build_test_detection_algorithm_PyodSoGaal.py index f14c5a0..4caa752 100644 --- a/primitive_tests/build_test_detection_algorithm_PyodSoGaal.py +++ b/primitive_tests/build_test_detection_algorithm_PyodSoGaal.py @@ -19,7 +19,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_spectral_residual_transform_pipeline.py b/primitive_tests/build_test_feature_analysis_spectral_residual_transform_pipeline.py index d481651..6278460 100644 --- a/primitive_tests/build_test_feature_analysis_spectral_residual_transform_pipeline.py +++ b/primitive_tests/build_test_feature_analysis_spectral_residual_transform_pipeline.py @@ -11,14 +11,14 @@ pipeline_description = Pipeline() pipeline_description.add_input(name='inputs') # Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common') +primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') step_0 = PrimitiveStep(primitive=primitive_0) step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_abs_energy.py b/primitive_tests/build_test_feature_analysis_statistical_abs_energy.py index fc616c4..cb28366 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_abs_energy.py +++ b/primitive_tests/build_test_feature_analysis_statistical_abs_energy.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_abs_sum.py b/primitive_tests/build_test_feature_analysis_statistical_abs_sum.py index 2a5f499..91b3d42 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_abs_sum.py +++ b/primitive_tests/build_test_feature_analysis_statistical_abs_sum.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_gmean.py b/primitive_tests/build_test_feature_analysis_statistical_gmean.py index 05230ff..5d54b3c 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_gmean.py +++ b/primitive_tests/build_test_feature_analysis_statistical_gmean.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_hmean.py b/primitive_tests/build_test_feature_analysis_statistical_hmean.py index 047a6c4..19fa5b2 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_hmean.py +++ b/primitive_tests/build_test_feature_analysis_statistical_hmean.py @@ -11,14 +11,14 @@ pipeline_description = Pipeline() pipeline_description.add_input(name='inputs') # Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common') +primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') step_0 = PrimitiveStep(primitive=primitive_0) step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_kurtosis.py b/primitive_tests/build_test_feature_analysis_statistical_kurtosis.py index ef5ea92..b6b01a7 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_kurtosis.py +++ b/primitive_tests/build_test_feature_analysis_statistical_kurtosis.py @@ -11,14 +11,14 @@ pipeline_description = Pipeline() pipeline_description.add_input(name='inputs') # Step 0: dataset_to_dataframe -primitive_0 = index.get_primitive('d3m.primitives.data_transformation.dataset_to_dataframe.Common') +primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') step_0 = PrimitiveStep(primitive=primitive_0) step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_maximum.py b/primitive_tests/build_test_feature_analysis_statistical_maximum.py index e3bb764..900a5c1 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_maximum.py +++ b/primitive_tests/build_test_feature_analysis_statistical_maximum.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_mean.py b/primitive_tests/build_test_feature_analysis_statistical_mean.py index 5112fec..29c7bb0 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_mean.py +++ b/primitive_tests/build_test_feature_analysis_statistical_mean.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_mean_abs.py b/primitive_tests/build_test_feature_analysis_statistical_mean_abs.py index 4c94ec6..6be3c45 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_mean_abs.py +++ b/primitive_tests/build_test_feature_analysis_statistical_mean_abs.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_mean_abs_temporal_derivative.py b/primitive_tests/build_test_feature_analysis_statistical_mean_abs_temporal_derivative.py index a1bdcdd..15c12aa 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_mean_abs_temporal_derivative.py +++ b/primitive_tests/build_test_feature_analysis_statistical_mean_abs_temporal_derivative.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_mean_temporal_derivative.py b/primitive_tests/build_test_feature_analysis_statistical_mean_temporal_derivative.py index e785efb..d63dddb 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_mean_temporal_derivative.py +++ b/primitive_tests/build_test_feature_analysis_statistical_mean_temporal_derivative.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_median.py b/primitive_tests/build_test_feature_analysis_statistical_median.py index b11b108..cefe002 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_median.py +++ b/primitive_tests/build_test_feature_analysis_statistical_median.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_median_absolute_deviation.py b/primitive_tests/build_test_feature_analysis_statistical_median_absolute_deviation.py index e219ad7..499a877 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_median_absolute_deviation.py +++ b/primitive_tests/build_test_feature_analysis_statistical_median_absolute_deviation.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_minimum.py b/primitive_tests/build_test_feature_analysis_statistical_minimum.py index be8a148..01c918d 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_minimum.py +++ b/primitive_tests/build_test_feature_analysis_statistical_minimum.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_skew.py b/primitive_tests/build_test_feature_analysis_statistical_skew.py index b506f03..7ca113c 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_skew.py +++ b/primitive_tests/build_test_feature_analysis_statistical_skew.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_std.py b/primitive_tests/build_test_feature_analysis_statistical_std.py index f7a61d7..66d3180 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_std.py +++ b/primitive_tests/build_test_feature_analysis_statistical_std.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_var.py b/primitive_tests/build_test_feature_analysis_statistical_var.py index 386c932..bd13e96 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_var.py +++ b/primitive_tests/build_test_feature_analysis_statistical_var.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_variation.py b/primitive_tests/build_test_feature_analysis_statistical_variation.py index bc1f680..5292e03 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_variation.py +++ b/primitive_tests/build_test_feature_analysis_statistical_variation.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_vec_sum.py b/primitive_tests/build_test_feature_analysis_statistical_vec_sum.py index 85f06d2..fa8f99b 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_vec_sum.py +++ b/primitive_tests/build_test_feature_analysis_statistical_vec_sum.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_willison_amplitude.py b/primitive_tests/build_test_feature_analysis_statistical_willison_amplitude.py index 6c578fc..f750dad 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_willison_amplitude.py +++ b/primitive_tests/build_test_feature_analysis_statistical_willison_amplitude.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_feature_analysis_statistical_zero_crossing.py b/primitive_tests/build_test_feature_analysis_statistical_zero_crossing.py index 7a33424..1c4efa1 100644 --- a/primitive_tests/build_test_feature_analysis_statistical_zero_crossing.py +++ b/primitive_tests/build_test_feature_analysis_statistical_zero_crossing.py @@ -18,7 +18,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/primitive_tests/build_test_time_series_seasonality_trend_decomposition.py b/primitive_tests/build_test_time_series_seasonality_trend_decomposition.py index 99168bf..ab172bf 100644 --- a/primitive_tests/build_test_time_series_seasonality_trend_decomposition.py +++ b/primitive_tests/build_test_time_series_seasonality_trend_decomposition.py @@ -19,7 +19,7 @@ step_0.add_output('produce') pipeline_description.add_step(step_0) # # Step 1: column_parser -primitive_1 = index.get_primitive('d3m.primitives.data_transformation.column_parser.Common') +primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') step_1.add_output('produce') diff --git a/replace.sh b/replace.sh new file mode 100644 index 0000000..2b0b986 --- /dev/null +++ b/replace.sh @@ -0,0 +1,9 @@ +# !/bin/bash + +files=$(ls primitive_tests) +for f in $files +do + f_path="./primitive_tests/"$f + save_path="./new_tests/"$f + cat $f_path | sed 's/d3m.primitives.data_transformation.dataset_to_dataframe.Common/d3m.primitives.tods.data_processing.dataset_to_dataframe/g'| sed 's/d3m.primitives.data_transformation.column_parser.Common/d3m.primitives.tods.data_processing.column_parser/g' | sed 's/d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common/d3m.primitives.tods.data_processing.extract_columns_by_semantic_types/g' | sed 's/d3m.primitives.data_transformation.construct_predictions.Common/d3m.primitives.tods.data_processing.construct_predictions/g' > $save_path +done diff --git a/test.sh b/test.sh index 3481545..9316de2 100644 --- a/test.sh +++ b/test.sh @@ -1,6 +1,6 @@ #!/bin/bash -test_scripts=$(ls new_tests) +test_scripts=$(ls primitiver_tests) #test_scripts=$(ls primitive_tests | grep -v -f tested_file.txt) for file in $test_scripts diff --git a/tested_file.txt b/tested_file.txt deleted file mode 100644 index 6672770..0000000 --- a/tested_file.txt +++ /dev/null @@ -1 +0,0 @@ -build_ABOD_pipline.py diff --git a/tods/data_processing/CategoricalToBinary.py b/tods/data_processing/CategoricalToBinary.py index f6dd7d5..1dad198 100644 --- a/tods/data_processing/CategoricalToBinary.py +++ b/tods/data_processing/CategoricalToBinary.py @@ -11,7 +11,6 @@ from d3m.primitive_interfaces import base, transformer from d3m.primitive_interfaces.base import CallResult, DockerContainer -import common_primitives import logging import math diff --git a/tods/data_processing/ColumnFilter.py b/tods/data_processing/ColumnFilter.py index 9c14a75..f16d24f 100644 --- a/tods/data_processing/ColumnFilter.py +++ b/tods/data_processing/ColumnFilter.py @@ -14,11 +14,8 @@ import logging, uuid from scipy import sparse from numpy import ndarray from collections import OrderedDict -from common_primitives import dataframe_utils, utils -from d3m import utils from d3m import container -from d3m.base import utils as base_utils from d3m.exceptions import PrimitiveNotFittedError from d3m.container import DataFrame as d3m_dataframe from d3m.container.numpy import ndarray as d3m_ndarray diff --git a/tods/data_processing/ColumnParser.py b/tods/data_processing/ColumnParser.py index c552ab5..37088a4 100644 --- a/tods/data_processing/ColumnParser.py +++ b/tods/data_processing/ColumnParser.py @@ -9,8 +9,7 @@ from d3m.base import utils as base_utils from d3m.metadata import base as metadata_base, hyperparams from d3m.primitive_interfaces import base, transformer -import common_primitives -from common_primitives import utils +from tods.data_processing import utils __all__ = ('ColumnParserPrimitive',) @@ -98,7 +97,7 @@ class ColumnParserPrimitive(transformer.TransformerPrimitiveBase[Inputs, Outputs 'name': "Parses strings into their types", 'python_path': 'd3m.primitives.tods.data_processing.column_parser', 'source': { - 'name': common_primitives.__author__, + 'name': "DataLab@Texas A&M University", 'contact': 'mailto:mitar.commonprimitives@tnode.com', 'uris': [ 'https://gitlab.com/datadrivendiscovery/common-primitives/blob/master/common_primitives/column_parser.py', diff --git a/tods/data_processing/ConstructPredictions.py b/tods/data_processing/ConstructPredictions.py new file mode 100644 index 0000000..37bc57f --- /dev/null +++ b/tods/data_processing/ConstructPredictions.py @@ -0,0 +1,260 @@ +import os +import typing + +from d3m import container, utils as d3m_utils +from d3m.metadata import base as metadata_base, hyperparams +from d3m.primitive_interfaces import base, transformer +from d3m.contrib.primitives import compute_scores + + +__all__ = ('ConstructPredictionsPrimitive',) + +Inputs = container.DataFrame +Outputs = container.DataFrame + + +class Hyperparams(hyperparams.Hyperparams): + use_columns = hyperparams.Set( + elements=hyperparams.Hyperparameter[int](-1), + default=(), + semantic_types=['https://metadata.datadrivendiscovery.org/types/ControlParameter'], + description="A set of column indices to force primitive to operate on. If metadata reconstruction happens, this is used for reference columns." + " If any specified column is not a primary index or a predicted target, it is skipped.", + ) + exclude_columns = hyperparams.Set( + elements=hyperparams.Hyperparameter[int](-1), + default=(), + semantic_types=['https://metadata.datadrivendiscovery.org/types/ControlParameter'], + description="A set of column indices to not operate on. If metadata reconstruction happens, this is used for reference columns. Applicable only if \"use_columns\" is not provided.", + ) + + +class ConstructPredictionsPrimitive(transformer.TransformerPrimitiveBase[Inputs, Outputs, Hyperparams]): + """ + A primitive which takes as input a DataFrame and outputs a DataFrame in Lincoln Labs predictions + format: first column is a d3mIndex column (and other primary index columns, e.g., for object detection + problem), and then predicted targets, each in its column, followed by optional confidence column(s). + + It supports both input columns annotated with semantic types (``https://metadata.datadrivendiscovery.org/types/PrimaryKey``, + ``https://metadata.datadrivendiscovery.org/types/PrimaryMultiKey``, ``https://metadata.datadrivendiscovery.org/types/PredictedTarget``, + ``https://metadata.datadrivendiscovery.org/types/Confidence``), or trying to reconstruct metadata. + This is why the primitive takes also additional input of a reference DataFrame which should + have metadata to help reconstruct missing metadata. If metadata is missing, the primitive + assumes that all ``inputs`` columns are predicted targets, without confidence column(s). + """ + + metadata = metadata_base.PrimitiveMetadata( + { + 'id': '8d38b340-f83f-4877-baaa-162f8e551736', + 'version': '0.3.0', + 'name': "Construct pipeline predictions output", + 'python_path': 'd3m.primitives.tods.data_processing.construct_predictions', + 'source': { + 'name': "DataLab@Texas A&M University", + 'contact': 'mailto:mitar.commonprimitives@tnode.com', + 'uris': [ + 'https://gitlab.com/datadrivendiscovery/common-primitives/blob/master/common_primitives/construct_predictions.py', + 'https://gitlab.com/datadrivendiscovery/common-primitives.git', + ], + }, + 'installation': [{ + 'type': metadata_base.PrimitiveInstallationType.PIP, + 'package_uri': 'git+https://gitlab.com/datadrivendiscovery/common-primitives.git@{git_commit}#egg=common_primitives'.format( + git_commit=d3m_utils.current_git_commit(os.path.dirname(__file__)), + ), + }], + 'algorithm_types': [ + metadata_base.PrimitiveAlgorithmType.DATA_CONVERSION, + ], + 'primitive_family': metadata_base.PrimitiveFamily.DATA_TRANSFORMATION, + }, + ) + + def produce(self, *, inputs: Inputs, reference: Inputs, timeout: float = None, iterations: int = None) -> base.CallResult[Outputs]: # type: ignore + index_columns = inputs.metadata.get_index_columns() + target_columns = inputs.metadata.list_columns_with_semantic_types(('https://metadata.datadrivendiscovery.org/types/PredictedTarget',)) + + # Target columns cannot be also index columns. This should not really happen, + # but it could happen with buggy primitives. + target_columns = [target_column for target_column in target_columns if target_column not in index_columns] + + if index_columns and target_columns: + outputs = self._produce_using_semantic_types(inputs, index_columns, target_columns) + else: + outputs = self._produce_reconstruct(inputs, reference, index_columns, target_columns) + + outputs = compute_scores.ComputeScoresPrimitive._encode_columns(outputs) + + # Generally we do not care about column names in DataFrame itself (but use names of columns from metadata), + # but in this case setting column names makes it easier to assure that "to_csv" call produces correct output. + # See: https://gitlab.com/datadrivendiscovery/d3m/issues/147 + column_names = [] + for column_index in range(len(outputs.columns)): + column_names.append(outputs.metadata.query_column(column_index).get('name', outputs.columns[column_index])) + outputs.columns = column_names + + return base.CallResult(outputs) + + def _filter_index_columns(self, inputs_metadata: metadata_base.DataMetadata, index_columns: typing.Sequence[int]) -> typing.Sequence[int]: + if self.hyperparams['use_columns']: + index_columns = [index_column_index for index_column_index in index_columns if index_column_index in self.hyperparams['use_columns']] + if not index_columns: + raise ValueError("No index columns listed in \"use_columns\" hyper-parameter, but index columns are required.") + + else: + index_columns = [index_column_index for index_column_index in index_columns if index_column_index not in self.hyperparams['exclude_columns']] + if not index_columns: + raise ValueError("All index columns listed in \"exclude_columns\" hyper-parameter, but index columns are required.") + + names = [] + for index_column in index_columns: + index_metadata = inputs_metadata.query_column(index_column) + # We do not care about empty strings for names either. + if index_metadata.get('name', None): + names.append(index_metadata['name']) + + if 'd3mIndex' not in names: + raise ValueError("\"d3mIndex\" index column is missing.") + + names_set = set(names) + if len(names) != len(names_set): + duplicate_names = names + for name in names_set: + # Removes just the first occurrence. + duplicate_names.remove(name) + + self.logger.warning("Duplicate names for index columns: %(duplicate_names)s", { + 'duplicate_names': list(set(duplicate_names)), + }) + + return index_columns + + def _get_columns(self, inputs_metadata: metadata_base.DataMetadata, index_columns: typing.Sequence[int], target_columns: typing.Sequence[int]) -> typing.List[int]: + assert index_columns + assert target_columns + + index_columns = self._filter_index_columns(inputs_metadata, index_columns) + + if self.hyperparams['use_columns']: + target_columns = [target_column_index for target_column_index in target_columns if target_column_index in self.hyperparams['use_columns']] + if not target_columns: + raise ValueError("No target columns listed in \"use_columns\" hyper-parameter, but target columns are required.") + + else: + target_columns = [target_column_index for target_column_index in target_columns if target_column_index not in self.hyperparams['exclude_columns']] + if not target_columns: + raise ValueError("All target columns listed in \"exclude_columns\" hyper-parameter, but target columns are required.") + + assert index_columns + assert target_columns + + return list(index_columns) + list(target_columns) + + def _get_confidence_columns(self, inputs_metadata: metadata_base.DataMetadata) -> typing.List[int]: + confidence_columns = inputs_metadata.list_columns_with_semantic_types(('https://metadata.datadrivendiscovery.org/types/Confidence',)) + + if self.hyperparams['use_columns']: + confidence_columns = [confidence_column_index for confidence_column_index in confidence_columns if confidence_column_index in self.hyperparams['use_columns']] + else: + confidence_columns = [confidence_column_index for confidence_column_index in confidence_columns if confidence_column_index not in self.hyperparams['exclude_columns']] + + return confidence_columns + + def _produce_using_semantic_types(self, inputs: Inputs, index_columns: typing.Sequence[int], + target_columns: typing.Sequence[int]) -> Outputs: + confidence_columns = self._get_confidence_columns(inputs.metadata) + + output_columns = self._get_columns(inputs.metadata, index_columns, target_columns) + confidence_columns + + # "get_index_columns" makes sure that "d3mIndex" is always listed first. + # And "select_columns" selects columns in order listed, which then + # always puts "d3mIndex" first. + outputs = inputs.select_columns(output_columns) + + if confidence_columns: + outputs.metadata = self._update_confidence_columns(outputs.metadata, confidence_columns) + + return outputs + + def _update_confidence_columns(self, inputs_metadata: metadata_base.DataMetadata, confidence_columns: typing.Sequence[int]) -> metadata_base.DataMetadata: + output_columns_length = inputs_metadata.query((metadata_base.ALL_ELEMENTS,))['dimension']['length'] + + outputs_metadata = inputs_metadata + + # All confidence columns have to be named "confidence". + for column_index in range(output_columns_length - len(confidence_columns), output_columns_length): + outputs_metadata = outputs_metadata.update((metadata_base.ALL_ELEMENTS, column_index), { + 'name': 'confidence', + }) + + return outputs_metadata + + def _produce_reconstruct(self, inputs: Inputs, reference: Inputs, index_columns: typing.Sequence[int], target_columns: typing.Sequence[int]) -> Outputs: + if not index_columns: + reference_index_columns = reference.metadata.get_index_columns() + + if not reference_index_columns: + raise ValueError("Cannot find an index column in reference data, but index column is required.") + + filtered_index_columns = self._filter_index_columns(reference.metadata, reference_index_columns) + index = reference.select_columns(filtered_index_columns) + else: + filtered_index_columns = self._filter_index_columns(inputs.metadata, index_columns) + index = inputs.select_columns(filtered_index_columns) + + if not target_columns: + if index_columns: + raise ValueError("No target columns in input data, but index column(s) present.") + + # We assume all inputs are targets. + targets = inputs + + # We make sure at least basic metadata is generated correctly, so we regenerate metadata. + targets.metadata = targets.metadata.generate(targets) + + # We set target column names from the reference. We set semantic types. + targets.metadata = self._update_targets_metadata(targets.metadata, self._get_target_names(reference.metadata)) + + else: + targets = inputs.select_columns(target_columns) + + return index.append_columns(targets) + + def multi_produce(self, *, produce_methods: typing.Sequence[str], inputs: Inputs, reference: Inputs, timeout: float = None, iterations: int = None) -> base.MultiCallResult: # type: ignore + return self._multi_produce(produce_methods=produce_methods, timeout=timeout, iterations=iterations, inputs=inputs, reference=reference) + + def fit_multi_produce(self, *, produce_methods: typing.Sequence[str], inputs: Inputs, reference: Inputs, timeout: float = None, iterations: int = None) -> base.MultiCallResult: # type: ignore + return self._fit_multi_produce(produce_methods=produce_methods, timeout=timeout, iterations=iterations, inputs=inputs, reference=reference) + + def _get_target_names(self, metadata: metadata_base.DataMetadata) -> typing.List[typing.Union[str, None]]: + target_names = [] + + for column_index in metadata.list_columns_with_semantic_types(('https://metadata.datadrivendiscovery.org/types/TrueTarget',)): + column_metadata = metadata.query((metadata_base.ALL_ELEMENTS, column_index)) + + target_names.append(column_metadata.get('name', None)) + + return target_names + + def _update_targets_metadata(self, metadata: metadata_base.DataMetadata, target_names: typing.Sequence[typing.Union[str, None]]) -> metadata_base.DataMetadata: + targets_length = metadata.query((metadata_base.ALL_ELEMENTS,))['dimension']['length'] + + if targets_length != len(target_names): + raise ValueError("Not an expected number of target columns to apply names for. Expected {target_names}, provided {targets_length}.".format( + target_names=len(target_names), + targets_length=targets_length, + )) + + for column_index, target_name in enumerate(target_names): + metadata = metadata.add_semantic_type((metadata_base.ALL_ELEMENTS, column_index), 'https://metadata.datadrivendiscovery.org/types/Target') + metadata = metadata.add_semantic_type((metadata_base.ALL_ELEMENTS, column_index), 'https://metadata.datadrivendiscovery.org/types/PredictedTarget') + + # We do not have it, let's skip it and hope for the best. + if target_name is None: + continue + + metadata = metadata.update_column(column_index, { + 'name': target_name, + }) + + return metadata diff --git a/tods/data_processing/ExtractColumnsBySemanticTypes.py b/tods/data_processing/ExtractColumnsBySemanticTypes.py index 283cb18..76d9676 100644 --- a/tods/data_processing/ExtractColumnsBySemanticTypes.py +++ b/tods/data_processing/ExtractColumnsBySemanticTypes.py @@ -6,8 +6,6 @@ from d3m.base import utils as base_utils from d3m.metadata import base as metadata_base, hyperparams from d3m.primitive_interfaces import base, transformer -import common_primitives - __all__ = ('ExtractColumnsBySemanticTypesPrimitive',) Inputs = container.DataFrame @@ -74,7 +72,7 @@ class ExtractColumnsBySemanticTypesPrimitive(transformer.TransformerPrimitiveBas 'name': "Extracts columns by semantic type", 'python_path': 'd3m.primitives.tods.data_processing.extract_columns_by_semantic_types', 'source': { - 'name': common_primitives.__author__, + 'name': "DataLab@Texas A&M University", 'contact': 'mailto:mitar.commonprimitives@tnode.com', 'uris': [ 'https://gitlab.com/datadrivendiscovery/common-primitives/blob/master/common_primitives/extract_columns_semantic_types.py', diff --git a/tods/data_processing/TimeIntervalTransform.py b/tods/data_processing/TimeIntervalTransform.py index 3231215..c202bf7 100644 --- a/tods/data_processing/TimeIntervalTransform.py +++ b/tods/data_processing/TimeIntervalTransform.py @@ -6,12 +6,10 @@ import collections import numpy as np import pandas as pd -import common_primitives -from common_primitives import dataframe_utils, utils from datetime import datetime, timezone from d3m.primitive_interfaces import base, transformer -from d3m import container, exceptions, utils as d3m_utils +from d3m import container from d3m.metadata import base as metadata_base, hyperparams diff --git a/tods/data_processing/utils.py b/tods/data_processing/utils.py new file mode 100644 index 0000000..d45dbf0 --- /dev/null +++ b/tods/data_processing/utils.py @@ -0,0 +1,192 @@ +import datetime +import logging +import typing + +import dateutil.parser +import numpy # type: ignore + +from d3m import container, deprecate +from d3m.base import utils as base_utils +from d3m.metadata import base as metadata_base + +logger = logging.getLogger(__name__) + +DEFAULT_DATETIME = datetime.datetime.fromtimestamp(0, datetime.timezone.utc) + + +@deprecate.function(message="it should not be used anymore") +def copy_elements_metadata(source_metadata: metadata_base.Metadata, target_metadata: metadata_base.DataMetadata, from_selector: metadata_base.Selector, + to_selector: metadata_base.Selector = (), *, ignore_all_elements: bool = False, check: bool = True, source: typing.Any = None) -> metadata_base.DataMetadata: + return source_metadata._copy_elements_metadata(target_metadata, list(from_selector), list(to_selector), [], ignore_all_elements) + + +@deprecate.function(message="use Metadata.copy_to method instead") +def copy_metadata(source_metadata: metadata_base.Metadata, target_metadata: metadata_base.DataMetadata, from_selector: metadata_base.Selector, + to_selector: metadata_base.Selector = (), *, ignore_all_elements: bool = False, check: bool = True, source: typing.Any = None) -> metadata_base.DataMetadata: + return source_metadata.copy_to(target_metadata, from_selector, to_selector, ignore_all_elements=ignore_all_elements) + + +@deprecate.function(message="use DataFrame.select_columns method instead") +@deprecate.arguments('source', message="argument ignored") +def select_columns(inputs: container.DataFrame, columns: typing.Sequence[metadata_base.SimpleSelectorSegment], *, + source: typing.Any = None) -> container.DataFrame: + return inputs.select_columns(columns) + + +@deprecate.function(message="use DataMetadata.select_columns method instead") +@deprecate.arguments('source', message="argument ignored") +def select_columns_metadata(inputs_metadata: metadata_base.DataMetadata, columns: typing.Sequence[metadata_base.SimpleSelectorSegment], *, + source: typing.Any = None) -> metadata_base.DataMetadata: + return inputs_metadata.select_columns(columns) + + +@deprecate.function(message="use DataMetadata.list_columns_with_semantic_types method instead") +def list_columns_with_semantic_types(metadata: metadata_base.DataMetadata, semantic_types: typing.Sequence[str], *, + at: metadata_base.Selector = ()) -> typing.Sequence[int]: + return metadata.list_columns_with_semantic_types(semantic_types, at=at) + + +@deprecate.function(message="use DataMetadata.list_columns_with_structural_types method instead") +def list_columns_with_structural_types(metadata: metadata_base.DataMetadata, structural_types: typing.Union[typing.Callable, typing.Sequence[typing.Union[str, type]]], *, + at: metadata_base.Selector = ()) -> typing.Sequence[int]: + return metadata.list_columns_with_structural_types(structural_types, at=at) + + +@deprecate.function(message="use DataFrame.remove_columns method instead") +@deprecate.arguments('source', message="argument ignored") +def remove_columns(inputs: container.DataFrame, column_indices: typing.Sequence[int], *, source: typing.Any = None) -> container.DataFrame: + return inputs.remove_columns(column_indices) + + +@deprecate.function(message="use DataMetadata.remove_columns method instead") +@deprecate.arguments('source', message="argument ignored") +def remove_columns_metadata(inputs_metadata: metadata_base.DataMetadata, column_indices: typing.Sequence[int], *, source: typing.Any = None) -> metadata_base.DataMetadata: + return inputs_metadata.remove_columns(column_indices) + + +@deprecate.function(message="use DataFrame.append_columns method instead") +@deprecate.arguments('source', message="argument ignored") +def append_columns(left: container.DataFrame, right: container.DataFrame, *, use_right_metadata: bool = False, source: typing.Any = None) -> container.DataFrame: + return left.append_columns(right, use_right_metadata=use_right_metadata) + + +@deprecate.function(message="use DataMetadata.append_columns method instead") +@deprecate.arguments('source', message="argument ignored") +def append_columns_metadata(left_metadata: metadata_base.DataMetadata, right_metadata: metadata_base.DataMetadata, use_right_metadata: bool = False, source: typing.Any = None) -> metadata_base.DataMetadata: + return left_metadata.append_columns(right_metadata, use_right_metadata=use_right_metadata) + + +@deprecate.function(message="use DataFrame.insert_columns method instead") +@deprecate.arguments('source', message="argument ignored") +def insert_columns(inputs: container.DataFrame, columns: container.DataFrame, at_column_index: int, *, source: typing.Any = None) -> container.DataFrame: + return inputs.insert_columns(columns, at_column_index) + + +@deprecate.function(message="use DataMetadata.insert_columns method instead") +@deprecate.arguments('source', message="argument ignored") +def insert_columns_metadata(inputs_metadata: metadata_base.DataMetadata, columns_metadata: metadata_base.DataMetadata, at_column_index: int, *, source: typing.Any = None) -> metadata_base.DataMetadata: + return inputs_metadata.insert_columns(columns_metadata, at_column_index) + + +@deprecate.function(message="use DataFrame.replace_columns method instead") +@deprecate.arguments('source', message="argument ignored") +def replace_columns(inputs: container.DataFrame, columns: container.DataFrame, column_indices: typing.Sequence[int], *, copy: bool = True, source: typing.Any = None) -> container.DataFrame: + return inputs.replace_columns(columns, column_indices, copy=copy) + + +@deprecate.function(message="use DataMetadata.replace_columns method instead") +@deprecate.arguments('source', message="argument ignored") +def replace_columns_metadata(inputs_metadata: metadata_base.DataMetadata, columns_metadata: metadata_base.DataMetadata, column_indices: typing.Sequence[int], *, source: typing.Any = None) -> metadata_base.DataMetadata: + return inputs_metadata.replace_columns(columns_metadata, column_indices) + + +@deprecate.function(message="use DataMetadata.get_index_columns method instead") +def get_index_columns(metadata: metadata_base.DataMetadata, *, at: metadata_base.Selector = ()) -> typing.Sequence[int]: + return metadata.get_index_columns(at=at) + + +@deprecate.function(message="use DataFrame.horizontal_concat method instead") +@deprecate.arguments('source', message="argument ignored") +def horizontal_concat(left: container.DataFrame, right: container.DataFrame, *, use_index: bool = True, + remove_second_index: bool = True, use_right_metadata: bool = False, source: typing.Any = None) -> container.DataFrame: + return left.horizontal_concat(right, use_index=use_index, remove_second_index=remove_second_index, use_right_metadata=use_right_metadata) + + +@deprecate.function(message="use DataMetadata.horizontal_concat method instead") +@deprecate.arguments('source', message="argument ignored") +def horizontal_concat_metadata(left_metadata: metadata_base.DataMetadata, right_metadata: metadata_base.DataMetadata, *, use_index: bool = True, + remove_second_index: bool = True, use_right_metadata: bool = False, source: typing.Any = None) -> metadata_base.DataMetadata: + return left_metadata.horizontal_concat(right_metadata, use_index=use_index, remove_second_index=remove_second_index, use_right_metadata=use_right_metadata) + + +@deprecate.function(message="use d3m.base.utils.get_columns_to_use function instead") +def get_columns_to_use(metadata: metadata_base.DataMetadata, use_columns: typing.Sequence[int], exclude_columns: typing.Sequence[int], + can_use_column: typing.Callable) -> typing.Tuple[typing.List[int], typing.List[int]]: + return base_utils.get_columns_to_use(metadata, use_columns, exclude_columns, can_use_column) + + +@deprecate.function(message="use d3m.base.utils.combine_columns function instead") +@deprecate.arguments('source', message="argument ignored") +def combine_columns(return_result: str, add_index_columns: bool, inputs: container.DataFrame, column_indices: typing.Sequence[int], + columns_list: typing.Sequence[container.DataFrame], *, source: typing.Any = None) -> container.DataFrame: + return base_utils.combine_columns(inputs, column_indices, columns_list, return_result=return_result, add_index_columns=add_index_columns) + + +@deprecate.function(message="use d3m.base.utils.combine_columns_metadata function instead") +@deprecate.arguments('source', message="argument ignored") +def combine_columns_metadata(return_result: str, add_index_columns: bool, inputs_metadata: metadata_base.DataMetadata, column_indices: typing.Sequence[int], + columns_metadata_list: typing.Sequence[metadata_base.DataMetadata], *, source: typing.Any = None) -> metadata_base.DataMetadata: + return base_utils.combine_columns_metadata(inputs_metadata, column_indices, columns_metadata_list, return_result=return_result, add_index_columns=add_index_columns) + + +@deprecate.function(message="use DataMetadata.set_table_metadata method instead") +@deprecate.arguments('source', message="argument ignored") +def set_table_metadata(inputs_metadata: metadata_base.DataMetadata, *, at: metadata_base.Selector = (), source: typing.Any = None) -> metadata_base.DataMetadata: + return inputs_metadata.set_table_metadata(at=at) + + +@deprecate.function(message="use DataMetadata.get_column_index_from_column_name method instead") +def get_column_index_from_column_name(inputs_metadata: metadata_base.DataMetadata, column_name: str, *, at: metadata_base.Selector = ()) -> int: + return inputs_metadata.get_column_index_from_column_name(column_name, at=at) + + +@deprecate.function(message="use Dataset.get_relations_graph method instead") +def build_relation_graph(dataset: container.Dataset) -> typing.Dict[str, typing.List[typing.Tuple[str, bool, int, int, typing.Dict]]]: + return dataset.get_relations_graph() + + +@deprecate.function(message="use d3m.base.utils.get_tabular_resource function instead") +def get_tabular_resource(dataset: container.Dataset, resource_id: typing.Optional[str], *, + pick_entry_point: bool = True, pick_one: bool = True, has_hyperparameter: bool = True) -> typing.Tuple[str, container.DataFrame]: + return base_utils.get_tabular_resource(dataset, resource_id, pick_entry_point=pick_entry_point, pick_one=pick_one, has_hyperparameter=has_hyperparameter) + + +@deprecate.function(message="use d3m.base.utils.get_tabular_resource_metadata function instead") +def get_tabular_resource_metadata(dataset_metadata: metadata_base.DataMetadata, resource_id: typing.Optional[metadata_base.SelectorSegment], *, + pick_entry_point: bool = True, pick_one: bool = True) -> metadata_base.SelectorSegment: + return base_utils.get_tabular_resource_metadata(dataset_metadata, resource_id, pick_entry_point=pick_entry_point, pick_one=pick_one) + + +@deprecate.function(message="use Dataset.select_rows method instead") +@deprecate.arguments('source', message="argument ignored") +def cut_dataset(dataset: container.Dataset, row_indices_to_keep: typing.Mapping[str, typing.Sequence[int]], *, + source: typing.Any = None) -> container.Dataset: + return dataset.select_rows(row_indices_to_keep) + + +def parse_datetime(value: str, *, fuzzy: bool = True) -> typing.Optional[datetime.datetime]: + try: + return dateutil.parser.parse(value, default=DEFAULT_DATETIME, fuzzy=fuzzy) + except (ValueError, OverflowError, TypeError): + return None + + +def parse_datetime_to_float(value: str, *, fuzzy: bool = True) -> float: + try: + parsed = parse_datetime(value, fuzzy=fuzzy) + if parsed is None: + return numpy.nan + else: + return parsed.timestamp() + except (ValueError, OverflowError, TypeError): + return numpy.nan diff --git a/tods/feature_analysis/AutoCorrelation.py b/tods/feature_analysis/AutoCorrelation.py index 021996d..2fe2bae 100644 --- a/tods/feature_analysis/AutoCorrelation.py +++ b/tods/feature_analysis/AutoCorrelation.py @@ -14,7 +14,6 @@ import logging, uuid from scipy import sparse from numpy import ndarray from collections import OrderedDict -from common_primitives import dataframe_utils, utils from d3m import utils from d3m import container diff --git a/tods/feature_analysis/DiscreteCosineTransform.py b/tods/feature_analysis/DiscreteCosineTransform.py index 77387ec..88b141e 100644 --- a/tods/feature_analysis/DiscreteCosineTransform.py +++ b/tods/feature_analysis/DiscreteCosineTransform.py @@ -8,7 +8,6 @@ from d3m.base import utils as base_utils from d3m.metadata import base as metadata_base, hyperparams from d3m.primitive_interfaces import base, transformer -import common_primitives import logging import math from scipy.fft import dct diff --git a/tods/feature_analysis/FastFourierTransform.py b/tods/feature_analysis/FastFourierTransform.py index 2ce4d69..a1bfdea 100644 --- a/tods/feature_analysis/FastFourierTransform.py +++ b/tods/feature_analysis/FastFourierTransform.py @@ -8,7 +8,6 @@ from d3m.base import utils as base_utils from d3m.metadata import base as metadata_base, hyperparams from d3m.primitive_interfaces import base, transformer -import common_primitives import logging from cmath import polar from scipy.fft import fft diff --git a/tods/feature_analysis/WaveletTransform.py b/tods/feature_analysis/WaveletTransform.py index 61758b3..00ea7d6 100644 --- a/tods/feature_analysis/WaveletTransform.py +++ b/tods/feature_analysis/WaveletTransform.py @@ -3,8 +3,8 @@ import typing from numpy import ndarray import numpy as np -from d3m import container, utils as d3m_utils from d3m.base import utils as base_utils +from d3m import container, exceptions from d3m.metadata import base as metadata_base, hyperparams from d3m.primitive_interfaces import base, transformer from typing import Union @@ -13,7 +13,6 @@ import pywt import pandas import math -import common_primitives import numpy from typing import Optional, List from collections import OrderedDict diff --git a/tods/resources/.requirements.txt b/tods/resources/.requirements.txt index b773c1d..8e2dd00 100644 --- a/tods/resources/.requirements.txt +++ b/tods/resources/.requirements.txt @@ -1,4 +1,3 @@ numpy==1.18.2 -e git+https://github.com/tods-doc/d3m@70aeefed6b7307941581357c4b7858bb3f88e1da#egg=d3m -e git+https://github.com/tods-doc/axolotl@af54e6970476a081bf0cd65990c9f56a1200d8a2#egg=axolotl --e git+https://gitlab.com/datadrivendiscovery/common-primitives.git@046b20d2f6d4543dcbe18f0a1d4bcbb1f61cf518#egg=common_primitives diff --git a/tods/timeseries_processing/SKAxiswiseScaler.py b/tods/timeseries_processing/SKAxiswiseScaler.py index a18e1e3..d34de79 100644 --- a/tods/timeseries_processing/SKAxiswiseScaler.py +++ b/tods/timeseries_processing/SKAxiswiseScaler.py @@ -9,7 +9,6 @@ from d3m.primitive_interfaces import base, transformer from sklearn.preprocessing import scale -import common_primitives import numpy from typing import Optional, List from collections import OrderedDict From 33e865aae795e3ea5218a2b7f1fa0604261769a2 Mon Sep 17 00:00:00 2001 From: lhenry15 Date: Tue, 10 Nov 2020 01:53:44 -0600 Subject: [PATCH 3/3] fix common-primitive in unit-tests Former-commit-id: 1255f6975922904382a049f8e3d4553280e0fc17 [formerly 1fc78354827e8866bbdef010c0df8e2cf1c52220] [formerly 5fca8c049b779c59c338ac865e2be32eea5b6a98 [formerly 295bef6a120654f4cd95ed29a53e2f1c7b78f75d]] [formerly 193aca8fbe165fb49d05670d0192ac700e1fa3fc [formerly 52340eb907d54bdbd4faed4b9ebb9f6f8a801fc1] [formerly bd9459d5e48ab8d2861011987c752b17571c7b4c [formerly 063574fc52874f3ce524e9038db5413a3a688021]]] [formerly c67c9b32f1c61c0ed405de8a70f02745a9c880da [formerly aebcb4a95ef09602bf335fe0d9a35fc4573f2ba8] [formerly 025dc3a999f362cb9e5c12442856c390df0d0a7b [formerly 181a0845a6dbca8fe022ac5cce88d3d2d0289b9b]] [formerly d0ed5183d5f0d07cf1966f34aa70926d2fbca7ef [formerly 9222f997151b299ad7a55246260098071b40df67] [formerly 726b98399c1b9618ed1ee7d77c17c7c2f6cf4442 [formerly 50b2fc43d9afe2d856bb853e784e7e563fef4660]]]] [formerly 5ad622b2ddf0c4d7879f3f3c40285bb2f84ca94c [formerly a7cef737d306f341154bf2b5103fa7aaf449bbb9] [formerly efa8ed6123cae99df83048e1ff0336ed8378c496 [formerly 5c765d6f675047461c8a28950f4121ec2c249828]] [formerly 4aae1e902d7a7df711de25bd6b337da1abd10e71 [formerly a3545863e9228de286ba6d1fa23bafadb9abd6e6] [formerly 1e3f494cf698431b86ada44bee37a5d5ca3b5e6c [formerly 38be12aab7e8303c76854918e6535f801630a8f4]]] [formerly 8393b7b8bf027f6480ce3ea96e24bd8541d808a6 [formerly 3667703ba5030040f0b8ffbadae40889b39b2b07] [formerly ce4809e8f04a2577ea6b20e778285eb27a6e1c2f [formerly 7ed01e3d3300283d9c3915fd36053927a74ef97f]] [formerly 2325899556b0e191f066d03fbbe614039dc1ce18 [formerly 1a22310b665d74a83759885531598b29ff40be0b] [formerly 34863168b2e2ecddf734777194d850535a5a0f3f [formerly 3c0dce43054a995166e1e78280f4cc9d69ceebe5]]]]] [formerly 153d8a90b76665480ee174e3d106ecd796b894b2 [formerly 11e5cb024d13546aa544497e65af7843abdca984] [formerly 97b760f381f97b239a647b126c1d8307fadf5eda [formerly c4e13a41fa8c6d9144ef717eb9df2b0637760c28]] [formerly 5342b984e4dc93a8ab261e657da54feddb9d58c2 [formerly 77c729f89caa07bfa7e9e4af450ab2716ee45f98] [formerly 008b12000bcf40fc6a425506e76f4aa2dcf4ae32 [formerly 52935cbf231adafbcfba9b045f6f27efb61f24fb]]] [formerly 96d418f47c6e96304e63d6ffc08fffb67db02417 [formerly 23a997533b4c17c08c64078f410827699f2dcb8e] [formerly e5942b092f9c8460d23105f955621ac81b11de20 [formerly 2d5f3e8e9334f61dc054f7b8f94e1bf16955b992]] [formerly c95216209d9cf56f75c2463fc352a333307c9255 [formerly d20dd0c363111e4973405540439c0f2ad94da73d] [formerly c884590b29220e8cda742e75470eefed190676be [formerly ce5689ef23f3c2f89e054834e18ea14c124a5dfa]]]] [formerly f520e9f25fecf7596e7e0c4f1ccf151e61fbd576 [formerly f9398437d9355e6e2d886b9d41505ca684fd257e] [formerly 3c6d2de7b33cbc9aec6637153b2b412c6a7f2430 [formerly 368d9779dfa7884ed2e0503173f5a991da959e33]] [formerly f84f082014dda3117d09473cdbb1a17fa0702956 [formerly a378fc5578eb8f3d1a6dd28b426ed4a9b4957239] [formerly 3961bd9eeb9184ba5a5ab69ead80b053c057ad4c [formerly 6b2c8994f60ee2f0584551bb2909647bf7b31442]]] [formerly cb4225c195cc5b409f8bfad8ac6140a0578ed559 [formerly 8a3effb229d8d882081939d25361c4e4647e56db] [formerly 6467150e879b20e12105745de849112d8f69348f [formerly 879a9de2cdc97fb0d86ff976730cd38d1b7b27ca]] [formerly 9b027d43cee7fababe679cd19a738acad68f43c0 [formerly 712899e9e181b22a3efbef1d9b0d8dd2da5e65c4] [formerly c322d3a5dda50d673226768be11926482f736bba [formerly 78cd35f29b326aa7bf7f58222e7606d2687eb470]]]]]] Former-commit-id: 9de8810985b451ebb38f8ae13455a38d9483fbe2 [formerly 44cd6c8b195f7ba70af36384975e86c68dd20a90] [formerly 52130cbeed61ee9894e9f3b87aadf6d9e71f576d [formerly bfc2acea82728cd3e56d914977077d1c76a11d77]] [formerly 44d643b15039bfc457c95235c6b0ae7b931ab0d6 [formerly 5ee7a3036f7b5e398a6733db49ffcb8ae9b22a3d] [formerly b0209917a11640ea5edb5398e76b10cad55a8c57 [formerly 7730c4faf400e7b62e75745d2a42336f51c00fed]]] [formerly e924f58e550b509adcde027e4182d3077141bdf4 [formerly 67bc8e67b08763c9baf1a1a9b4ab4f30b8d10a61] [formerly 45fee9505f7b92d75b33b74c9aa325a6b9be956c [formerly 2afa9adcc235538b22da2400719fbafa1b665d80]] [formerly 32b7061061359f46ea0ff03269791255d2d503c7 [formerly a122399488cad826028328740226ec3fa68f51d8] [formerly 60526684ddedb5ae0898ffcbfb832d7200fa35c2 [formerly be4026d29830f87a4a394279235bd8c479e69ae9]]]] [formerly 666caaef9f781b6c5df2a7df6caa9dfcfabde161 [formerly 5887b9e434380f18fe745b8ac8da2ea8193b00b8] [formerly a2dd2fcd71caf5bbc1f58b4bd86d009acc89911d [formerly bf2eaad5e5760b7f01c660e785d646052234202b]] [formerly 3ff9f265139b4a0d59344f1492e9e78213fe1371 [formerly 6d4ec5f06725797683c4f5501d2412aabed3dc2d] [formerly 891497cac244bbae5b9797e1f747c736df7a832e [formerly f1cf9602c5dc71de91b508730307adb5bbdcc751]]] [formerly 93e3d4874562446a991bfad34bd55904f91c53bf [formerly 7817e8819d79022a2db6e8b812017f2160c4b511] [formerly 832e091fa01e494191085d19d843c3b828387fb9 [formerly b808874c66ee18fe36ee2b3bb280a6b0866a804a]] [formerly cbecca05b5275db42a3b1d9b8480be69d640a36c [formerly e1cbdd84f5be5a1c049f2587abdb90e3891c3610] [formerly c322d3a5dda50d673226768be11926482f736bba]]]] Former-commit-id: 700df8c1c57dc5f7e6fbd298001bc1fa36da93fb [formerly 2b39d94e9013d56da6cb85747f5565a966e8bbd1] [formerly bda28c3f9e8b955294f68b566ff395d55049dcaa [formerly 238d159dc35857db2771e8d43eb24231599c9fbc]] [formerly 92360cd5d23eefd9e7474638c5de68b3b8a83d8d [formerly 115084b2c44cf08c5bed9b66f54acc5b96713d0e] [formerly eb576742f1ff07a387f28e257e3943537006ae53 [formerly 2e565d5f4bc984ccb8e95372b0fcfde3e913c03c]]] [formerly 5f09f536d09f9ca10cdd1d4da7d2df335f3d9a0e [formerly 2908f3b19c0ab73ee343e6d50d0dbb117124d720] [formerly 0f903a071d3f52c1f64a9ac558facbcf4d7a40af [formerly dab02efc260ede3f5393f06e0cabd343781aa165]] [formerly d2491c1dee3883394f7166f3579bc1304bc35192 [formerly f3ad9b81dbea997dce64da3227f287b613518a4d] [formerly c96f7f95f56b45469b9adcde955e26eb234d3387 [formerly d470a21151adc783e3895e31c27d9fb6846bf041]]]] Former-commit-id: 50f9a736017c71a1e640482d0d70dd3727cabb27 [formerly d90967186191460704126cad2e76680e1cd05ac5] [formerly 1e941dd083ee6a69be6a62d4fa047e268e0d22e9 [formerly f516319382065abc810deba4dc130b853714775a]] [formerly 3bc9b590b9c1a6bc9cfb453eac9d7748f7896a93 [formerly c0afdb4e251245def694cd232562ddcce4d4cce4] [formerly c00a451355d0fabdc5e76c73d34abaf80d63b883 [formerly f1304bf36abf6507d93245a81b9243c9095bcd91]]] Former-commit-id: 7207a1dfed3b055a6a6c5fc2fc77c823075f940e [formerly 4215b7423b77339bc7157cbc5c4e62efdefd7024] [formerly 97982dca03da9f5b872da07c3134e92ac6c1b72c [formerly 6391372a445b2e1ad8caac3e8df8c3f7bb121a4d]] Former-commit-id: 2f8c05774e87690df0dd0b105d382118cfab488d [formerly 65dedb96f8fd89124125425db38ed5749325e3ef] Former-commit-id: 99844b02d0ddd0b94dd12af42a8df4e1eadd8766 --- tods/tests/data_processing/test_CategoricalBinary.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tods/tests/data_processing/test_CategoricalBinary.py b/tods/tests/data_processing/test_CategoricalBinary.py index 0b3d45b..660bb69 100644 --- a/tods/tests/data_processing/test_CategoricalBinary.py +++ b/tods/tests/data_processing/test_CategoricalBinary.py @@ -8,7 +8,6 @@ import numpy as np import pandas as pd import utils as test_utils import os -from common_primitives import dataset_to_dataframe,column_parser class CategoricalBinaryTestCase(unittest.TestCase): def test_basic(self):