diff --git a/primitive_tests/data_processing/CategoricalToBinary_pipeline.py b/primitive_tests/data_processing/CategoricalToBinary_pipeline.py index ea6ccea..da3eee8 100644 --- a/primitive_tests/data_processing/CategoricalToBinary_pipeline.py +++ b/primitive_tests/data_processing/CategoricalToBinary_pipeline.py @@ -10,20 +10,20 @@ pipeline_description.add_input(name='inputs') # Step 0: dataset_to_dataframe primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') +step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data='inputs.0') step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: Column Parser primitive_1 = index.get_primitive('d3m.primitives.tods.data_processing.column_parser') step_1 = PrimitiveStep(primitive=primitive_1) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') +step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) # Step 2: extract_columns_by_semantic_types(attributes) step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') +step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data='steps.1.produce') step_2.add_output('produce') step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, data=['https://metadata.datadrivendiscovery.org/types/Attribute']) @@ -34,7 +34,7 @@ step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_p step_3.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True) step_3.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(3,)) step_3.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append') -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') +step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data='steps.2.produce') step_3.add_output('produce') pipeline_description.add_step(step_3) diff --git a/primitive_tests/data_processing/ColumnFilter_pipeline.py b/primitive_tests/data_processing/ColumnFilter_pipeline.py index 83f0087..c6c3c9c 100644 --- a/primitive_tests/data_processing/ColumnFilter_pipeline.py +++ b/primitive_tests/data_processing/ColumnFilter_pipeline.py @@ -12,19 +12,19 @@ pipeline_description.add_input(name='inputs') # Step 0: dataset_to_dataframe primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') +step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data='inputs.0') step_0.add_output('produce') pipeline_description.add_step(step_0) #Step 1: column_parser step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') +step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) # Step 2: extract_columns_by_semantic_types(attributes) step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.extract_columns_by_semantic_types')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') +step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data='steps.1.produce') step_2.add_output('produce') step_2.add_hyperparameter(name='semantic_types', argument_type=ArgumentType.VALUE, data=['https://metadata.datadrivendiscovery.org/types/Attribute']) @@ -32,7 +32,7 @@ pipeline_description.add_step(step_2) # Step 3: column_filter step_3 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_filter')) -step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.2.produce') +step_3.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data='steps.2.produce') step_3.add_output('produce') pipeline_description.add_step(step_3) diff --git a/primitive_tests/data_processing/ContinuityValidation_pipline.py b/primitive_tests/data_processing/ContinuityValidation_pipline.py index bf6f772..0129e1a 100644 --- a/primitive_tests/data_processing/ContinuityValidation_pipline.py +++ b/primitive_tests/data_processing/ContinuityValidation_pipline.py @@ -8,19 +8,19 @@ pipeline_description.add_input(name='inputs') # Step 0: dataset_to_dataframe step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe')) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') +step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data='inputs.0') step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: column_parser step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') +step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) # Step 3: ContinuityValidation step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.continuity_validation')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') +step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data='steps.1.produce') step_2.add_output('produce') step_2.add_hyperparameter(name = 'continuity_option', argument_type=ArgumentType.VALUE, data = 'imputation') step_2.add_hyperparameter(name = 'interval', argument_type=ArgumentType.VALUE, data = 0.3) diff --git a/primitive_tests/data_processing/DuplicationValidation_pipeline.py b/primitive_tests/data_processing/DuplicationValidation_pipeline.py index 15788d6..02d6732 100644 --- a/primitive_tests/data_processing/DuplicationValidation_pipeline.py +++ b/primitive_tests/data_processing/DuplicationValidation_pipeline.py @@ -9,19 +9,19 @@ pipeline_description.add_input(name='inputs') # Step 0: dataset_to_dataframe step_0 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe')) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') +step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data='inputs.0') step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: column_parser step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') +step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) # Step 2: DuplicationValidation step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.duplication_validation')) -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') +step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data='steps.1.produce') step_2.add_output('produce') step_2.add_hyperparameter(name = 'keep_option', argument_type=ArgumentType.VALUE, data = 'average') # Or: 'first' pipeline_description.add_step(step_2) diff --git a/primitive_tests/data_processing/TimeIntervalTransform_pipeline.py b/primitive_tests/data_processing/TimeIntervalTransform_pipeline.py index fa26892..333b4b4 100644 --- a/primitive_tests/data_processing/TimeIntervalTransform_pipeline.py +++ b/primitive_tests/data_processing/TimeIntervalTransform_pipeline.py @@ -10,20 +10,20 @@ pipeline_description.add_input(name='inputs') # Step 0: dataset_to_dataframe primitive_0 = index.get_primitive('d3m.primitives.tods.data_processing.dataset_to_dataframe') step_0 = PrimitiveStep(primitive=primitive_0) -step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='inputs.0') +step_0.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data='inputs.0') step_0.add_output('produce') pipeline_description.add_step(step_0) # Step 1: column_parser step_1 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.column_parser')) -step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.0.produce') +step_1.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data='steps.0.produce') step_1.add_output('produce') pipeline_description.add_step(step_1) # Step 2: time_interval_transform step_2 = PrimitiveStep(primitive=index.get_primitive('d3m.primitives.tods.data_processing.time_interval_transform')) step_2.add_hyperparameter(name="time_interval", argument_type=ArgumentType.VALUE, data = 'T') -step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce') +step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data='steps.1.produce') step_2.add_output('produce') pipeline_description.add_step(step_2) diff --git a/primitive_tests/test.sh b/primitive_tests/test.sh index 30e02d2..6aea88a 100644 --- a/primitive_tests/test.sh +++ b/primitive_tests/test.sh @@ -1,8 +1,9 @@ #!/bin/bash -modules="data_processing timeseries_processing feature_analysis detection_algorithm reinforcement" +#modules="data_processing timeseries_processing feature_analysis detection_algorithms reinforcement" #modules="data_processing timeseries_processing" -#modules="detection_algorithm" +modules="data_processing" +#test_scripts=$(ls primitive_tests | grep -v -f tested_file.txt) for module in $modules do diff --git a/primitive_tests/tested_file.txt b/primitive_tests/tested_file.txt new file mode 100644 index 0000000..181859c --- /dev/null +++ b/primitive_tests/tested_file.txt @@ -0,0 +1 @@ +CategoricalToBinary_pipeline.py diff --git a/setup.py b/setup.py index 52a9a39..516a10e 100644 --- a/setup.py +++ b/setup.py @@ -35,19 +35,17 @@ setup( ] }, install_requires=[ - #'tamu_d3m', - #'tamu_axolotl', - #'Jinja2', - 'numpy==1.18.2', + 'tamu_d3m==2021.11.24', + 'tamu_axolotl', + 'numpy<=1.21.2', 'combo', 'simplejson==3.12.0', - #'scikit-learn==0.22.0', 'scikit-learn', 'statsmodels==0.11.1', 'PyWavelets>=1.1.1', 'pillow==7.1.2', - 'tensorflow==2.2', # should be removed later - 'keras', # should be removed later + 'tensorflow==2.4', + 'keras==2.4.0', 'pyod', 'nimfa==1.4.0', 'stumpy==1.4.0', diff --git a/tods/detection_algorithm/DAGMM.py b/tods/detection_algorithm/DAGMM.py index 0a2bff4..7daa574 100644 --- a/tods/detection_algorithm/DAGMM.py +++ b/tods/detection_algorithm/DAGMM.py @@ -113,7 +113,7 @@ class DAGMMPrimitive(UnsupervisedOutlierDetectorBase[Inputs, Outputs, Params, Hy 'python_path': 'd3m.primitives.tods.detection_algorithm.dagmm', 'source': {'name': "DATALAB @Taxes A&M University", 'contact': 'mailto:khlai037@tamu.edu', 'uris': ['https://gitlab.com/lhenry15/tods/-/blob/Yile/anomaly-primitives/anomaly_primitives/DAGMM.py']}, - 'algorithm_types': [metadata_base.PrimitiveAlgorithmType.DEEPLOG], + 'algorithm_types': [metadata_base.PrimitiveAlgorithmType.TODS_PRIMITIVE], 'primitive_family': metadata_base.PrimitiveFamily.ANOMALY_DETECTION, 'id': str(uuid.uuid3(uuid.NAMESPACE_DNS, 'DAGMMPrimitive')), 'hyperparams_to_tune': ['comp_hiddens','est_hiddens','est_dropout_ratio','minibatch_size','epoch_size','rand_seed', diff --git a/tods/detection_algorithm/Ensemble.py b/tods/detection_algorithm/Ensemble.py index fe7df28..e6b9423 100644 --- a/tods/detection_algorithm/Ensemble.py +++ b/tods/detection_algorithm/Ensemble.py @@ -8,8 +8,7 @@ import numpy import typing import pandas as pd # Custom import commands if any -from sklearn.preprocessing.data import Normalizer -from statsmodels.tsa.api import ExponentialSmoothing, SimpleExpSmoothing, Holt +from sklearn.preprocessing import Normalizer import uuid diff --git a/tods/detection_algorithm/core/CollectiveCommonTest.py b/tods/detection_algorithm/core/CollectiveCommonTest.py index 7ab7083..1eb6b98 100755 --- a/tods/detection_algorithm/core/CollectiveCommonTest.py +++ b/tods/detection_algorithm/core/CollectiveCommonTest.py @@ -9,13 +9,13 @@ import sys import numpy as np import unittest # noinspection PyProtectedMember -from sklearn.utils.testing import assert_allclose -from sklearn.utils.testing import assert_array_less -from sklearn.utils.testing import assert_equal -from sklearn.utils.testing import assert_greater -from sklearn.utils.testing import assert_greater_equal -from sklearn.utils.testing import assert_less_equal -from sklearn.utils.testing import assert_raises + +from numpy.testing import assert_equal +from numpy.testing import assert_allclose +from numpy.testing import assert_array_less +from numpy.testing import assert_raises + +from unittest import TestCase from sklearn.utils.estimator_checks import check_estimator @@ -28,6 +28,12 @@ sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) from pyod.utils.data import generate_data +_dummy = TestCase('__init__') +assert_greater = _dummy.assertGreater +assert_greater_equal = _dummy.assertGreaterEqual +assert_less = _dummy.assertLess +assert_less_equal = _dummy.assertLessEqual + class CollectiveCommonTest: def __init__(self, diff --git a/tods/detection_algorithm/core/UODCommonTest.py b/tods/detection_algorithm/core/UODCommonTest.py index 6140c85..4e9f7c4 100755 --- a/tods/detection_algorithm/core/UODCommonTest.py +++ b/tods/detection_algorithm/core/UODCommonTest.py @@ -9,13 +9,12 @@ import sys import numpy as np import unittest # noinspection PyProtectedMember -from sklearn.utils.testing import assert_allclose -from sklearn.utils.testing import assert_array_less -from sklearn.utils.testing import assert_equal -from sklearn.utils.testing import assert_greater -from sklearn.utils.testing import assert_greater_equal -from sklearn.utils.testing import assert_less_equal -from sklearn.utils.testing import assert_raises +from numpy.testing import assert_equal +from numpy.testing import assert_allclose +from numpy.testing import assert_array_less +from numpy.testing import assert_raises + +from unittest import TestCase from sklearn.utils.estimator_checks import check_estimator @@ -28,6 +27,12 @@ sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) from pyod.utils.data import generate_data +_dummy = TestCase('__init__') +assert_greater = _dummy.assertGreater +assert_greater_equal = _dummy.assertGreaterEqual +assert_less = _dummy.assertLess +assert_less_equal = _dummy.assertLessEqual + class UODCommonTest: def __init__(self, diff --git a/tods/detection_algorithm/core/dagmm/dagmm.py b/tods/detection_algorithm/core/dagmm/dagmm.py index 47c70c9..02f882f 100644 --- a/tods/detection_algorithm/core/dagmm/dagmm.py +++ b/tods/detection_algorithm/core/dagmm/dagmm.py @@ -1,7 +1,7 @@ import tensorflow as tf import numpy as np from sklearn.preprocessing import StandardScaler -from sklearn.externals import joblib +import joblib from .compression_net import CompressionNet from .estimation_net import EstimationNet diff --git a/tods/feature_analysis/SKTruncatedSVD.py b/tods/feature_analysis/SKTruncatedSVD.py index 24ba96c..b42f060 100644 --- a/tods/feature_analysis/SKTruncatedSVD.py +++ b/tods/feature_analysis/SKTruncatedSVD.py @@ -10,7 +10,7 @@ import time import uuid # Custom import commands if any -from sklearn.decomposition.truncated_svd import TruncatedSVD +from sklearn.decomposition import TruncatedSVD from d3m.container.numpy import ndarray as d3m_ndarray diff --git a/tods/feature_analysis/TRMF.py b/tods/feature_analysis/TRMF.py index a4a3fd9..1bb8ea0 100644 --- a/tods/feature_analysis/TRMF.py +++ b/tods/feature_analysis/TRMF.py @@ -10,8 +10,6 @@ import time import uuid # Custom import commands if any -from sklearn.decomposition.truncated_svd import TruncatedSVD - from d3m.container.numpy import ndarray as d3m_ndarray from d3m.container import DataFrame as d3m_dataframe @@ -21,7 +19,6 @@ from d3m.base import utils as base_utils from d3m.exceptions import PrimitiveNotFittedError from d3m.primitive_interfaces.base import CallResult, DockerContainer from d3m.primitive_interfaces import base, transformer -# from d3m.primitive_interfaces.unsupervised_learning import UnsupervisedLearnerPrimitiveBase from ..common.TODSBasePrimitives import TODSTransformerPrimitiveBase @@ -249,7 +246,7 @@ class TRMFPrimitive(TODSTransformerPrimitiveBase[Inputs, Outputs, Hyperparams]): inputs: Container DataFrame. Returns: - Container DataFrame after Truncated SVD. + Container DataFrame after TRMF. """ self._clf = trmf( lags=self.hyperparams['lags'], diff --git a/tods/resources/.entry_points.ini b/tods/resources/.entry_points.ini index 4104e46..7d545ff 100644 --- a/tods/resources/.entry_points.ini +++ b/tods/resources/.entry_points.ini @@ -75,6 +75,7 @@ tods.detection_algorithm.AutoRegODetector = tods.detection_algorithm.AutoRegODet tods.detection_algorithm.LSTMODetector = tods.detection_algorithm.LSTMODetect:LSTMODetectorPrimitive tods.detection_algorithm.PCAODetector = tods.detection_algorithm.PCAODetect:PCAODetectorPrimitive tods.detection_algorithm.KDiscordODetector = tods.detection_algorithm.KDiscordODetect:KDiscordODetectorPrimitive +tods.detection_algorithm.dagmm = tods.detection_algorithm.DAGMM:DAGMMPrimitive. tods.detection_algorithm.deeplog = tods.detection_algorithm.DeepLog:DeepLogPrimitive tods.detection_algorithm.telemanom = tods.detection_algorithm.Telemanom:TelemanomPrimitive tods.detection_algorithm.system_wise_detection = tods.detection_algorithm.SystemWiseDetection:SystemWiseDetectionPrimitive diff --git a/tods/timeseries_processing/HoltSmoothing.py b/tods/timeseries_processing/HoltSmoothing.py index 383df14..11be548 100644 --- a/tods/timeseries_processing/HoltSmoothing.py +++ b/tods/timeseries_processing/HoltSmoothing.py @@ -9,7 +9,7 @@ import numpy import typing import pandas as pd # Custom import commands if any -from sklearn.preprocessing.data import Normalizer +from sklearn.preprocessing import Normalizer from statsmodels.tsa.api import ExponentialSmoothing, SimpleExpSmoothing, Holt @@ -191,9 +191,6 @@ class HoltSmoothingPrimitive(UnsupervisedLearnerPrimitiveBase[Inputs, Outputs, P except Exception as e: self.logger.error("Error in Calculating Holt smoothing",e) self._update_metadata(outputs) - #print(inputs) - #print("-------------") - print(outputs) return base.CallResult(outputs) diff --git a/tods/timeseries_processing/HoltWintersExponentialSmoothing.py b/tods/timeseries_processing/HoltWintersExponentialSmoothing.py index 5b435f5..df8819d 100644 --- a/tods/timeseries_processing/HoltWintersExponentialSmoothing.py +++ b/tods/timeseries_processing/HoltWintersExponentialSmoothing.py @@ -9,7 +9,7 @@ import uuid import typing import pandas as pd # Custom import commands if any -from sklearn.preprocessing.data import Normalizer +from sklearn.preprocessing import Normalizer from statsmodels.tsa.api import ExponentialSmoothing, SimpleExpSmoothing, Holt diff --git a/tods/timeseries_processing/MovingAverageTransformer.py b/tods/timeseries_processing/MovingAverageTransformer.py index 2bdfc0f..dea7b22 100644 --- a/tods/timeseries_processing/MovingAverageTransformer.py +++ b/tods/timeseries_processing/MovingAverageTransformer.py @@ -9,7 +9,7 @@ import typing import pandas as pd import uuid # Custom import commands if any -from sklearn.preprocessing.data import Normalizer +from sklearn.preprocessing import Normalizer from d3m.container.numpy import ndarray as d3m_ndarray diff --git a/tods/timeseries_processing/SKQuantileTransformer.py b/tods/timeseries_processing/SKQuantileTransformer.py index 184fadd..cd69ce6 100644 --- a/tods/timeseries_processing/SKQuantileTransformer.py +++ b/tods/timeseries_processing/SKQuantileTransformer.py @@ -8,7 +8,7 @@ import numpy import typing # Custom import commands if any -from sklearn.preprocessing.data import QuantileTransformer +from sklearn.preprocessing import QuantileTransformer from d3m.container.numpy import ndarray as d3m_ndarray diff --git a/tods/timeseries_processing/SimpleExponentialSmoothing.py b/tods/timeseries_processing/SimpleExponentialSmoothing.py index 95c9a0b..6245959 100644 --- a/tods/timeseries_processing/SimpleExponentialSmoothing.py +++ b/tods/timeseries_processing/SimpleExponentialSmoothing.py @@ -8,7 +8,7 @@ import numpy import typing import pandas as pd # Custom import commands if any -from sklearn.preprocessing.data import Normalizer +from sklearn.preprocessing import Normalizer from statsmodels.tsa.api import ExponentialSmoothing, SimpleExpSmoothing, Holt import uuid