wangwei
/
tods
Not watched
1
0
Fork 0
Code
Releases 0 Wiki Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain
Browse Source

fix matrix profile bugs and commented print statements 

Former-commit-id: d5d68788ba [formerly 3ea8b88539] [formerly 11c332bde2 [formerly 001d6c5c60]] [formerly 584506e925 [formerly 62979ef5b1] [formerly 94b8293614 [formerly acc26603ed]]] [formerly e15e5f119a [formerly d8b2943461] [formerly 41e2e99fc0 [formerly 02ac0ecf29]] [formerly 969dbaeb0c [formerly e6c3f59710] [formerly 26d7b495ae [formerly 728e54fe25]]]] [formerly 5760bb6431 [formerly be6481f0aa] [formerly 8c2909c548 [formerly 7895c083c8]] [formerly bcceaee555 [formerly f3eb686f78] [formerly 4af2033b9b [formerly c216fffac3]]] [formerly 8dcc7a4820 [formerly 6caf8a99fc] [formerly 2e36b7881f [formerly aa3b04f341]] [formerly 906a45cfbb [formerly 3e63dc2b21] [formerly 3a17606781 [formerly ef6bdf6174]]]]] [formerly dcdf237a45 [formerly 94e93d59ab] [formerly 38f4ad147f [formerly eb9f815cfd]] [formerly 313eb88e1d [formerly 92c89acddc] [formerly 57742fd789 [formerly 1b6d9cc7a4]]] [formerly d1e3de4007 [formerly 9f92e6d7dc] [formerly a7a49fb63b [formerly 15845a7f70]] [formerly 8deac3f8a8 [formerly 6870e1427e] [formerly 92acfadf56 [formerly b7afedf98c]]]] [formerly 270b2996bf [formerly f6d0a306fe] [formerly f74aaa772c [formerly b98ed65544]] [formerly 6df7b06413 [formerly b7de4064b1] [formerly cce04f79ae [formerly 3b6dfc1015]]] [formerly 5a05937136 [formerly 73f29918b3] [formerly 4e491c65bb [formerly 4c7293a0b2]] [formerly 62eabf448c [formerly 67efab5e5f] [formerly ed38699fe6 [formerly 08eac827e9]]]]]]
Former-commit-id: 31e0d0ac46 [formerly 8ae28ff1b2] [formerly d193f55bd4 [formerly 66dad117ea]] [formerly a6b060d84f [formerly 015b2f7bf6] [formerly e4383e08ab [formerly cc215e27c8]]] [formerly c7a0aeae4d [formerly d0c1c29b05] [formerly afe15612a2 [formerly 770eaab86d]] [formerly a0415d29b9 [formerly b0409a5277] [formerly 29ae6b63c3 [formerly 6c382b54b2]]]] [formerly 3c1c160a0c [formerly 2fe520c45c] [formerly e936b0ea98 [formerly 52f1238308]] [formerly 410f1d65b0 [formerly d946cc6851] [formerly 964d88a60d [formerly 2b19c11dba]]] [formerly deef0ec38f [formerly 57a990f1fc] [formerly ac44fc88ad [formerly 4394b4e0ba]] [formerly 70392add6b [formerly 1a94fc1667] [formerly ed38699fe6]]]]
Former-commit-id: 37fe210c2d [formerly bcbd468991] [formerly 57fc967ad4 [formerly f95d3183bd]] [formerly 5e113fbfa1 [formerly c0619619f0] [formerly 0c844871c0 [formerly c52920552f]]] [formerly e64a5f930e [formerly 3457e6f15e] [formerly 2b28ca86b8 [formerly 2ca90326a2]] [formerly 953e06bd92 [formerly 649aee4f76] [formerly ab1f4bc3e9 [formerly dd5cfdd4ec]]]]
Former-commit-id: 0e97066c65 [formerly bfd1605bbb] [formerly 28b9f6c057 [formerly 41c8e83708]] [formerly 33bedffc6b [formerly 74c330e405] [formerly 280d4992fa [formerly 1cb08aae59]]]
Former-commit-id: b3551107c4 [formerly 3b5ea85abf] [formerly 32f20c6a43 [formerly affac967d1]]
Former-commit-id: 88e4b8ae37 [formerly 72b345babb]
Former-commit-id: 434a2bb28c
master
YileAllenChen1 5 years ago
parent
1707b89247
commit
18c81e9a91
7 changed files with 243 additions and 44 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +1
    -1
      primitive_tests/build_MatrixProfile_pipeline.py
  2. +216
    -0
      tested_file.txt
  3. +5
    -4
      tods/detection_algorithm/DeepLog.py
  4. +17
    -33
      tods/detection_algorithm/MatrixProfile.py
  5. +2
    -5
      tods/feature_analysis/AutoCorrelation.py
  6. +1
    -1
      tods/tests/data_processing/test_ColumnFilter.py
  7. +1
    -0
      tods/tests/detection_algorithm/test_MatrixProfile.py

+ 1
- 1
primitive_tests/build_MatrixProfile_pipeline.py View File

@@ -29,7 +29,7 @@ primitive_2 = index.get_primitive('d3m.primitives.tods.detection_algorithm.matri

step_2 = PrimitiveStep(primitive=primitive_2)
step_2.add_hyperparameter(name='use_semantic_types', argument_type=ArgumentType.VALUE, data=True)
step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,4)) # There is sth wrong with multi-dimensional
step_2.add_hyperparameter(name='use_columns', argument_type=ArgumentType.VALUE, data=(2,3,)) # There is sth wrong with multi-dimensional
step_2.add_hyperparameter(name='window_size', argument_type=ArgumentType.VALUE, data=3) # There is sth wrong with multi-dimensional
step_2.add_hyperparameter(name='return_result', argument_type=ArgumentType.VALUE, data='append')
step_2.add_argument(name='inputs', argument_type=ArgumentType.CONTAINER, data_reference='steps.1.produce')


+ 216
- 0
tested_file.txt View File

@@ -0,0 +1,216 @@
build_ABOD_pipline.py
build_AutoEncoder.py
build_AutoRegODetect_pipeline.py
build_AxiswiseScale_pipline.py
build_BKFilter_pipline.py
build_CategoricalToBinary.py
build_CBLOF_pipline.py
build_ColumnFilter_pipeline.py
build_ContinuityValidation_pipline.py
build_DeepLog_pipeline.py
build_DiscreteCosineTransform.py
build_DuplicationValidation_pipline.py
build_FastFourierTransform.py
build_HBOS_pipline.py
build_HBOS_score_pipline.py
build_HoltSmoothing_pipline.py
build_HoltWintersExponentialSmoothing_pipline.py
build_HPFilter_pipline.py
build_IsolationForest_pipline.py
build_KDiscord_pipeline.py
build_KNN_pipline.py
build_LODA_pipline.py
build_LOF_pipline.py
build_LSTMOD_pipline.py
build_ABOD_pipline.py
build_AutoEncoder.py
build_AutoRegODetect_pipeline.py
build_AxiswiseScale_pipline.py
build_BKFilter_pipline.py
build_CategoricalToBinary.py
build_CBLOF_pipline.py
build_ColumnFilter_pipeline.py
build_ContinuityValidation_pipline.py
build_DeepLog_pipeline.py
build_DiscreteCosineTransform.py
build_DuplicationValidation_pipline.py
build_FastFourierTransform.py
build_HBOS_pipline.py
build_HBOS_score_pipline.py
build_HoltSmoothing_pipline.py
build_HoltWintersExponentialSmoothing_pipline.py
build_HPFilter_pipline.py
build_IsolationForest_pipline.py
build_KDiscord_pipeline.py
build_KNN_pipline.py
build_LODA_pipline.py
build_LOF_pipline.py
build_LSTMOD_pipline.py
build_MatrixProfile_pipeline.py
build_MeanAverageTransform_pipline.py
build_NonNegativeMatrixFactorization.py
build_OCSVM_pipline.py
build_PCAODetect_pipeline.py
build_PowerTransform_pipline.py
build_PyodCOF.py
build_QuantileTransform_pipline.py
build_RuleBasedFilter_pipline.py
build_SimpleExponentialSmoothing_pipline.py
build_SOD_pipeline.py
build_Standardize_pipline.py
build_Telemanom.py
build_test_detection_algorithm_PyodMoGaal.py
build_test_detection_algorithm_PyodSoGaal.py
build_test_feature_analysis_spectral_residual_transform_pipeline.py
build_test_feature_analysis_statistical_abs_energy.py
build_test_feature_analysis_statistical_abs_sum.py
build_test_feature_analysis_statistical_gmean.py
build_test_feature_analysis_statistical_hmean.py
build_test_feature_analysis_statistical_kurtosis.py
build_test_feature_analysis_statistical_maximum.py
build_test_feature_analysis_statistical_mean_abs.py
build_test_feature_analysis_statistical_mean_abs_temporal_derivative.py
build_test_feature_analysis_statistical_mean.py
build_test_feature_analysis_statistical_mean_temporal_derivative.py
build_test_feature_analysis_statistical_median_absolute_deviation.py
build_test_feature_analysis_statistical_median.py
build_test_feature_analysis_statistical_minimum.py
build_test_feature_analysis_statistical_skew.py
build_test_feature_analysis_statistical_std.py
build_test_feature_analysis_statistical_variation.py
build_test_feature_analysis_statistical_var.py
build_test_feature_analysis_statistical_vec_sum.py
build_test_feature_analysis_statistical_willison_amplitude.py
build_test_feature_analysis_statistical_zero_crossing.py
build_test_time_series_seasonality_trend_decomposition.py
build_TimeIntervalTransform_pipeline.py
build_TRMF_pipline.py
build_TruncatedSVD_pipline.py
build_VariationalAutoEncoder.py
build_WaveletTransform_pipline.py
build_ABOD_pipline.py
build_AutoEncoder.py
build_AutoRegODetect_pipeline.py
build_AxiswiseScale_pipline.py
build_BKFilter_pipline.py
build_CategoricalToBinary.py
build_CBLOF_pipline.py
build_ColumnFilter_pipeline.py
build_ContinuityValidation_pipline.py
build_DeepLog_pipeline.py
build_DiscreteCosineTransform.py
build_DuplicationValidation_pipline.py
build_FastFourierTransform.py
build_HBOS_pipline.py
build_HBOS_score_pipline.py
build_HoltSmoothing_pipline.py
build_HoltWintersExponentialSmoothing_pipline.py
build_HPFilter_pipline.py
build_IsolationForest_pipline.py
build_KDiscord_pipeline.py
build_KNN_pipline.py
build_LODA_pipline.py
build_LOF_pipline.py
build_LSTMOD_pipline.py
build_MatrixProfile_pipeline.py
build_MeanAverageTransform_pipline.py
build_NonNegativeMatrixFactorization.py
build_OCSVM_pipline.py
build_PCAODetect_pipeline.py
build_PowerTransform_pipline.py
build_PyodCOF.py
build_QuantileTransform_pipline.py
build_RuleBasedFilter_pipline.py
build_SimpleExponentialSmoothing_pipline.py
build_SOD_pipeline.py
build_Standardize_pipline.py
build_Telemanom.py
build_test_detection_algorithm_PyodMoGaal.py
build_ABOD_pipline.py
build_AutoEncoder.py
build_AutoRegODetect_pipeline.py
build_AxiswiseScale_pipline.py
build_BKFilter_pipline.py
build_CategoricalToBinary.py
build_CBLOF_pipline.py
build_ColumnFilter_pipeline.py
build_ContinuityValidation_pipline.py
build_DeepLog_pipeline.py
build_DiscreteCosineTransform.py
build_DuplicationValidation_pipline.py
build_FastFourierTransform.py
build_HBOS_pipline.py
build_HBOS_score_pipline.py
build_HoltSmoothing_pipline.py
build_HoltWintersExponentialSmoothing_pipline.py
build_HPFilter_pipline.py
build_IsolationForest_pipline.py
build_KDiscord_pipeline.py
build_KNN_pipline.py
build_LODA_pipline.py
build_ABOD_pipline.py
build_AutoEncoder.py
build_AutoRegODetect_pipeline.py
build_AxiswiseScale_pipline.py
build_BKFilter_pipline.py
build_CategoricalToBinary.py
build_CBLOF_pipline.py
build_ColumnFilter_pipeline.py
build_ContinuityValidation_pipline.py
build_DeepLog_pipeline.py
build_DiscreteCosineTransform.py
build_DuplicationValidation_pipline.py
build_FastFourierTransform.py
build_HBOS_pipline.py
build_HBOS_score_pipline.py
build_HoltSmoothing_pipline.py
build_HoltWintersExponentialSmoothing_pipline.py
build_HPFilter_pipline.py
build_IsolationForest_pipline.py
build_KDiscord_pipeline.py
build_KNN_pipline.py
build_LODA_pipline.py
build_LOF_pipline.py
build_LSTMOD_pipline.py
build_MatrixProfile_pipeline.py
build_MeanAverageTransform_pipline.py
build_NonNegativeMatrixFactorization.py
build_OCSVM_pipline.py
build_PCAODetect_pipeline.py
build_PowerTransform_pipline.py
build_PyodCOF.py
build_QuantileTransform_pipline.py
build_RuleBasedFilter_pipline.py
build_SimpleExponentialSmoothing_pipline.py
build_SOD_pipeline.py
build_Standardize_pipline.py
build_Telemanom.py
build_test_detection_algorithm_PyodMoGaal.py
build_test_detection_algorithm_PyodSoGaal.py
build_test_feature_analysis_spectral_residual_transform_pipeline.py
build_test_feature_analysis_statistical_abs_energy.py
build_test_feature_analysis_statistical_abs_sum.py
build_test_feature_analysis_statistical_gmean.py
build_test_feature_analysis_statistical_hmean.py
build_test_feature_analysis_statistical_kurtosis.py
build_test_feature_analysis_statistical_maximum.py
build_test_feature_analysis_statistical_mean_abs.py
build_test_feature_analysis_statistical_mean_abs_temporal_derivative.py
build_test_feature_analysis_statistical_mean.py
build_test_feature_analysis_statistical_mean_temporal_derivative.py
build_test_feature_analysis_statistical_median_absolute_deviation.py
build_test_feature_analysis_statistical_median.py
build_test_feature_analysis_statistical_minimum.py
build_test_feature_analysis_statistical_skew.py
build_test_feature_analysis_statistical_std.py
build_test_feature_analysis_statistical_variation.py
build_test_feature_analysis_statistical_var.py
build_test_feature_analysis_statistical_vec_sum.py
build_test_feature_analysis_statistical_willison_amplitude.py
build_test_feature_analysis_statistical_zero_crossing.py
build_test_time_series_seasonality_trend_decomposition.py
build_TimeIntervalTransform_pipeline.py
build_TRMF_pipline.py
build_TruncatedSVD_pipline.py
build_VariationalAutoEncoder.py
build_WaveletTransform_pipline.py

+ 5
- 4
tods/detection_algorithm/DeepLog.py View File

@@ -311,7 +311,8 @@ class DeeplogLstm(BaseDetector):
# Compile model
model.compile(loss=self.loss, optimizer=self.optimizer)
if self.verbose >= 1:
print(model.summary())
#print(model.summary())
pass
return model

def fit(self,X,y=None):
@@ -396,9 +397,9 @@ class DeeplogLstm(BaseDetector):
check_is_fitted(self, ['model_', 'history_'])

X = check_array(X)
print("inside")
print(X.shape)
print(X[0])
#print("inside")
#print(X.shape)
#print(X[0])
X_norm,Y_norm = self._preprocess_data_for_LSTM(X)
pred_scores = np.zeros(X.shape)
pred_scores[self.window_size:] = self.model_.predict(X_norm)


+ 17
- 33
tods/detection_algorithm/MatrixProfile.py View File

@@ -83,6 +83,15 @@ class MP:
return self

def _get_right_inds(self, data):
right_inds = []
for row in data[1]:
right_inds.append(row+self._window_size-1)
right_inds = pd.DataFrame(right_inds)
data = pd.concat([data,right_inds], axis=1)
data.columns = range(0,len(data.columns))
return data

def produce(self, data):

"""
@@ -93,39 +102,15 @@ class MP:
nparray

"""
"""
#print(data.shape[0s])
rows = data.shape[0]
columns = data.shape[1]
convert_data = np.reshape(data, (columns, rows))
T_data = data.transpose()
#print(T_data)
transformed_columns=utils.pandas.DataFrame()
transformed_columns=d3m_dataframe
print(len(data))
for col in range(len(data)):
output = stumpy.stump(data[col], m = self._window_size)
output = pd.DataFrame(output)
#print("output", output)
transformed_columns=pd.concat([transformed_columns,output],axis=1)
#transformed_columns[col]=output
#print(transformed_columns)
return transformed_columns
# transformed_data = []
# for row in T_data:
# print(row)
# output = stumpy.stump(row, m = self._window_size)
# print(output)
"""
#input from UODBasePrimitive is np.ndarray not dataframe
print("data ",type(data))
#only keep first two columns of MP results, the second column is left index, use windowsize to get right index
transformed_columns=utils.pandas.DataFrame()
for col in data:
print(col)
for col in data.transpose(): #data.reshape(1,len(data)):
output = stumpy.stump(col, m = self._window_size)
output = pd.DataFrame(output)
transformed_columns=pd.concat([transformed_columns,output])
#print(transformed_columns)
output=output.drop(columns=[2,3])
output = self._get_right_inds(output)
transformed_columns=pd.concat([transformed_columns,output], axis=1)
return transformed_columns

def predict(self, data):
@@ -214,10 +199,9 @@ class MatrixProfilePrimitive(UnsupervisedOutlierDetectorBase[Inputs, Outputs, Pa
Container DataFrame
1 marks Outliers, 0 marks normal.
"""
print("inputs ",type(inputs))
return super().produce(inputs=inputs, timeout=timeout, iterations=iterations)

def get_params(self) -> Params:
def get_params(self) -> Params: # pragma: no cover
"""
Return parameters.
Args:
@@ -228,7 +212,7 @@ class MatrixProfilePrimitive(UnsupervisedOutlierDetectorBase[Inputs, Outputs, Pa
"""
return super().get_params()

def set_params(self, *, params: Params) -> None:
def set_params(self, *, params: Params) -> None: # pragma: no cover
"""
Set parameters for outlier detection.
Args:


+ 2
- 5
tods/feature_analysis/AutoCorrelation.py View File

@@ -263,7 +263,6 @@ class AutoCorrelationPrimitive(transformer.TransformerPrimitiveBase[Inputs, Outp
self._training_inputs, self._training_indices = self._get_columns_to_fit(inputs, self.hyperparams)
self._input_column_names = self._training_inputs.columns

print("training_indices_ ", self._training_indices)
if len(self._training_indices) > 0:
self._fitted = True
else: # pragma: no cover
@@ -279,7 +278,6 @@ class AutoCorrelationPrimitive(transformer.TransformerPrimitiveBase[Inputs, Outp
sk_inputs = inputs.iloc[:, self._training_indices]
output_columns = []
if len(self._training_indices) > 0:
print("sk_inputs ", sk_inputs)
sk_output = self._clf.produce(sk_inputs)
if sparse.issparse(sk_output): # pragma: no cover
sk_output = sk_output.toarray()
@@ -357,13 +355,12 @@ class AutoCorrelationPrimitive(transformer.TransformerPrimitiveBase[Inputs, Outp
accepted_structural_types = (int, float, np.integer, np.float64) #changed numpy to np
accepted_semantic_types = set()
accepted_semantic_types.add("https://metadata.datadrivendiscovery.org/types/Attribute")
print("accepted_semantic_types ", accepted_semantic_types)
print("column_metadata['structural_type'] ",column_metadata['structural_type'])

if not issubclass(column_metadata['structural_type'], accepted_structural_types):
return False

semantic_types = set(column_metadata.get('semantic_types', []))
print("semantic_types ", semantic_types)
if len(semantic_types) == 0:
cls.logger.warning("No semantic types found in column metadata")
return False


+ 1
- 1
tods/tests/data_processing/test_ColumnFilter.py View File

@@ -12,7 +12,7 @@ class ColumnFilterTest(unittest.TestCase):
columns=['a', 'b', 'c'],
generate_metadata=True)

print(main)
#print(main)


self.assertEqual(utils.to_json_structure(main.metadata.to_internal_simple_structure()), [{


+ 1
- 0
tods/tests/detection_algorithm/test_MatrixProfile.py View File

@@ -63,6 +63,7 @@ class MatrixProfileTest(unittest.TestCase):
primitive.set_training_data(inputs=main)
primitive.fit()
new_main = primitive.produce(inputs=main).value
#print("new_main.drop([1,2]) ",new_main.drop(columns=[1,2]))
print(new_main)




Write Preview
Loading…
Cancel
Save