From 816fe79e7604afedb8e7ee94735da70fb740d9e2 Mon Sep 17 00:00:00 2001 From: Daochen Zha Date: Tue, 8 Sep 2020 23:20:51 -0500 Subject: [PATCH] Add debugging code Former-commit-id: 460a720b351a7e9fa1fe0b3dfc0d2ac7a4f89f03 [formerly dc201790aa523d6b19310e7221e3f1a4e9001d58] [formerly 6cb3f23787bef3c1ba9a4205ecad44e3490e7da0 [formerly d2da8f3d780cf4f95a0e91f6f83998a4726f60a3]] [formerly 972eddaec224e567d96da92d0403862dfc1d37ab [formerly 42cb5a95a475c9fdf420466e56c51a1e19880a1f] [formerly 8e085919c07633a244c9bb48e75cde964a7aee52 [formerly 88a5fc24182a77bca998adc32edadfc2b6da6db9]]] [formerly b8df80dd50b1720cd15b73d1061600a7d055603a [formerly eabbefd2f4abe471c5f8f222bfbd8109e46097ae] [formerly 142ed8d7bf89443fb31eb1bb8c35577e3da6be99 [formerly 55c66a2fe103491aa844b8a202a9e040c46def0b]] [formerly 756bbdd0f163eaa2ca7421253160023853e401f5 [formerly d7d77a3176e1ab894b2539afac468e979ff54fa0] [formerly d7e8fd634973f8ee860e2f013e3120dc1fa15b2a [formerly 852d4aae4541a74cc7b5df6fb3a0782b68a1d365]]]] Former-commit-id: 5b6a09aa063fc278548f49d51061c433e9cd6122 [formerly 8337654605bdad97874f1bce0b7ae03d79a4e1b4] [formerly 889926be38ed50135022d78befaed518cf075c8a [formerly 8f2a9132a746f48f57fb12bd87e0f003b5bc3d9c]] [formerly 4b53c873863361479c3821d073da8c460a4db0f2 [formerly 7e2caedf3910dbafb675e5a282faf76f3c592950] [formerly d7e8fd634973f8ee860e2f013e3120dc1fa15b2a]] Former-commit-id: 1c30823b7ca40a857c961b84587e2fcd218c370c [formerly 619ddcdde689c729455cea3a9d43395a4ff344e8] [formerly a82c6cfb30c6b20139b3fb65c53bab621992f1b9 [formerly 5e021b5cfdcc319923fa963725e94a13102e4a6f]] Former-commit-id: 271491a2b0037bca173d6650b48755c6f8309cae [formerly 85b126b77a3076a11176ed5560df481dd5ab22ba] Former-commit-id: c95e5345c3f8baef04ab87d2c86a9ee499aedf45 --- examples/run_automl.py | 2 +- tods/tods/search/brute_force_search.py | 110 +++++++++++++------------ 2 files changed, 58 insertions(+), 54 deletions(-) diff --git a/examples/run_automl.py b/examples/run_automl.py index 77b9747..8153555 100644 --- a/examples/run_automl.py +++ b/examples/run_automl.py @@ -28,7 +28,7 @@ backend = SimpleRunner(random_seed=0) search = BruteForceSearch(problem_description=problem_description, backend=backend) # Find the best pipeline -best_runtime, best_pipeline_result = search.search_fit(input_data=[dataset], time_limit=80) +best_runtime, best_pipeline_result = search.search_fit(input_data=[dataset], time_limit=10) best_pipeline = best_runtime.pipeline best_output = best_pipeline_result.output # Evaluate the best pipeline diff --git a/tods/tods/search/brute_force_search.py b/tods/tods/search/brute_force_search.py index c6841ca..24970e6 100644 --- a/tods/tods/search/brute_force_search.py +++ b/tods/tods/search/brute_force_search.py @@ -61,62 +61,59 @@ class BruteForceSearch(PipelineSearchBase): def _return_pipelines(self, task_type, task_subtype, data_type): pipeline_candidates = _generate_pipelines(primitive_python_paths) - print(len(pipeline_candidates)) - exit() return pipeline_candidates primitive_python_paths = { 'data_processing': [ - 'd3m.primitives.tods.data_processing.dataset_to_dataframe', - 'd3m.primitives.tods.data_processing.time_interval_transform', - 'd3m.primitives.tods.data_processing.categorical_to_binary', - 'd3m.primitives.tods.data_processing.column_filter', - 'd3m.primitives.tods.data_processing.timestamp_validation', + #'d3m.primitives.tods.data_processing.time_interval_transform', + #'d3m.primitives.tods.data_processing.categorical_to_binary', + #'d3m.primitives.tods.data_processing.column_filter', + #'d3m.primitives.tods.data_processing.timestamp_validation', 'd3m.primitives.tods.data_processing.duplication_validation', - 'd3m.primitives.tods.data_processing.continuity_validation', + #'d3m.primitives.tods.data_processing.continuity_validation', ], 'timeseries_processing': [ - 'd3m.primitives.tods.timeseries_processing.transformation.axiswise_scaler', + #'d3m.primitives.tods.timeseries_processing.transformation.axiswise_scaler', 'd3m.primitives.tods.timeseries_processing.transformation.standard_scaler', - 'd3m.primitives.tods.timeseries_processing.transformation.power_transformer', - 'd3m.primitives.tods.timeseries_processing.transformation.quantile_transformer', - 'd3m.primitives.tods.timeseries_processing.transformation.moving_average_transform', - 'd3m.primitives.tods.timeseries_processing.transformation.simple_exponential_smoothing', - 'd3m.primitives.tods.timeseries_processing.transformation.holt_smoothing', - 'd3m.primitives.tods.timeseries_processing.transformation.holt_winters_exponential_smoothing', - 'd3m.primitives.tods.timeseries_processing.decomposition.time_series_seasonality_trend_decomposition', + #'d3m.primitives.tods.timeseries_processing.transformation.power_transformer', + #'d3m.primitives.tods.timeseries_processing.transformation.quantile_transformer', + #'d3m.primitives.tods.timeseries_processing.transformation.moving_average_transform', + #'d3m.primitives.tods.timeseries_processing.transformation.simple_exponential_smoothing', + #'d3m.primitives.tods.timeseries_processing.transformation.holt_smoothing', + #'d3m.primitives.tods.timeseries_processing.transformation.holt_winters_exponential_smoothing', + #'d3m.primitives.tods.timeseries_processing.decomposition.time_series_seasonality_trend_decomposition', ], 'feature_analysis': [ - 'd3m.primitives.tods.feature_analysis.auto_correlation', + #'d3m.primitives.tods.feature_analysis.auto_correlation', 'd3m.primitives.tods.feature_analysis.statistical_mean', - 'd3m.primitives.tods.feature_analysis.statistical_median', - 'd3m.primitives.tods.feature_analysis.statistical_g_mean', - 'd3m.primitives.tods.feature_analysis.statistical_abs_energy', - 'd3m.primitives.tods.feature_analysis.statistical_abs_sum', - 'd3m.primitives.tods.feature_analysis.statistical_h_mean', - 'd3m.primitives.tods.feature_analysis.statistical_maximum', - 'd3m.primitives.tods.feature_analysis.statistical_minimum', - 'd3m.primitives.tods.feature_analysis.statistical_mean_abs', - 'd3m.primitives.tods.feature_analysis.statistical_mean_abs_temporal_derivative', - 'd3m.primitives.tods.feature_analysis.statistical_mean_temporal_derivative', - 'd3m.primitives.tods.feature_analysis.statistical_median_abs_deviation', - 'd3m.primitives.tods.feature_analysis.statistical_kurtosis', - 'd3m.primitives.tods.feature_analysis.statistical_skew', - 'd3m.primitives.tods.feature_analysis.statistical_std', - 'd3m.primitives.tods.feature_analysis.statistical_var', - 'd3m.primitives.tods.feature_analysis.statistical_variation', - 'd3m.primitives.tods.feature_analysis.statistical_vec_sum', - 'd3m.primitives.tods.feature_analysis.statistical_willison_amplitude', - 'd3m.primitives.tods.feature_analysis.statistical_zero_crossing', - 'd3m.primitives.tods.feature_analysis.spectral_residual_transform', - 'd3m.primitives.tods.feature_analysis.fast_fourier_transform', - 'd3m.primitives.tods.feature_analysis.discrete_cosine_transform', - 'd3m.primitives.tods.feature_analysis.non_negative_matrix_factorization', - 'd3m.primitives.tods.feature_analysis.bk_filter', - 'd3m.primitives.tods.feature_analysis.hp_filter', - 'd3m.primitives.tods.feature_analysis.truncated_svd', - 'd3m.primitives.tods.feature_analysis.wavelet_transform', - 'd3m.primitives.tods.feature_analysis.trmf', + #'d3m.primitives.tods.feature_analysis.statistical_median', + #'d3m.primitives.tods.feature_analysis.statistical_g_mean', + #'d3m.primitives.tods.feature_analysis.statistical_abs_energy', + #'d3m.primitives.tods.feature_analysis.statistical_abs_sum', + #'d3m.primitives.tods.feature_analysis.statistical_h_mean', + #'d3m.primitives.tods.feature_analysis.statistical_maximum', + #'d3m.primitives.tods.feature_analysis.statistical_minimum', + #'d3m.primitives.tods.feature_analysis.statistical_mean_abs', + #'d3m.primitives.tods.feature_analysis.statistical_mean_abs_temporal_derivative', + #'d3m.primitives.tods.feature_analysis.statistical_mean_temporal_derivative', + #'d3m.primitives.tods.feature_analysis.statistical_median_abs_deviation', + #'d3m.primitives.tods.feature_analysis.statistical_kurtosis', + #'d3m.primitives.tods.feature_analysis.statistical_skew', + #'d3m.primitives.tods.feature_analysis.statistical_std', + #'d3m.primitives.tods.feature_analysis.statistical_var', + #'d3m.primitives.tods.feature_analysis.statistical_variation', + #'d3m.primitives.tods.feature_analysis.statistical_vec_sum', + #'d3m.primitives.tods.feature_analysis.statistical_willison_amplitude', + #'d3m.primitives.tods.feature_analysis.statistical_zero_crossing', + #'d3m.primitives.tods.feature_analysis.spectral_residual_transform', + #'d3m.primitives.tods.feature_analysis.fast_fourier_transform', + #'d3m.primitives.tods.feature_analysis.discrete_cosine_transform', + #'d3m.primitives.tods.feature_analysis.non_negative_matrix_factorization', + #'d3m.primitives.tods.feature_analysis.bk_filter', + #'d3m.primitives.tods.feature_analysis.hp_filter', + #'d3m.primitives.tods.feature_analysis.truncated_svd', + #'d3m.primitives.tods.feature_analysis.wavelet_transform', + #'d3m.primitives.tods.feature_analysis.trmf', ], 'detection_algorithm': [ 'd3m.primitives.tods.detection_algorithm.pyod_ae', @@ -133,19 +130,26 @@ primitive_python_paths = { 'd3m.primitives.tods.detection_algorithm.pyod_cblof', 'd3m.primitives.tods.detection_algorithm.pyod_sogaal', 'd3m.primitives.tods.detection_algorithm.pyod_mogaal', - 'd3m.primitives.tods.detection_algorithm.matrix_profile', - 'd3m.primitives.tods.detection_algorithm.AutoRegODetector', - 'd3m.primitives.tods.detection_algorithm.LSTMODetector', - 'd3m.primitives.tods.detection_algorithm.AutoRegODetector', - 'd3m.primitives.tods.detection_algorithm.PCAODetector', - 'd3m.primitives.tods.detection_algorithm.KDiscordODetector', - 'd3m.primitives.tods.detection_algorithm.deeplog', - 'd3m.primitives.tods.detection_algorithm.telemanom', + #'d3m.primitives.tods.detection_algorithm.matrix_profile', + #'d3m.primitives.tods.detection_algorithm.AutoRegODetector', + #'d3m.primitives.tods.detection_algorithm.LSTMODetector', + #'d3m.primitives.tods.detection_algorithm.AutoRegODetector', + #'d3m.primitives.tods.detection_algorithm.PCAODetector', + #'d3m.primitives.tods.detection_algorithm.KDiscordODetector', + #'d3m.primitives.tods.detection_algorithm.deeplog', + #'d3m.primitives.tods.detection_algorithm.telemanom', ] } def _f1_rank(pipeline_result): + try: + for error in pipeline_result.error: + if error is not None: + raise error + except: + import traceback + traceback.print_exc() if pipeline_result.status == 'COMPLETED': scores = pipeline_result.scores pipeline_result.rank = -scores['value'][0]