import contextlib import json import gzip import io import logging import os.path import pickle import random import shutil import sys import tempfile import traceback import unittest import pandas COMMON_PRIMITIVES_DIR = os.path.join(os.path.dirname(__file__), 'common-primitives') # NOTE: This insertion should appear before any code attempting to resolve or load primitives, # so the git submodule version of `common-primitives` is looked at first. sys.path.insert(0, COMMON_PRIMITIVES_DIR) TEST_PRIMITIVES_DIR = os.path.join(os.path.dirname(__file__), 'data', 'primitives') sys.path.insert(0, TEST_PRIMITIVES_DIR) from common_primitives.column_parser import ColumnParserPrimitive from common_primitives.construct_predictions import ConstructPredictionsPrimitive from common_primitives.dataset_to_dataframe import DatasetToDataFramePrimitive from common_primitives.no_split import NoSplitDatasetSplitPrimitive from common_primitives.random_forest import RandomForestClassifierPrimitive from common_primitives.train_score_split import TrainScoreDatasetSplitPrimitive from test_primitives.random_classifier import RandomClassifierPrimitive from test_primitives.fake_score import FakeScorePrimitive from d3m import cli, index, runtime, utils from d3m.container import dataset as dataset_module from d3m.contrib.primitives.compute_scores import ComputeScoresPrimitive from d3m.metadata import base as metadata_base, pipeline as pipeline_module, pipeline_run as pipeline_run_module, problem as problem_module TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), 'data') PROBLEM_DIR = os.path.join(TEST_DATA_DIR, 'problems') DATASET_DIR = os.path.join(TEST_DATA_DIR, 'datasets') PIPELINE_DIR = os.path.join(TEST_DATA_DIR, 'pipelines') class TestCLIRuntime(unittest.TestCase): def setUp(self): self.test_dir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.test_dir) @classmethod def setUpClass(cls): to_register = { 'd3m.primitives.data_transformation.dataset_to_dataframe.Common': DatasetToDataFramePrimitive, 'd3m.primitives.classification.random_forest.Common': RandomForestClassifierPrimitive, 'd3m.primitives.classification.random_classifier.Test': RandomClassifierPrimitive, 'd3m.primitives.data_transformation.column_parser.Common': ColumnParserPrimitive, 'd3m.primitives.data_transformation.construct_predictions.Common': ConstructPredictionsPrimitive, 'd3m.primitives.evaluation.no_split_dataset_split.Common': NoSplitDatasetSplitPrimitive, 'd3m.primitives.evaluation.compute_scores.Test': FakeScorePrimitive, 'd3m.primitives.evaluation.train_score_dataset_split.Common': TrainScoreDatasetSplitPrimitive, # We do not have to load this primitive, but loading it here prevents the package from loading all primitives. 'd3m.primitives.evaluation.compute_scores.Core': ComputeScoresPrimitive, } # To hide any logging or stdout output. with utils.silence(): for python_path, primitive in to_register.items(): index.register_primitive(python_path, primitive) def _call_cli_runtime(self, arg): logger = logging.getLogger('d3m.runtime') with utils.silence(): with self.assertLogs(logger=logger) as cm: # So that at least one message is logged. logger.warning("Debugging.") cli.main(arg) # We skip our "debugging" message. return cm.records[1:] def _call_cli_runtime_without_fail(self, arg): try: return self._call_cli_runtime(arg) except Exception as e: self.fail(traceback.format_exc()) def _assert_valid_saved_pipeline_runs(self, pipeline_run_save_path): with open(pipeline_run_save_path, 'r') as f: for pipeline_run_dict in list(utils.yaml_load_all(f)): try: pipeline_run_module.validate_pipeline_run(pipeline_run_dict) except Exception as e: self.fail(traceback.format_exc()) def _validate_previous_pipeline_run_ids(self, pipeline_run_save_path): ids = set() prev_ids = set() with open(pipeline_run_save_path, 'r') as f: for pipeline_run_dict in list(utils.yaml_load_all(f)): ids.add(pipeline_run_dict['id']) if 'previous_pipeline_run' in pipeline_run_dict: prev_ids.add(pipeline_run_dict['previous_pipeline_run']['id']) self.assertTrue( prev_ids.issubset(ids), 'Some previous pipeline run ids {} are not in the set of pipeline run ids {}'.format(prev_ids, ids) ) def test_fit_multi_input(self): pipeline_run_save_path = os.path.join(self.test_dir, 'pipeline_run.yml') arg = [ '', 'runtime', 'fit', '--input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--problem', os.path.join(PROBLEM_DIR, 'iris_problem_1/problemDoc.json'), '--pipeline', os.path.join(PIPELINE_DIR, 'multi-input-test.json'), '--expose-produced-outputs', self.test_dir, '-O', pipeline_run_save_path, ] self._call_cli_runtime_without_fail(arg) self._assert_valid_saved_pipeline_runs(pipeline_run_save_path) self._assert_standard_output_metadata() def test_fit_without_problem(self): pipeline_run_save_path = os.path.join(self.test_dir, 'pipeline_run.yml') fitted_pipeline_path = os.path.join(self.test_dir, 'fitted-pipeline') output_csv_path = os.path.join(self.test_dir, 'output.csv') arg = [ '', 'runtime', 'fit', '--input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--pipeline', os.path.join(PIPELINE_DIR, 'multi-input-test.json'), '--save', fitted_pipeline_path, '--expose-produced-outputs', self.test_dir, '--output', output_csv_path, '-O', pipeline_run_save_path, ] self._call_cli_runtime_without_fail(arg) self.assertEqual(utils.list_files(self.test_dir), [ 'fitted-pipeline', 'output.csv', 'outputs.0/data.csv', 'outputs.0/metadata.json', 'pipeline_run.yml', 'steps.0.produce/data.csv', 'steps.0.produce/metadata.json', 'steps.1.produce/data.csv', 'steps.1.produce/metadata.json', 'steps.2.produce/data.csv', 'steps.2.produce/metadata.json' ]) self._assert_valid_saved_pipeline_runs(pipeline_run_save_path) self._assert_standard_output_metadata() self._assert_prediction_sum(prediction_sum=11225, outputs_path='outputs.0/data.csv') self._assert_prediction_sum(prediction_sum=11225, outputs_path='output.csv') def test_produce_without_problem(self): pipeline_run_save_path = os.path.join(self.test_dir, 'pipeline_run.yml') fitted_pipeline_path = os.path.join(self.test_dir, 'fitted-no-problem-pipeline') output_csv_path = os.path.join(self.test_dir, 'output.csv') arg = [ '', 'runtime', 'fit', '--input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--pipeline', os.path.join(PIPELINE_DIR, 'multi-input-test.json'), '--save', fitted_pipeline_path, ] self._call_cli_runtime_without_fail(arg) arg = [ '', 'runtime', 'produce', '--test-input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--test-input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--output', output_csv_path, '--fitted-pipeline', fitted_pipeline_path, '--expose-produced-outputs', self.test_dir, '-O', pipeline_run_save_path, ] self._call_cli_runtime_without_fail(arg) self.assertEqual(utils.list_files(self.test_dir), [ 'fitted-no-problem-pipeline', 'output.csv', 'outputs.0/data.csv', 'outputs.0/metadata.json', 'pipeline_run.yml', 'steps.0.produce/data.csv', 'steps.0.produce/metadata.json', 'steps.1.produce/data.csv', 'steps.1.produce/metadata.json', 'steps.2.produce/data.csv', 'steps.2.produce/metadata.json' ]) self._assert_valid_saved_pipeline_runs(pipeline_run_save_path) self._assert_standard_output_metadata() self._assert_prediction_sum(prediction_sum=11008, outputs_path='outputs.0/data.csv') self._assert_prediction_sum(prediction_sum=11008, outputs_path='output.csv') def test_fit_produce_without_problem(self): pipeline_run_save_path = os.path.join(self.test_dir, 'pipeline_run.yml') output_csv_path = os.path.join(self.test_dir, 'output.csv') arg = [ '', 'runtime', 'fit-produce', '--input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--test-input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--test-input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--pipeline', os.path.join(PIPELINE_DIR, 'multi-input-test.json'), '--output', output_csv_path, '--expose-produced-outputs', self.test_dir, '-O', pipeline_run_save_path, ] self._call_cli_runtime_without_fail(arg) self.assertEqual(utils.list_files(self.test_dir), [ 'output.csv', 'outputs.0/data.csv', 'outputs.0/metadata.json', 'pipeline_run.yml', 'steps.0.produce/data.csv', 'steps.0.produce/metadata.json', 'steps.1.produce/data.csv', 'steps.1.produce/metadata.json', 'steps.2.produce/data.csv', 'steps.2.produce/metadata.json' ]) self._assert_valid_saved_pipeline_runs(pipeline_run_save_path) self._validate_previous_pipeline_run_ids(pipeline_run_save_path) self._assert_standard_output_metadata() self._assert_prediction_sum(prediction_sum=11008, outputs_path='outputs.0/data.csv') self._assert_prediction_sum(prediction_sum=11008, outputs_path='output.csv') def test_nonstandard_fit_without_problem(self): pipeline_run_save_path = os.path.join(self.test_dir, 'pipeline_run.yml') fitted_pipeline_path = os.path.join(self.test_dir, 'fitted-pipeline') arg = [ '', 'runtime', 'fit', '--input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--pipeline', os.path.join(PIPELINE_DIR, 'semi-standard-pipeline.json'), '--save', fitted_pipeline_path, '--expose-produced-outputs', self.test_dir, '--not-standard-pipeline', '-O', pipeline_run_save_path, ] self._call_cli_runtime_without_fail(arg) self.assertEqual(utils.list_files(self.test_dir), [ 'fitted-pipeline', 'outputs.0/data.csv', 'outputs.0/metadata.json', 'outputs.1/data.csv', 'outputs.1/metadata.json', 'pipeline_run.yml', 'steps.0.produce/data.csv', 'steps.0.produce/metadata.json', 'steps.1.produce/data.csv', 'steps.1.produce/metadata.json', ]) self._assert_valid_saved_pipeline_runs(pipeline_run_save_path) self._assert_standard_output_metadata() self._assert_prediction_sum(prediction_sum=10710, outputs_path='outputs.0/data.csv') self._assert_nonstandard_output(outputs_name='outputs.1') def test_nonstandard_produce_without_problem(self): pipeline_run_save_path = os.path.join(self.test_dir, 'pipeline_run.yml') fitted_pipeline_path = os.path.join(self.test_dir, 'fitted-pipeline') arg = [ '', 'runtime', 'fit', '--input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--pipeline', os.path.join(PIPELINE_DIR, 'semi-standard-pipeline.json'), '--save', fitted_pipeline_path, '--not-standard-pipeline' ] self._call_cli_runtime_without_fail(arg) arg = [ '', 'runtime', 'produce', '--test-input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--fitted-pipeline', fitted_pipeline_path, '--expose-produced-outputs', self.test_dir, '-O', pipeline_run_save_path, ] self._call_cli_runtime_without_fail(arg) self.assertEqual(utils.list_files(self.test_dir), [ 'fitted-pipeline', 'outputs.0/data.csv', 'outputs.0/metadata.json', 'outputs.1/data.csv', 'outputs.1/metadata.json', 'pipeline_run.yml', 'steps.0.produce/data.csv', 'steps.0.produce/metadata.json', 'steps.1.produce/data.csv', 'steps.1.produce/metadata.json' ]) self._assert_valid_saved_pipeline_runs(pipeline_run_save_path) self._assert_standard_output_metadata() self._assert_prediction_sum(prediction_sum=12106, outputs_path='outputs.0/data.csv') self._assert_nonstandard_output(outputs_name='outputs.1') def test_nonstandard_fit_produce_without_problem(self): pipeline_run_save_path = os.path.join(self.test_dir, 'pipeline_run.yml') arg = [ '', 'runtime', 'fit-produce', '--input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--test-input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--pipeline', os.path.join(PIPELINE_DIR, 'semi-standard-pipeline.json'), '--expose-produced-outputs', self.test_dir, '--not-standard-pipeline', '-O', pipeline_run_save_path, ] self._call_cli_runtime_without_fail(arg) self.assertEqual(utils.list_files(self.test_dir), [ 'outputs.0/data.csv', 'outputs.0/metadata.json', 'outputs.1/data.csv', 'outputs.1/metadata.json', 'pipeline_run.yml', 'steps.0.produce/data.csv', 'steps.0.produce/metadata.json', 'steps.1.produce/data.csv', 'steps.1.produce/metadata.json', ]) self._assert_valid_saved_pipeline_runs(pipeline_run_save_path) self._validate_previous_pipeline_run_ids(pipeline_run_save_path) self._assert_standard_output_metadata() self._assert_prediction_sum(prediction_sum=12106, outputs_path='outputs.0/data.csv') self._assert_nonstandard_output(outputs_name='outputs.1') def test_fit_produce_multi_input(self): pipeline_run_save_path = os.path.join(self.test_dir, 'pipeline_run.yml') arg = [ '', 'runtime', 'fit-produce', '--input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--problem', os.path.join(PROBLEM_DIR, 'iris_problem_1/problemDoc.json'), '--test-input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--test-input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--pipeline', os.path.join(PIPELINE_DIR, 'multi-input-test.json'), '--expose-produced-outputs', self.test_dir, '-O', pipeline_run_save_path, ] self._call_cli_runtime_without_fail(arg) self.assertEqual(utils.list_files(self.test_dir), [ 'outputs.0/data.csv', 'outputs.0/metadata.json', 'pipeline_run.yml', 'steps.0.produce/data.csv', 'steps.0.produce/metadata.json', 'steps.1.produce/data.csv', 'steps.1.produce/metadata.json', 'steps.2.produce/data.csv', 'steps.2.produce/metadata.json', ]) self._assert_valid_saved_pipeline_runs(pipeline_run_save_path) self._validate_previous_pipeline_run_ids(pipeline_run_save_path) self._assert_standard_output_metadata() self._assert_prediction_sum(prediction_sum=11008, outputs_path='outputs.0/data.csv') def test_fit_score(self): pipeline_run_save_path = os.path.join(self.test_dir, 'pipeline_run.yml') arg = [ '', 'runtime', 'fit-score', '--input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--problem', os.path.join(PROBLEM_DIR, 'iris_problem_1/problemDoc.json'), '--test-input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--score-input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--pipeline', os.path.join(PIPELINE_DIR, 'random-forest-classifier.yml'), '--scores', os.path.join(self.test_dir, 'scores.csv'), '-O', pipeline_run_save_path, ] self._call_cli_runtime_without_fail(arg) self._assert_valid_saved_pipeline_runs(pipeline_run_save_path) self._validate_previous_pipeline_run_ids(pipeline_run_save_path) dataframe = pandas.read_csv(os.path.join(self.test_dir, 'scores.csv')) self.assertEqual(list(dataframe.columns), ['metric', 'value', 'normalized', 'randomSeed']) self.assertEqual(dataframe.values.tolist(), [['ACCURACY', 1.0, 1.0, 0]]) def test_fit_score_without_problem(self): pipeline_run_save_path = os.path.join(self.test_dir, 'pipeline_run.yml') arg = [ '', 'runtime', 'fit-score', '--input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--test-input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--score-input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--pipeline', os.path.join(PIPELINE_DIR, 'random-classifier.yml'), '--scoring-pipeline', os.path.join(PIPELINE_DIR, 'fake_compute_score.yml'), # this argument has no effect '--metric', 'F1_MACRO', '--metric', 'ACCURACY', '--scores', os.path.join(self.test_dir, 'scores.csv'), '-O', pipeline_run_save_path, ] logging_records = self._call_cli_runtime_without_fail(arg) self.assertEqual(len(logging_records), 1) self.assertEqual(logging_records[0].msg, "Not all provided hyper-parameters for the scoring pipeline %(pipeline_id)s were used: %(unused_params)s") self._assert_valid_saved_pipeline_runs(pipeline_run_save_path) self._validate_previous_pipeline_run_ids(pipeline_run_save_path) dataframe = pandas.read_csv(os.path.join(self.test_dir, 'scores.csv')) self.assertEqual(list(dataframe.columns), ['metric', 'value', 'normalized', 'randomSeed']) self.assertEqual(dataframe.values.tolist(), [['ACCURACY', 1.0, 1.0, 0]]) @staticmethod def _get_iris_dataset_path(): return os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json') @staticmethod def _get_iris_problem_path(): return os.path.join(PROBLEM_DIR, 'iris_problem_1/problemDoc.json') @staticmethod def _get_random_forest_pipeline_path(): return os.path.join(PIPELINE_DIR, 'random-forest-classifier.yml') @staticmethod def _get_no_split_data_pipeline_path(): return os.path.join(PIPELINE_DIR, 'data-preparation-no-split.yml') @staticmethod def _get_train_test_split_data_pipeline_path(): return os.path.join(PIPELINE_DIR, 'data-preparation-train-test-split.yml') def _get_pipeline_run_save_path(self): return os.path.join(self.test_dir, 'pipeline_run.yml') def _get_predictions_path(self): return os.path.join(self.test_dir, 'predictions.csv') def _get_scores_path(self): return os.path.join(self.test_dir, 'scores.csv') def _get_pipeline_rerun_save_path(self): return os.path.join(self.test_dir, 'pipeline_rerun.yml') def _get_rescores_path(self): return os.path.join(self.test_dir, 'rescores.csv') def _fit_iris_random_forest( self, *, predictions_path=None, fitted_pipeline_path=None, pipeline_run_save_path=None ): if pipeline_run_save_path is None: pipeline_run_save_path = self._get_pipeline_run_save_path() arg = [ '', 'runtime', 'fit', '--input', self._get_iris_dataset_path(), '--problem', self._get_iris_problem_path(), '--pipeline', self._get_random_forest_pipeline_path(), '-O', pipeline_run_save_path ] if predictions_path is not None: arg.append('--output') arg.append(predictions_path) if fitted_pipeline_path is not None: arg.append('--save') arg.append(fitted_pipeline_path) self._call_cli_runtime_without_fail(arg) def _fit_iris_random_classifier_without_problem(self, *, fitted_pipeline_path): pipeline_run_save_path = os.path.join(self.test_dir, 'pipeline_run.yml') arg = [ '', 'runtime', 'fit', '--input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--pipeline', os.path.join(PIPELINE_DIR, 'random-classifier.yml'), '-O', pipeline_run_save_path ] if fitted_pipeline_path is not None: arg.append('--save') arg.append(fitted_pipeline_path) self._call_cli_runtime_without_fail(arg) def test_fit(self): pipeline_run_save_path = self._get_pipeline_run_save_path() fitted_pipeline_path = os.path.join(self.test_dir, 'fitted-pipeline') self._fit_iris_random_forest( fitted_pipeline_path=fitted_pipeline_path, pipeline_run_save_path=pipeline_run_save_path ) self._assert_valid_saved_pipeline_runs(pipeline_run_save_path) self.assertTrue(os.path.isfile(fitted_pipeline_path)) self.assertTrue(os.path.isfile(pipeline_run_save_path)) def test_evaluate(self): pipeline_run_save_path = os.path.join(self.test_dir, 'pipeline_run.yml') scores_path = os.path.join(self.test_dir, 'scores.csv') arg = [ '', 'runtime', 'evaluate', '--input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--problem', os.path.join(PROBLEM_DIR, 'iris_problem_1/problemDoc.json'), '--pipeline', os.path.join(PIPELINE_DIR, 'random-forest-classifier.yml'), '--data-pipeline', os.path.join(PIPELINE_DIR, 'data-preparation-no-split.yml'), '--scores', scores_path, '--metric', 'ACCURACY', '--metric', 'F1_MACRO', '-O', pipeline_run_save_path ] self._call_cli_runtime_without_fail(arg) self._assert_valid_saved_pipeline_runs(pipeline_run_save_path) self._validate_previous_pipeline_run_ids(pipeline_run_save_path) dataframe = pandas.read_csv(scores_path) self.assertEqual(list(dataframe.columns), ['metric', 'value', 'normalized', 'randomSeed', 'fold']) self.assertEqual(dataframe.values.tolist(), [['ACCURACY', 1.0, 1.0, 0, 0], ['F1_MACRO', 1.0, 1.0, 0, 0]]) def test_evaluate_without_problem(self): pipeline_run_save_path = os.path.join(self.test_dir, 'pipeline_run.yml') scores_path = os.path.join(self.test_dir, 'scores.csv') arg = [ '', 'runtime', 'evaluate', '--input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--pipeline', os.path.join(PIPELINE_DIR, 'random-classifier.yml'), '--data-pipeline', os.path.join(PIPELINE_DIR, 'data-preparation-no-split.yml'), '--scoring-pipeline', os.path.join(PIPELINE_DIR, 'fake_compute_score.yml'), # this argument has no effect '--metric', 'ACCURACY', '--scores', scores_path, '-O', pipeline_run_save_path ] logging_records = self._call_cli_runtime_without_fail(arg) self.assertEqual(len(logging_records), 1) self.assertEqual(logging_records[0].msg, "Not all provided hyper-parameters for the scoring pipeline %(pipeline_id)s were used: %(unused_params)s") self._assert_valid_saved_pipeline_runs(pipeline_run_save_path) self._validate_previous_pipeline_run_ids(pipeline_run_save_path) dataframe = pandas.read_csv(scores_path) self.assertEqual(list(dataframe.columns), ['metric', 'value', 'normalized', 'randomSeed', 'fold']) self.assertEqual(dataframe.values.tolist(), [['ACCURACY', 1.0, 1.0, 0, 0]]) def test_score(self): pipeline_run_save_path = os.path.join(self.test_dir, 'pipeline_run.yml') fitted_pipeline_path = os.path.join(self.test_dir, 'iris-pipeline') self._fit_iris_random_forest(fitted_pipeline_path=fitted_pipeline_path) self.assertTrue(os.path.isfile(fitted_pipeline_path)) scores_path = os.path.join(self.test_dir, 'scores.csv') arg = [ '', 'runtime', 'score', '--fitted-pipeline', fitted_pipeline_path, '--test-input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--score-input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--scores', scores_path, '--metric', 'F1_MACRO', '--metric', 'ACCURACY', '-O', pipeline_run_save_path, ] self._call_cli_runtime_without_fail(arg) self._assert_valid_saved_pipeline_runs(pipeline_run_save_path) self.assertTrue(os.path.isfile(scores_path), 'scores were not generated') dataframe = pandas.read_csv(scores_path) self.assertEqual(list(dataframe.columns), ['metric', 'value', 'normalized', 'randomSeed']) self.assertEqual(dataframe.values.tolist(), [['F1_MACRO', 1.0, 1.0, 0], ['ACCURACY', 1.0, 1.0, 0]]) def test_score_without_problem_without_metric(self): pipeline_run_save_path = os.path.join(self.test_dir, 'pipeline_run.yml') fitted_pipeline_path = os.path.join(self.test_dir, 'iris-pipeline') self._fit_iris_random_classifier_without_problem(fitted_pipeline_path=fitted_pipeline_path) self.assertTrue(os.path.isfile(fitted_pipeline_path)) scores_path = os.path.join(self.test_dir, 'scores.csv') arg = [ '', 'runtime', 'score', '--fitted-pipeline', fitted_pipeline_path, '--test-input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--score-input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--scoring-pipeline', os.path.join(PIPELINE_DIR, 'fake_compute_score.yml'), '--scores', scores_path, '-O', pipeline_run_save_path, ] self._call_cli_runtime_without_fail(arg) self._assert_valid_saved_pipeline_runs(pipeline_run_save_path) self.assertTrue(os.path.isfile(scores_path), 'scores were not generated') dataframe = pandas.read_csv(scores_path) self.assertEqual(list(dataframe.columns), ['metric', 'value', 'normalized', 'randomSeed']) self.assertEqual(dataframe.values.tolist(), [['ACCURACY', 1.0, 1.0, 0]]) def test_score_without_problem(self): pipeline_run_save_path = os.path.join(self.test_dir, 'pipeline_run.yml') fitted_pipeline_path = os.path.join(self.test_dir, 'iris-pipeline') self._fit_iris_random_classifier_without_problem(fitted_pipeline_path=fitted_pipeline_path) self.assertTrue(os.path.isfile(fitted_pipeline_path)) scores_path = os.path.join(self.test_dir, 'scores.csv') arg = [ '', 'runtime', 'score', '--fitted-pipeline', fitted_pipeline_path, '--test-input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--score-input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--scoring-pipeline', os.path.join(PIPELINE_DIR, 'fake_compute_score.yml'), # this argument has no effect '--metric', 'ACCURACY', '--scores', scores_path, '-O', pipeline_run_save_path, ] logging_records = self._call_cli_runtime_without_fail(arg) self.assertEqual(len(logging_records), 1) self.assertEqual(logging_records[0].msg, "Not all provided hyper-parameters for the scoring pipeline %(pipeline_id)s were used: %(unused_params)s") self._assert_valid_saved_pipeline_runs(pipeline_run_save_path) self.assertTrue(os.path.isfile(scores_path), 'scores were not generated') dataframe = pandas.read_csv(scores_path) self.assertEqual(list(dataframe.columns), ['metric', 'value', 'normalized', 'randomSeed']) self.assertEqual(dataframe.values.tolist(), [['ACCURACY', 1.0, 1.0, 0]]) def test_produce(self): pipeline_run_save_path = os.path.join(self.test_dir, 'pipeline_run.yml') fitted_pipeline_path = os.path.join(self.test_dir, 'iris-pipeline') self._fit_iris_random_forest(fitted_pipeline_path=fitted_pipeline_path) self.assertTrue(os.path.isfile(fitted_pipeline_path)) arg = [ '', 'runtime', 'produce', '--fitted-pipeline', fitted_pipeline_path, '--test-input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '-O', pipeline_run_save_path, ] self._call_cli_runtime_without_fail(arg) self._assert_valid_saved_pipeline_runs(pipeline_run_save_path) def test_score_predictions(self): predictions_path = os.path.join(self.test_dir, 'predictions.csv') self._fit_iris_random_forest(predictions_path=predictions_path) self.assertTrue(os.path.isfile(predictions_path)) scores_path = os.path.join(self.test_dir, 'scores.csv') arg = [ '', 'runtime', 'score-predictions', '--score-input', os.path.join(DATASET_DIR, 'iris_dataset_1/datasetDoc.json'), '--problem', os.path.join(PROBLEM_DIR, 'iris_problem_1/problemDoc.json'), '--predictions', predictions_path, '--metric', 'ACCURACY', '--metric', 'F1_MACRO', '--scores', scores_path, ] self._call_cli_runtime_without_fail(arg) self.assertTrue(os.path.isfile(scores_path), 'scores were not generated') dataframe = pandas.read_csv(scores_path) self.assertEqual(list(dataframe.columns), ['metric', 'value', 'normalized']) self.assertEqual(dataframe.values.tolist(), [['ACCURACY', 1.0, 1.0], ['F1_MACRO', 1.0, 1.0]]) def test_sklearn_dataset_fit_produce(self): self._create_sklearn_iris_problem_doc() pipeline_run_save_path = os.path.join(self.test_dir, 'pipeline_run.yml') arg = [ '', 'runtime', 'fit-produce', '--input', 'sklearn://iris', '--input', 'sklearn://iris', '--problem', os.path.join(self.test_dir, 'problemDoc.json'), '--test-input', 'sklearn://iris', '--test-input', 'sklearn://iris', '--pipeline', os.path.join(PIPELINE_DIR, 'multi-input-test.json'), '--expose-produced-outputs', self.test_dir, '-O', pipeline_run_save_path, ] self._call_cli_runtime_without_fail(arg) self._assert_valid_saved_pipeline_runs(pipeline_run_save_path) self._validate_previous_pipeline_run_ids(pipeline_run_save_path) self.assertEqual(utils.list_files(self.test_dir), [ 'outputs.0/data.csv', 'outputs.0/metadata.json', 'pipeline_run.yml', 'problemDoc.json', 'steps.0.produce/data.csv', 'steps.0.produce/metadata.json', 'steps.1.produce/data.csv', 'steps.1.produce/metadata.json', 'steps.2.produce/data.csv', 'steps.2.produce/metadata.json' ]) self._assert_standard_output_metadata(prediction_type='numpy.int64') self._assert_prediction_sum(prediction_sum=10648, outputs_path='outputs.0/data.csv') def test_sklearn_dataset_fit_produce_without_problem(self): output_csv_path = os.path.join(self.test_dir, 'output.csv') pipeline_run_save_path = os.path.join(self.test_dir, 'pipeline_run.yml') fitted_pipeline_path = os.path.join(self.test_dir, 'fitted-pipeline') arg = [ '', 'runtime', 'fit-produce', '--input', 'sklearn://iris', '--test-input', 'sklearn://iris', '--pipeline', os.path.join(PIPELINE_DIR, 'random-classifier.yml'), '--save', fitted_pipeline_path, '--output', output_csv_path, '--expose-produced-outputs', self.test_dir, '-O', pipeline_run_save_path, ] self._call_cli_runtime_without_fail(arg) self._assert_valid_saved_pipeline_runs(pipeline_run_save_path) self._validate_previous_pipeline_run_ids(pipeline_run_save_path) self.assertEqual(utils.list_files(self.test_dir), [ 'fitted-pipeline', 'output.csv', 'outputs.0/data.csv', 'outputs.0/metadata.json', 'pipeline_run.yml', 'steps.0.produce/data.csv', 'steps.0.produce/metadata.json', 'steps.1.produce/data.csv', 'steps.1.produce/metadata.json', 'steps.2.produce/data.csv', 'steps.2.produce/metadata.json', ]) self._assert_standard_output_metadata(prediction_type='numpy.int64') self._assert_prediction_sum(prediction_sum=10648, outputs_path='outputs.0/data.csv') self._assert_prediction_sum(prediction_sum=10648, outputs_path='output.csv') def _create_sklearn_iris_problem_doc(self): with open(os.path.join(PROBLEM_DIR, 'iris_problem_1/problemDoc.json'), 'r', encoding='utf8') as problem_doc_file: problem_doc = json.load(problem_doc_file) problem_doc['inputs']['data'][0]['datasetID'] = 'sklearn://iris' with open(os.path.join(self.test_dir, 'problemDoc.json'), 'x', encoding='utf8') as problem_doc_file: json.dump(problem_doc, problem_doc_file) def test_sklearn_dataset_evaluate(self): self._create_sklearn_iris_problem_doc() pipeline_run_save_path = os.path.join(self.test_dir, 'pipeline_run.yml') scores_path = os.path.join(self.test_dir, 'scores.csv') arg = [ '', 'runtime', 'evaluate', '--input', 'sklearn://iris', '--problem', os.path.join(self.test_dir, 'problemDoc.json'), '--pipeline', os.path.join(PIPELINE_DIR, 'random-forest-classifier.yml'), '--data-pipeline', os.path.join(PIPELINE_DIR, 'data-preparation-no-split.yml'), '--scores', scores_path, '--metric', 'ACCURACY', '--metric', 'F1_MACRO', '-O', pipeline_run_save_path ] self._call_cli_runtime_without_fail(arg) self._assert_valid_saved_pipeline_runs(pipeline_run_save_path) self._validate_previous_pipeline_run_ids(pipeline_run_save_path) dataframe = pandas.read_csv(scores_path) self.assertEqual(list(dataframe.columns), ['metric', 'value', 'normalized', 'randomSeed', 'fold']) self.assertEqual(dataframe.values.tolist(), [['ACCURACY', 1.0, 1.0, 0, 0], ['F1_MACRO', 1.0, 1.0, 0, 0]]) def test_sklearn_dataset_evaluate_without_problem(self): pipeline_run_save_path = os.path.join(self.test_dir, 'pipeline_run.yml') scores_path = os.path.join(self.test_dir, 'scores.csv') arg = [ '', 'runtime', 'evaluate', '--input', 'sklearn://iris', '--pipeline', os.path.join(PIPELINE_DIR, 'random-classifier.yml'), '--data-pipeline', os.path.join(PIPELINE_DIR, 'data-preparation-no-split.yml'), '--scoring-pipeline', os.path.join(PIPELINE_DIR, 'fake_compute_score.yml'), # this argument has no effect '--metric', 'ACCURACY', '--scores', scores_path, '-O', pipeline_run_save_path ] logging_records = self._call_cli_runtime_without_fail(arg) self.assertEqual(len(logging_records), 1) self.assertEqual(logging_records[0].msg, "Not all provided hyper-parameters for the scoring pipeline %(pipeline_id)s were used: %(unused_params)s") self._assert_valid_saved_pipeline_runs(pipeline_run_save_path) self._validate_previous_pipeline_run_ids(pipeline_run_save_path) dataframe = pandas.read_csv(scores_path) self.assertEqual(list(dataframe.columns), ['metric', 'value', 'normalized', 'randomSeed', 'fold']) self.assertEqual(dataframe.values.tolist(), [['ACCURACY', 1.0, 1.0, 0, 0]]) def _assert_prediction_sum(self, prediction_sum, outputs_path): if prediction_sum is not None: with open(os.path.join(self.test_dir, outputs_path), 'r') as csv_file: self.assertEqual(sum([int(v) for v in list(csv_file)[1:]]), prediction_sum) def _assert_standard_output_metadata(self, outputs_name='outputs.0', prediction_type='str'): with open(os.path.join(self.test_dir, outputs_name, 'metadata.json'), 'r') as metadata_file: metadata = json.load(metadata_file) self.assertEqual( metadata, [ { "selector": [], "metadata": { "dimension": { "length": 150, "name": "rows", "semantic_types": ["https://metadata.datadrivendiscovery.org/types/TabularRow"], }, "schema": "https://metadata.datadrivendiscovery.org/schemas/v0/container.json", "semantic_types": ["https://metadata.datadrivendiscovery.org/types/Table"], "structural_type": "d3m.container.pandas.DataFrame", }, }, { "selector": ["__ALL_ELEMENTS__"], "metadata": { "dimension": { "length": 1, "name": "columns", "semantic_types": ["https://metadata.datadrivendiscovery.org/types/TabularColumn"], } }, }, {"selector": ["__ALL_ELEMENTS__", 0], "metadata": {"name": "predictions", "structural_type": prediction_type}}, ], ) def _assert_nonstandard_output(self, outputs_name='outputs.1'): with open(os.path.join(self.test_dir, outputs_name, 'data.csv'), 'r') as csv_file: output_dataframe = pandas.read_csv(csv_file, index_col=False) learning_dataframe = pandas.read_csv( os.path.join(DATASET_DIR, 'iris_dataset_1/tables/learningData.csv'), index_col=False) self.assertTrue(learning_dataframe.equals(output_dataframe)) with open(os.path.join(self.test_dir, outputs_name, 'metadata.json'), 'r') as metadata_file: metadata = json.load(metadata_file) self.assertEqual( metadata, [ { "metadata": { "dimension": { "length": 150, "name": "rows", "semantic_types": [ "https://metadata.datadrivendiscovery.org/types/TabularRow" ] }, "schema": "https://metadata.datadrivendiscovery.org/schemas/v0/container.json", "semantic_types": [ "https://metadata.datadrivendiscovery.org/types/Table" ], "structural_type": "d3m.container.pandas.DataFrame" }, "selector": [] }, { "metadata": { "dimension": { "length": 6, "name": "columns", "semantic_types": [ "https://metadata.datadrivendiscovery.org/types/TabularColumn" ] } }, "selector": [ "__ALL_ELEMENTS__" ] }, { "metadata": { "name": "d3mIndex", "semantic_types": [ "http://schema.org/Integer", "https://metadata.datadrivendiscovery.org/types/PrimaryKey" ], "structural_type": "str" }, "selector": [ "__ALL_ELEMENTS__", 0 ] }, { "metadata": { "name": "sepalLength", "semantic_types": [ "http://schema.org/Float", "https://metadata.datadrivendiscovery.org/types/Attribute" ], "structural_type": "str" }, "selector": [ "__ALL_ELEMENTS__", 1 ] }, { "metadata": { "name": "sepalWidth", "semantic_types": [ "http://schema.org/Float", "https://metadata.datadrivendiscovery.org/types/Attribute" ], "structural_type": "str" }, "selector": [ "__ALL_ELEMENTS__", 2 ] }, { "metadata": { "name": "petalLength", "semantic_types": [ "http://schema.org/Float", "https://metadata.datadrivendiscovery.org/types/Attribute" ], "structural_type": "str" }, "selector": [ "__ALL_ELEMENTS__", 3 ] }, { "metadata": { "name": "petalWidth", "semantic_types": [ "http://schema.org/Float", "https://metadata.datadrivendiscovery.org/types/Attribute" ], "structural_type": "str" }, "selector": [ "__ALL_ELEMENTS__", 4 ] }, { "metadata": { "name": "species", "semantic_types": [ "https://metadata.datadrivendiscovery.org/types/CategoricalData", "https://metadata.datadrivendiscovery.org/types/SuggestedTarget", "https://metadata.datadrivendiscovery.org/types/Attribute" ], "structural_type": "str" }, "selector": [ "__ALL_ELEMENTS__", 5 ] } ] ) def _assert_pipeline_runs_equal(self, pipeline_run_save_path1, pipeline_run_save_path2): with open(pipeline_run_save_path1, 'r') as f: pipeline_runs1 = list(utils.yaml_load_all(f)) with open(pipeline_run_save_path2, 'r') as f: pipeline_runs2 = list(utils.yaml_load_all(f)) self.assertEqual(len(pipeline_runs1), len(pipeline_runs2)) for pipeline_run1, pipeline_run2 in zip(pipeline_runs1, pipeline_runs2): self.assertTrue(pipeline_run_module.PipelineRun.json_structure_equals(pipeline_run1, pipeline_run2)) def test_pipeline_run_json_structure_equals(self): pipeline_run_save_path1 = os.path.join(self.test_dir, 'pipeline_run1.yml') self._fit_iris_random_forest(pipeline_run_save_path=pipeline_run_save_path1) self._assert_valid_saved_pipeline_runs(pipeline_run_save_path1) pipeline_run_save_path2 = os.path.join(self.test_dir, 'pipeline_run2.yml') self._fit_iris_random_forest(pipeline_run_save_path=pipeline_run_save_path2) self._assert_valid_saved_pipeline_runs(pipeline_run_save_path2) self._assert_pipeline_runs_equal(pipeline_run_save_path1, pipeline_run_save_path2) def _cache_pipeline_for_rerun(self, pipeline_path, cache_dir=None): """make pipeline searchable by id in test_dir""" with open(pipeline_path, 'r') as f: pipeline = utils.yaml_load(f) if cache_dir is None: cache_dir = self.test_dir temp_pipeline_path = os.path.join(cache_dir, pipeline['id'] + '.yml') with open(temp_pipeline_path, 'w') as f: utils.yaml_dump(pipeline, f) @staticmethod def _generate_seed(): return random.randint(2**31, 2**32-1) def test_fit_rerun(self): dataset_path = self._get_iris_dataset_path() problem_path = self._get_iris_problem_path() pipeline_path = self._get_random_forest_pipeline_path() pipeline_run_save_path = self._get_pipeline_run_save_path() problem = problem_module.get_problem(problem_path) inputs = [dataset_module.get_dataset(dataset_path)] with open(pipeline_path) as f: pipeline = pipeline_module.Pipeline.from_yaml(f) hyperparams = [{}, {}, {'n_estimators': 19}, {}] random_seed = self._generate_seed() with utils.silence(): fitted_pipeline, predictions, fit_result = runtime.fit( pipeline, inputs, problem_description=problem, hyperparams=hyperparams, random_seed=random_seed, context=metadata_base.Context.TESTING, ) with open(pipeline_run_save_path, 'w') as f: fit_result.pipeline_run.to_yaml(f) self._cache_pipeline_for_rerun(pipeline_path) pipeline_rerun_save_path = self._get_pipeline_rerun_save_path() rerun_arg = [ '', '--pipelines-path', self.test_dir, 'runtime', '--datasets', TEST_DATA_DIR, 'fit', '--input-run', pipeline_run_save_path, '--output-run', pipeline_rerun_save_path, ] self._call_cli_runtime_without_fail(rerun_arg) self._assert_valid_saved_pipeline_runs(pipeline_rerun_save_path) self._assert_pipeline_runs_equal(pipeline_run_save_path, pipeline_rerun_save_path) def test_produce_rerun(self): dataset_path = self._get_iris_dataset_path() problem_path = self._get_iris_problem_path() pipeline_path = self._get_random_forest_pipeline_path() pipeline_run_save_path = self._get_pipeline_run_save_path() fitted_pipeline_path = os.path.join(self.test_dir, 'iris-pipeline') self._fit_iris_random_forest(fitted_pipeline_path=fitted_pipeline_path) self.assertTrue(os.path.isfile(fitted_pipeline_path)) arg = [ '', 'runtime', 'produce', '--fitted-pipeline', fitted_pipeline_path, '--test-input', dataset_path, '--output-run', pipeline_run_save_path, ] self._call_cli_runtime_without_fail(arg) self._assert_valid_saved_pipeline_runs(pipeline_run_save_path) self._cache_pipeline_for_rerun(pipeline_path) pipeline_rerun_save_path = self._get_pipeline_rerun_save_path() rerun_arg = [ '', '--pipelines-path', self.test_dir, 'runtime', '--datasets', TEST_DATA_DIR, 'produce', '--fitted-pipeline', fitted_pipeline_path, '--input-run', pipeline_run_save_path, '--output-run', pipeline_rerun_save_path, ] self._call_cli_runtime_without_fail(rerun_arg) self._assert_valid_saved_pipeline_runs(pipeline_rerun_save_path) self._assert_pipeline_runs_equal(pipeline_run_save_path, pipeline_rerun_save_path) def _assert_scores_equal(self, scores_path, rescores_path): scores = pandas.read_csv(scores_path) rescores = pandas.read_csv(rescores_path) self.assertTrue(scores.equals(rescores), '\n{}\n\n{}'.format(scores, rescores)) def _assert_scores_equal_pipeline_run(self, scores_path, pipeline_run_save_path): scores = pandas.read_csv(scores_path) scores.drop('fold', axis=1, inplace=True, errors='ignore') scores_no_seed = scores.drop('randomSeed', axis=1, errors='ignore') with open(pipeline_run_save_path) as f: # TODO: always use -1? pipeline_run = list(utils.yaml_load_all(f))[-1] self.assertEqual(pipeline_run['run']['phase'], metadata_base.PipelineRunPhase.PRODUCE.name) # TODO: clean up preprocessing? pipeline_run_scores_df = pandas.DataFrame(pipeline_run['run']['results']['scores']) # TODO: is it possible to make pipeline run schema more compatible with scores csv schema? pipeline_run_scores_df['metric'] = pipeline_run_scores_df['metric'].map(lambda cell: cell['metric']) pipeline_run_scores_df = pipeline_run_scores_df[scores_no_seed.columns.tolist()] pandas.testing.assert_frame_equal(scores_no_seed, pipeline_run_scores_df) self.assertEqual(scores['randomSeed'].iloc[0], pipeline_run['random_seed']) def test_score_rerun(self): dataset_path = self._get_iris_dataset_path() problem_path = self._get_iris_problem_path() pipeline_path = self._get_random_forest_pipeline_path() pipeline_run_save_path = self._get_pipeline_run_save_path() fitted_pipeline_path = os.path.join(self.test_dir, 'iris-pipeline') scores_path = os.path.join(self.test_dir, 'scores.csv') random_seed = self._generate_seed() metrics = runtime.get_metrics_from_list(['ACCURACY', 'F1_MACRO']) scoring_params = {'add_normalized_scores': 'false'} scoring_random_seed = self._generate_seed() problem = problem_module.get_problem(problem_path) inputs = [dataset_module.get_dataset(dataset_path)] with open(pipeline_path) as f: pipeline = pipeline_module.Pipeline.from_yaml(f) with open(runtime.DEFAULT_SCORING_PIPELINE_PATH) as f: scoring_pipeline = pipeline_module.Pipeline.from_yaml(f) with utils.silence(): fitted_pipeline, predictions, fit_result = runtime.fit( pipeline, inputs, problem_description=problem, random_seed=random_seed, context=metadata_base.Context.TESTING, ) with open(fitted_pipeline_path, 'wb') as f: pickle.dump(fitted_pipeline, f) predictions, produce_result = runtime.produce(fitted_pipeline, inputs) scores, score_result = runtime.score( predictions, inputs, scoring_pipeline=scoring_pipeline, problem_description=problem, metrics=metrics, predictions_random_seed=random_seed, context=metadata_base.Context.TESTING, scoring_params=scoring_params, random_seed=scoring_random_seed ) self.assertFalse(score_result.has_error(), score_result.error) scores.to_csv(scores_path) runtime.combine_pipeline_runs( produce_result.pipeline_run, scoring_pipeline_run=score_result.pipeline_run, score_inputs=inputs, metrics=metrics, scores=scores ) with open(pipeline_run_save_path, 'w') as f: produce_result.pipeline_run.to_yaml(f) self.assertTrue(os.path.isfile(fitted_pipeline_path)) self.assertTrue(os.path.isfile(scores_path), 'scores were not generated') self._assert_valid_saved_pipeline_runs(pipeline_run_save_path) dataframe = pandas.read_csv(scores_path) self.assertEqual(list(dataframe.columns), ['metric', 'value', 'randomSeed']) self.assertEqual(dataframe.values.tolist(), [['ACCURACY', 1.0, random_seed], ['F1_MACRO', 1.0, random_seed]]) self._cache_pipeline_for_rerun(pipeline_path) pipeline_rerun_save_path = self._get_pipeline_rerun_save_path() rescores_path = self._get_rescores_path() rerun_arg = [ '', '--pipelines-path', self.test_dir, 'runtime', '--datasets', TEST_DATA_DIR, 'score', '--fitted-pipeline', fitted_pipeline_path, '--input-run', pipeline_run_save_path, '--output-run', pipeline_rerun_save_path, '--scores', rescores_path, ] self._call_cli_runtime_without_fail(rerun_arg) self.assertTrue(os.path.isfile(pipeline_rerun_save_path)) self._assert_valid_saved_pipeline_runs(pipeline_rerun_save_path) self._assert_scores_equal(scores_path, rescores_path) self._assert_scores_equal_pipeline_run(scores_path, pipeline_rerun_save_path) self._assert_pipeline_runs_equal(pipeline_run_save_path, pipeline_rerun_save_path) def test_fit_produce_rerun(self): dataset_path = self._get_iris_dataset_path() problem_path = self._get_iris_problem_path() pipeline_path = self._get_random_forest_pipeline_path() pipeline_run_save_path = self._get_pipeline_run_save_path() hyperparams = [{}, {}, {'n_estimators': 19}, {}] random_seed = self._generate_seed() problem = problem_module.get_problem(problem_path) inputs = [dataset_module.get_dataset(dataset_path)] with open(pipeline_path) as f: pipeline = pipeline_module.Pipeline.from_yaml(f) with utils.silence(): fitted_pipeline, predictions, fit_result = runtime.fit( pipeline, inputs, problem_description=problem, hyperparams=hyperparams, random_seed=random_seed, context=metadata_base.Context.TESTING, ) predictions, produce_result = runtime.produce(fitted_pipeline, inputs) with open(pipeline_run_save_path, 'w') as f: fit_result.pipeline_run.to_yaml(f) produce_result.pipeline_run.to_yaml(f, appending=True) self._cache_pipeline_for_rerun(pipeline_path) pipeline_rerun_save_path = self._get_pipeline_rerun_save_path() rerun_arg = [ '', '--pipelines-path', self.test_dir, '--strict-digest', 'runtime', '--datasets', TEST_DATA_DIR, 'fit-produce', '--input-run', pipeline_run_save_path, '--output-run', pipeline_rerun_save_path, ] self._call_cli_runtime_without_fail(rerun_arg) self._assert_valid_saved_pipeline_runs(pipeline_rerun_save_path) self._assert_pipeline_runs_equal(pipeline_run_save_path, pipeline_rerun_save_path) def test_fit_score_rerun(self): dataset_path = self._get_iris_dataset_path() problem_path = self._get_iris_problem_path() pipeline_path = self._get_random_forest_pipeline_path() pipeline_run_save_path = self._get_pipeline_run_save_path() scores_path = self._get_scores_path() hyperparams = [{}, {}, {'n_estimators': 19}, {}] random_seed = self._generate_seed() metrics = runtime.get_metrics_from_list(['ACCURACY', 'F1_MACRO']) scoring_params = {'add_normalized_scores': 'false'} scoring_random_seed = self._generate_seed() problem = problem_module.get_problem(problem_path) inputs = [dataset_module.get_dataset(dataset_path)] with open(pipeline_path) as f: pipeline = pipeline_module.Pipeline.from_yaml(f) with open(runtime.DEFAULT_SCORING_PIPELINE_PATH) as f: scoring_pipeline = pipeline_module.Pipeline.from_yaml(f) with utils.silence(): fitted_pipeline, predictions, fit_result = runtime.fit( pipeline, inputs, problem_description=problem, hyperparams=hyperparams, random_seed=random_seed, context=metadata_base.Context.TESTING, ) self.assertFalse(fit_result.has_error(), fit_result.error) predictions, produce_result = runtime.produce(fitted_pipeline, inputs) self.assertFalse(produce_result.has_error(), produce_result.error) scores, score_result = runtime.score( predictions, inputs, scoring_pipeline=scoring_pipeline, problem_description=problem, metrics=metrics, predictions_random_seed=fitted_pipeline.random_seed, context=metadata_base.Context.TESTING, scoring_params=scoring_params, random_seed=scoring_random_seed ) self.assertFalse(score_result.has_error(), score_result.error) scores.to_csv(scores_path) runtime.combine_pipeline_runs( produce_result.pipeline_run, scoring_pipeline_run=score_result.pipeline_run, score_inputs=inputs, metrics=metrics, scores=scores ) with open(pipeline_run_save_path, 'w') as f: fit_result.pipeline_run.to_yaml(f) produce_result.pipeline_run.to_yaml(f, appending=True) self._assert_valid_saved_pipeline_runs(pipeline_run_save_path) self._cache_pipeline_for_rerun(pipeline_path) pipeline_rerun_save_path = self._get_pipeline_rerun_save_path() rescores_path = self._get_rescores_path() rerun_arg = [ '', '--pipelines-path', self.test_dir, '--strict-digest', 'runtime', '--datasets', TEST_DATA_DIR, 'fit-score', '--input-run', pipeline_run_save_path, '--scores', rescores_path, '--output-run', pipeline_rerun_save_path, ] self._call_cli_runtime_without_fail(rerun_arg) self._assert_valid_saved_pipeline_runs(pipeline_rerun_save_path) self._assert_scores_equal(scores_path, rescores_path) self._assert_scores_equal_pipeline_run(scores_path, pipeline_rerun_save_path) self._assert_pipeline_runs_equal(pipeline_run_save_path, pipeline_rerun_save_path) def test_evaluate_rerun(self): dataset_path = self._get_iris_dataset_path() problem_path = self._get_iris_problem_path() pipeline_path = self._get_random_forest_pipeline_path() data_pipeline_path = self._get_train_test_split_data_pipeline_path() pipeline_run_save_path = self._get_pipeline_run_save_path() scores_path = self._get_scores_path() hyperparams = [{}, {}, {'n_estimators': 19}, {}] random_seed = self._generate_seed() metrics = runtime.get_metrics_from_list(['ACCURACY', 'F1_MACRO']) scoring_params = {'add_normalized_scores': 'false'} scoring_random_seed = self._generate_seed() data_params = {'shuffle': 'true', 'stratified': 'true', 'train_score_ratio': '0.59'} data_random_seed = self._generate_seed() problem = problem_module.get_problem(problem_path) inputs = [dataset_module.get_dataset(dataset_path)] with open(pipeline_path) as f: pipeline = pipeline_module.Pipeline.from_yaml(f) with open(data_pipeline_path) as f: data_pipeline = pipeline_module.Pipeline.from_yaml(f) with open(runtime.DEFAULT_SCORING_PIPELINE_PATH) as f: scoring_pipeline = pipeline_module.Pipeline.from_yaml(f) with utils.silence(): dummy_runtime_environment = pipeline_run_module.RuntimeEnvironment(worker_id='dummy worker id') all_scores, all_results = runtime.evaluate( pipeline, inputs, data_pipeline=data_pipeline, scoring_pipeline=scoring_pipeline, problem_description=problem, data_params=data_params, metrics=metrics, context=metadata_base.Context.TESTING, scoring_params=scoring_params, hyperparams=hyperparams, random_seed=random_seed, data_random_seed=data_random_seed, scoring_random_seed=scoring_random_seed, runtime_environment=dummy_runtime_environment, ) self.assertEqual(len(all_scores), 1) scores = runtime.combine_folds(all_scores) scores.to_csv(scores_path) if any(result.has_error() for result in all_results): self.fail([result.error for result in all_results if result.has_error()][0]) with open(pipeline_run_save_path, 'w') as f: for i, pipeline_run in enumerate(all_results.pipeline_runs): pipeline_run.to_yaml(f, appending=i>0) self._cache_pipeline_for_rerun(pipeline_path) self._cache_pipeline_for_rerun(data_pipeline_path) pipeline_rerun_save_path = self._get_pipeline_rerun_save_path() rescores_path = self._get_rescores_path() rerun_arg = [ '', '--pipelines-path', self.test_dir, 'runtime', '--datasets', TEST_DATA_DIR, 'evaluate', '--input-run', pipeline_run_save_path, '--output-run', pipeline_rerun_save_path, '--scores', rescores_path, ] self._call_cli_runtime_without_fail(rerun_arg) self._assert_valid_saved_pipeline_runs(pipeline_rerun_save_path) self._assert_scores_equal(scores_path, rescores_path) self._assert_scores_equal_pipeline_run(scores_path, pipeline_rerun_save_path) self._assert_pipeline_runs_equal(pipeline_run_save_path, pipeline_rerun_save_path) # See: https://gitlab.com/datadrivendiscovery/d3m/issues/406 # TODO: Test rerun validation code (that we throw exceptions on invalid pipeline runs). # TODO: Test rerun with multiple inputs (non-standard pipeline). # TODO: Test rerun without problem description. # TODO: Test evaluate rerun with data split file. def test_validate_gzipped_pipeline_run(self): # First, generate the pipeline run file pipeline_run_save_path = self._get_pipeline_run_save_path() gzip_pipeline_run_save_path = '{pipeline_run_save_path}.gz'.format(pipeline_run_save_path=pipeline_run_save_path) fitted_pipeline_path = os.path.join(self.test_dir, 'fitted-pipeline') self._fit_iris_random_forest( fitted_pipeline_path=fitted_pipeline_path, pipeline_run_save_path=pipeline_run_save_path ) # Second, gzip the pipeline run file with open(pipeline_run_save_path, 'rb') as file_in: with gzip.open(gzip_pipeline_run_save_path, 'wb') as file_out: shutil.copyfileobj(file_in, file_out) os.remove(pipeline_run_save_path) # Third, ensure that calling 'pipeline-run validate' on the gzipped pipeline run file is successful arg = [ '', 'pipeline-run', 'validate', gzip_pipeline_run_save_path, ] self._call_cli_runtime_without_fail(arg) def test_help_message(self): arg = [ '', 'runtime', 'fit', '--version', ] with io.StringIO() as buffer: with contextlib.redirect_stderr(buffer): with self.assertRaises(SystemExit): cli.main(arg) help = buffer.getvalue() self.assertTrue('usage: d3m runtime fit' in help, help) if __name__ == '__main__': unittest.main()