From 8f86fd99b06b0bc93cd689a41c5214ed8c1ed326 Mon Sep 17 00:00:00 2001 From: Jaikiran Pai Date: Fri, 17 Aug 2018 16:44:47 +0530 Subject: [PATCH] [junitlauncher] add more fine grained tests for junitlauncher task --- .../taskdefs/optional/junitlauncher.xml | 56 +++++- .../junitlauncher/JUnitLauncherTaskTest.java | 161 ++++++++++++------ .../org/example/junitlauncher/Tracker.java | 136 +++++++++++++++ .../junitlauncher/vintage/ForkedTest.java | 37 ++++ 4 files changed, 335 insertions(+), 55 deletions(-) create mode 100644 src/tests/junit/org/example/junitlauncher/Tracker.java create mode 100644 src/tests/junit/org/example/junitlauncher/vintage/ForkedTest.java diff --git a/src/etc/testcases/taskdefs/optional/junitlauncher.xml b/src/etc/testcases/taskdefs/optional/junitlauncher.xml index 0facaddf2..35b2c3f36 100644 --- a/src/etc/testcases/taskdefs/optional/junitlauncher.xml +++ b/src/etc/testcases/taskdefs/optional/junitlauncher.xml @@ -50,6 +50,11 @@ + + @@ -59,6 +64,11 @@ + + @@ -67,6 +77,10 @@ + @@ -75,6 +89,11 @@ + + @@ -82,6 +101,11 @@ + + @@ -90,8 +114,18 @@ - - + + + + + + @@ -100,8 +134,14 @@ + + + @@ -112,8 +152,16 @@ - - + + + + + + + diff --git a/src/tests/junit/org/apache/tools/ant/taskdefs/optional/junitlauncher/JUnitLauncherTaskTest.java b/src/tests/junit/org/apache/tools/ant/taskdefs/optional/junitlauncher/JUnitLauncherTaskTest.java index c17ca57b3..8c3fbf2c0 100644 --- a/src/tests/junit/org/apache/tools/ant/taskdefs/optional/junitlauncher/JUnitLauncherTaskTest.java +++ b/src/tests/junit/org/apache/tools/ant/taskdefs/optional/junitlauncher/JUnitLauncherTaskTest.java @@ -17,15 +17,25 @@ */ package org.apache.tools.ant.taskdefs.optional.junitlauncher; -import org.apache.tools.ant.BuildEvent; import org.apache.tools.ant.BuildException; import org.apache.tools.ant.BuildFileRule; -import org.apache.tools.ant.BuildListener; -import org.apache.tools.ant.Project; +import org.example.junitlauncher.jupiter.JupiterSampleTest; +import org.example.junitlauncher.vintage.AlwaysFailingJUnit4Test; +import org.example.junitlauncher.vintage.ForkedTest; +import org.example.junitlauncher.vintage.JUnit4SampleTest; +import org.junit.Assert; import org.junit.Before; import org.junit.Rule; import org.junit.Test; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static org.example.junitlauncher.Tracker.verifyFailed; +import static org.example.junitlauncher.Tracker.verifySkipped; +import static org.example.junitlauncher.Tracker.verifySuccess; +import static org.example.junitlauncher.Tracker.wasTestRun; + /** * Tests the {@link JUnitLauncherTask} */ @@ -40,47 +50,27 @@ public class JUnitLauncherTaskTest { @Before public void setUp() { buildRule.configureProject("src/etc/testcases/taskdefs/optional/junitlauncher.xml"); - buildRule.getProject().addBuildListener(new BuildListener() { - @Override - public void buildStarted(final BuildEvent event) { - } - - @Override - public void buildFinished(final BuildEvent event) { - } - - @Override - public void targetStarted(final BuildEvent event) { - } - - @Override - public void targetFinished(final BuildEvent event) { - } - - @Override - public void taskStarted(final BuildEvent event) { - } - - @Override - public void taskFinished(final BuildEvent event) { - } - - @Override - public void messageLogged(final BuildEvent event) { - if (event.getPriority() <= Project.MSG_INFO) { - System.out.println(event.getMessage()); - } - } - }); } /** * Tests that when a test, that's configured with {@code haltOnFailure=true}, stops the build, when the * test fails */ - @Test(expected = BuildException.class) - public void testFailureStopsBuild() { - buildRule.executeTarget("test-failure-stops-build"); + @Test + public void testFailureStopsBuild() throws Exception { + final String targetName = "test-failure-stops-build"; + final Path trackerFile = setupTrackerProperty(targetName); + try { + buildRule.executeTarget(targetName); + Assert.fail(targetName + " was expected to fail"); + } catch (BuildException e) { + // expected, but do further tests to make sure the build failed for expected reason + if (!verifyFailed(trackerFile, AlwaysFailingJUnit4Test.class.getName(), + "testWillFail")) { + // throw back the original cause + throw e; + } + } } /** @@ -88,33 +78,71 @@ public class JUnitLauncherTaskTest { * build even when there are test failures */ @Test - public void testFailureContinuesBuild() { - buildRule.executeTarget("test-failure-continues-build"); + public void testFailureContinuesBuild() throws Exception { + final String targetName = "test-failure-continues-build"; + final Path trackerFile = setupTrackerProperty(targetName); + buildRule.executeTarget(targetName); + // make sure the test that was expected to be run (and fail), did indeed fail + Assert.assertTrue("AlwaysFailingJUnit4Test#testWillFail was expected to run", wasTestRun(trackerFile, + AlwaysFailingJUnit4Test.class.getName(), "testWillFail")); + Assert.assertTrue("AlwaysFailingJUnit4Test#testWillFail was expected to fail", verifyFailed(trackerFile, + AlwaysFailingJUnit4Test.class.getName(), "testWillFail")); } /** * Tests the execution of test that's expected to succeed */ @Test - public void testSuccessfulTests() { - buildRule.executeTarget("test-success"); + public void testSuccessfulTests() throws Exception { + final String targetName = "test-success"; + final Path trackerFile = setupTrackerProperty(targetName); + buildRule.executeTarget(targetName); + // make sure the right test(s) were run + Assert.assertTrue("JUnit4SampleTest test was expected to be run", wasTestRun(trackerFile, JUnit4SampleTest.class.getName())); + Assert.assertTrue("JUnit4SampleTest#testFoo was expected to succeed", verifySuccess(trackerFile, + JUnit4SampleTest.class.getName(), "testFoo")); } /** * Tests execution of a test which is configured to execute only a particular set of test methods */ @Test - public void testSpecificMethodTest() { - buildRule.executeTarget("test-one-specific-method"); - buildRule.executeTarget("test-multiple-specific-methods"); + public void testSpecificMethodTest() throws Exception { + final String targetSpecificMethod = "test-one-specific-method"; + final Path tracker1 = setupTrackerProperty(targetSpecificMethod); + buildRule.executeTarget(targetSpecificMethod); + // verify only that specific method was run + Assert.assertTrue("testBar was expected to be run", wasTestRun(tracker1, JUnit4SampleTest.class.getName(), + "testBar")); + Assert.assertFalse("testFoo wasn't expected to be run", wasTestRun(tracker1, JUnit4SampleTest.class.getName(), + "testFoo")); + + + final String targetMultipleMethods = "test-multiple-specific-methods"; + final Path tracker2 = setupTrackerProperty(targetMultipleMethods); + buildRule.executeTarget(targetMultipleMethods); + Assert.assertTrue("testFooBar was expected to be run", wasTestRun(tracker2, JUnit4SampleTest.class.getName(), + "testFooBar")); + Assert.assertTrue("testFoo was expected to be run", wasTestRun(tracker2, JUnit4SampleTest.class.getName(), + "testFoo")); + Assert.assertFalse("testBar wasn't expected to be run", wasTestRun(tracker2, JUnit4SampleTest.class.getName(), + "testBar")); } /** * Tests the execution of more than one {@code <test>} elements in the {@code <junitlauncher>} task */ @Test - public void testMultipleIndividualTests() { - buildRule.executeTarget("test-multiple-individual"); + public void testMultipleIndividualTests() throws Exception { + final String targetName = "test-multiple-individual"; + final Path trackerFile1 = setupTrackerProperty(targetName + "-1"); + final Path trackerFile2 = setupTrackerProperty(targetName + "-2"); + buildRule.executeTarget(targetName); + + Assert.assertTrue("AlwaysFailingJUnit4Test#testWillFail was expected to be run", wasTestRun(trackerFile1, + AlwaysFailingJUnit4Test.class.getName(), "testWillFail")); + Assert.assertTrue("JUnit4SampleTest#testFoo was expected to be run", wasTestRun(trackerFile2, + JUnit4SampleTest.class.getName(), "testFoo")); } /** @@ -122,16 +150,47 @@ public class JUnitLauncherTaskTest { * of the {@code <junitlauncher>} task */ @Test - public void testTestClasses() { - buildRule.executeTarget("test-batch"); + public void testTestClasses() throws Exception { + final String targetName = "test-batch"; + final Path trackerFile = setupTrackerProperty(targetName); + buildRule.executeTarget(targetName); + + Assert.assertTrue("JUnit4SampleTest#testFoo was expected to succeed", verifySuccess(trackerFile, + JUnit4SampleTest.class.getName(), "testFoo")); + Assert.assertTrue("AlwaysFailingJUnit4Test#testWillFail was expected to fail", verifyFailed(trackerFile, + AlwaysFailingJUnit4Test.class.getName(), "testWillFail")); + Assert.assertTrue("JupiterSampleTest#testSucceeds was expected to succeed", verifySuccess(trackerFile, + JupiterSampleTest.class.getName(), "testSucceeds")); + Assert.assertTrue("JupiterSampleTest#testFails was expected to succeed", verifyFailed(trackerFile, + JupiterSampleTest.class.getName(), "testFails")); + Assert.assertTrue("JupiterSampleTest#testSkipped was expected to be skipped", verifySkipped(trackerFile, + JupiterSampleTest.class.getName(), "testSkipped")); + Assert.assertFalse("ForkedTest wasn't expected to be run", wasTestRun(trackerFile, ForkedTest.class.getName())); } /** * Tests the execution of a forked test */ @Test - public void testBasicFork() { - buildRule.executeTarget("test-basic-fork"); + public void testBasicFork() throws Exception { + final String targetName = "test-basic-fork"; + final Path trackerFile = setupTrackerProperty(targetName); + // setup a dummy and incorrect value of a sysproperty that's used in the test + // being forked + System.setProperty(ForkedTest.SYS_PROP_ONE, "dummy"); + buildRule.executeTarget(targetName); + // verify that our JVM's sysprop value didn't get changed + Assert.assertEquals("System property " + ForkedTest.SYS_PROP_ONE + " was unexpected updated", + "dummy", System.getProperty(ForkedTest.SYS_PROP_ONE)); + + Assert.assertTrue("ForkedTest#testSysProp was expected to succeed", verifySuccess(trackerFile, + ForkedTest.class.getName(), "testSysProp")); + } + private Path setupTrackerProperty(final String targetName) { + final String filename = targetName + "-tracker.txt"; + buildRule.getProject().setProperty(targetName + ".tracker", filename); + final String outputDir = buildRule.getProject().getProperty("output.dir"); + return Paths.get(outputDir, filename); } } diff --git a/src/tests/junit/org/example/junitlauncher/Tracker.java b/src/tests/junit/org/example/junitlauncher/Tracker.java new file mode 100644 index 000000000..8ef89b2cb --- /dev/null +++ b/src/tests/junit/org/example/junitlauncher/Tracker.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.example.junitlauncher; + +import org.apache.tools.ant.taskdefs.optional.junitlauncher.TestExecutionContext; +import org.apache.tools.ant.taskdefs.optional.junitlauncher.TestResultFormatter; +import org.junit.platform.engine.TestExecutionResult; +import org.junit.platform.engine.support.descriptor.ClassSource; +import org.junit.platform.engine.support.descriptor.MethodSource; +import org.junit.platform.launcher.TestIdentifier; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +/** + * Utility listener used in {@link org.apache.tools.ant.taskdefs.optional.junitlauncher.JUnitLauncherTaskTest} + * to verify that tests launched through junitlauncher task are executed as expected + */ +public class Tracker implements TestResultFormatter { + + private static final String PREFIX_TEST_CLASS_STARTED = "started:test-class:"; + private static final String PREFIX_TEST_METHOD_STARTED = "started:test-method:"; + private static final String PREFIX_TEST_CLASS_SKIPPED = "skipped:test-class:"; + private static final String PREFIX_TEST_METHOD_SKIPPED = "skipped:test-method:"; + + private PrintWriter writer; + private TestExecutionContext context; + + @Override + public void setDestination(final OutputStream os) { + this.writer = new PrintWriter(os, true); + } + + @Override + public void setContext(final TestExecutionContext context) { + this.context = context; + } + + @Override + public void close() throws IOException { + this.writer.flush(); + } + + @Override + public void executionStarted(final TestIdentifier testIdentifier) { + testIdentifier.getSource().ifPresent((s) -> { + if (s instanceof MethodSource) { + writer.println(PREFIX_TEST_METHOD_STARTED + ((MethodSource) s).getClassName() + + "#" + ((MethodSource) s).getMethodName()); + return; + } + if (s instanceof ClassSource) { + writer.println(PREFIX_TEST_CLASS_STARTED + ((ClassSource) s).getClassName()); + return; + } + + }); + } + + @Override + public void executionFinished(final TestIdentifier testIdentifier, final TestExecutionResult testExecutionResult) { + testIdentifier.getSource().ifPresent((s) -> { + if (s instanceof MethodSource) { + writer.println(testExecutionResult.getStatus().name() + ":test-method:" + ((MethodSource) s).getClassName() + + "#" + ((MethodSource) s).getMethodName()); + return; + } + if (s instanceof ClassSource) { + writer.println(testExecutionResult.getStatus().name() + ":test-class:" + ((ClassSource) s).getClassName()); + return; + } + + }); + } + + @Override + public void executionSkipped(final TestIdentifier testIdentifier, final String reason) { + testIdentifier.getSource().ifPresent((s) -> { + if (s instanceof MethodSource) { + writer.println(PREFIX_TEST_METHOD_SKIPPED + ((MethodSource) s).getClassName() + "#" + + ((MethodSource) s).getMethodName()); + return; + } + if (s instanceof ClassSource) { + writer.println(PREFIX_TEST_CLASS_SKIPPED + ((ClassSource) s).getClassName()); + return; + } + + }); + } + + public static boolean wasTestRun(final Path trackerFile, final String className) throws IOException { + final List lines = Files.readAllLines(trackerFile); + return lines.contains(PREFIX_TEST_CLASS_STARTED + className); + } + + public static boolean wasTestRun(final Path trackerFile, final String className, final String methodName) throws IOException { + final List lines = Files.readAllLines(trackerFile); + return lines.contains(PREFIX_TEST_METHOD_STARTED + className + "#" + methodName); + } + + public static boolean verifyFailed(final Path trackerFile, final String className, final String methodName) throws IOException { + final List lines = Files.readAllLines(trackerFile); + return lines.contains(TestExecutionResult.Status.FAILED + ":test-method:" + className + "#" + methodName); + } + + public static boolean verifySuccess(final Path trackerFile, final String className, final String methodName) throws IOException { + final List lines = Files.readAllLines(trackerFile); + return lines.contains(TestExecutionResult.Status.SUCCESSFUL + ":test-method:" + className + "#" + methodName); + } + + public static boolean verifySkipped(final Path trackerFile, final String className, final String methodName) throws IOException { + final List lines = Files.readAllLines(trackerFile); + return lines.contains(PREFIX_TEST_METHOD_SKIPPED + className + "#" + methodName); + } +} diff --git a/src/tests/junit/org/example/junitlauncher/vintage/ForkedTest.java b/src/tests/junit/org/example/junitlauncher/vintage/ForkedTest.java new file mode 100644 index 000000000..4c305702a --- /dev/null +++ b/src/tests/junit/org/example/junitlauncher/vintage/ForkedTest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.example.junitlauncher.vintage; + +import org.junit.Assert; +import org.junit.Test; + +/** + * + */ +public class ForkedTest { + + public static String SYS_PROP_ONE = "junitlauncher.test.sysprop.one"; + + + @Test + public void testSysProp() { + Assert.assertEquals("Unexpected value for system property", + "forked", System.getProperty(SYS_PROP_ONE)); + } +}