[junitlauncher] add more fine grained tests for junitlauncher task

src/etc/testcases/taskdefs/optional/junitlauncher.xml

@@ -50,6 +50,11 @@
 
     <target name="test-failure-stops-build" depends="init">
         <junitlauncher>
+            <listener classname="org.example.junitlauncher.Tracker"
+                      outputDir="${output.dir}"
+                      resultFile="${test-failure-stops-build.tracker}"
+                      if="test-failure-stops-build.tracker"/>
+
             <!-- A specific test meant to fail -->
             <test name="org.example.junitlauncher.vintage.AlwaysFailingJUnit4Test" haltOnFailure="true"/>
             <!-- classpath to be used for the tests -->
@@ -59,6 +64,11 @@
 
     <target name="test-failure-continues-build" depends="init">
         <junitlauncher>
+            <listener classname="org.example.junitlauncher.Tracker"
+                      outputDir="${output.dir}"
+                      resultFile="${test-failure-continues-build.tracker}"
+                      if="test-failure-continues-build.tracker"/>
+
             <!-- A specific test meant to fail -->
             <test name="org.example.junitlauncher.vintage.AlwaysFailingJUnit4Test"/>
             <classpath refid="test.classpath"/>
@@ -67,6 +77,10 @@
 
     <target name="test-success" depends="init">
         <junitlauncher>
+            <listener classname="org.example.junitlauncher.Tracker"
+                      outputDir="${output.dir}"
+                      resultFile="${test-success.tracker}"
+                      if="test-success.tracker"/>
             <!-- A specific test meant to pass -->
             <test name="org.example.junitlauncher.vintage.JUnit4SampleTest"/>
             <classpath refid="test.classpath"/>
@@ -75,6 +89,11 @@
 
     <target name="test-one-specific-method" depends="init">
         <junitlauncher>
+            <listener classname="org.example.junitlauncher.Tracker"
+                      outputDir="${output.dir}"
+                      resultFile="${test-one-specific-method.tracker}"
+                      if="test-one-specific-method.tracker"/>
+
             <test name="org.example.junitlauncher.vintage.JUnit4SampleTest" methods="testBar" haltonfailure="true"/>
             <classpath refid="test.classpath"/>
         </junitlauncher>
@@ -82,6 +101,11 @@
 
     <target name="test-multiple-specific-methods" depends="init">
         <junitlauncher>
+            <listener classname="org.example.junitlauncher.Tracker"
+                      outputDir="${output.dir}"
+                      resultFile="${test-multiple-specific-methods.tracker}"
+                      if="test-multiple-specific-methods.tracker"/>
+
             <test name="org.example.junitlauncher.vintage.JUnit4SampleTest" methods=" testFoo, testFooBar "
                   haltonfailure="true"/>
             <classpath refid="test.classpath"/>
@@ -90,8 +114,18 @@
 
     <target name="test-multiple-individual" depends="init">
         <junitlauncher>
-            <test name="org.example.junitlauncher.vintage.AlwaysFailingJUnit4Test"/>
-            <test name="org.example.junitlauncher.vintage.JUnit4SampleTest"/>
+            <test name="org.example.junitlauncher.vintage.AlwaysFailingJUnit4Test">
+                <listener classname="org.example.junitlauncher.Tracker"
+                          outputDir="${output.dir}"
+                          resultFile="${test-multiple-individual-1.tracker}"
+                          if="test-multiple-individual-1.tracker"/>
+            </test>
+            <test name="org.example.junitlauncher.vintage.JUnit4SampleTest">
+                <listener classname="org.example.junitlauncher.Tracker"
+                          outputDir="${output.dir}"
+                          resultFile="${test-multiple-individual-2.tracker}"
+                          if="test-multiple-individual-2.tracker"/>
+            </test>
             <classpath refid="test.classpath"/>
         </junitlauncher>
     </target>
@@ -100,8 +134,14 @@
         <junitlauncher>
             <classpath refid="test.classpath"/>
             <testclasses outputdir="${output.dir}">
+                <listener classname="org.example.junitlauncher.Tracker"
+                          outputDir="${output.dir}"
+                          resultFile="${test-batch.tracker}"
+                          if="test-batch.tracker"/>
+
                 <fileset dir="${build.classes.dir}">
                     <include name="org/example/**/junitlauncher/**/"/>
+                    <exclude name="org/example/**/junitlauncher/**/ForkedTest.class"/>
                 </fileset>
                 <listener type="legacy-brief" sendSysOut="true"/>
                 <listener type="legacy-xml" sendSysErr="true" sendSysOut="true"/>
@@ -112,8 +152,16 @@
     <target name="test-basic-fork" depends="init">
         <junitlauncher>
             <classpath refid="test.classpath"/>
-            <test name="org.example.junitlauncher.vintage.JUnit4SampleTest" outputdir="${output.dir}">
-                <fork dir="${basedir}"/>
+            <test name="org.example.junitlauncher.vintage.ForkedTest" outputdir="${output.dir}">
+                <listener classname="org.example.junitlauncher.Tracker"
+                          outputDir="${output.dir}"
+                          resultFile="${test-basic-fork.tracker}"
+                          if="test-basic-fork.tracker"/>
+
+                <fork dir="${basedir}">
+                    <sysproperty key="junitlauncher.test.sysprop.one" value="forked"/>
+                </fork>
+
                 <listener type="legacy-xml" sendSysErr="true" sendSysOut="true"/>
             </test>
         </junitlauncher>

src/tests/junit/org/apache/tools/ant/taskdefs/optional/junitlauncher/JUnitLauncherTaskTest.java

@@ -17,15 +17,25 @@
  */
 package org.apache.tools.ant.taskdefs.optional.junitlauncher;
 
-import org.apache.tools.ant.BuildEvent;
 import org.apache.tools.ant.BuildException;
 import org.apache.tools.ant.BuildFileRule;
-import org.apache.tools.ant.BuildListener;
-import org.apache.tools.ant.Project;
+import org.example.junitlauncher.jupiter.JupiterSampleTest;
+import org.example.junitlauncher.vintage.AlwaysFailingJUnit4Test;
+import org.example.junitlauncher.vintage.ForkedTest;
+import org.example.junitlauncher.vintage.JUnit4SampleTest;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import static org.example.junitlauncher.Tracker.verifyFailed;
+import static org.example.junitlauncher.Tracker.verifySkipped;
+import static org.example.junitlauncher.Tracker.verifySuccess;
+import static org.example.junitlauncher.Tracker.wasTestRun;
+
 /**
  * Tests the {@link JUnitLauncherTask}
  */
@@ -40,47 +50,27 @@ public class JUnitLauncherTaskTest {
     @Before
     public void setUp() {
         buildRule.configureProject("src/etc/testcases/taskdefs/optional/junitlauncher.xml");
-        buildRule.getProject().addBuildListener(new BuildListener() {
-            @Override
-            public void buildStarted(final BuildEvent event) {
-            }
-
-            @Override
-            public void buildFinished(final BuildEvent event) {
-            }
-
-            @Override
-            public void targetStarted(final BuildEvent event) {
-            }
-
-            @Override
-            public void targetFinished(final BuildEvent event) {
-            }
-
-            @Override
-            public void taskStarted(final BuildEvent event) {
-            }
-
-            @Override
-            public void taskFinished(final BuildEvent event) {
-            }
-
-            @Override
-            public void messageLogged(final BuildEvent event) {
-                if (event.getPriority() <= Project.MSG_INFO) {
-                    System.out.println(event.getMessage());
-                }
-            }
-        });
     }
 
     /**
      * Tests that when a test, that's configured with {@code haltOnFailure=true}, stops the build, when the
      * test fails
      */
-    @Test(expected = BuildException.class)
-    public void testFailureStopsBuild() {
-        buildRule.executeTarget("test-failure-stops-build");
+    @Test
+    public void testFailureStopsBuild() throws Exception {
+        final String targetName = "test-failure-stops-build";
+        final Path trackerFile = setupTrackerProperty(targetName);
+        try {
+            buildRule.executeTarget(targetName);
+            Assert.fail(targetName + " was expected to fail");
+        } catch (BuildException e) {
+            // expected, but do further tests to make sure the build failed for expected reason
+            if (!verifyFailed(trackerFile, AlwaysFailingJUnit4Test.class.getName(),
+                    "testWillFail")) {
+                // throw back the original cause
+                throw e;
+            }
+        }
     }
 
     /**
@@ -88,33 +78,71 @@ public class JUnitLauncherTaskTest {
      * build even when there are test failures
      */
     @Test
-    public void testFailureContinuesBuild() {
-        buildRule.executeTarget("test-failure-continues-build");
+    public void testFailureContinuesBuild() throws Exception {
+        final String targetName = "test-failure-continues-build";
+        final Path trackerFile = setupTrackerProperty(targetName);
+        buildRule.executeTarget(targetName);
+        // make sure the test that was expected to be run (and fail), did indeed fail
+        Assert.assertTrue("AlwaysFailingJUnit4Test#testWillFail was expected to run", wasTestRun(trackerFile,
+                AlwaysFailingJUnit4Test.class.getName(), "testWillFail"));
+        Assert.assertTrue("AlwaysFailingJUnit4Test#testWillFail was expected to fail", verifyFailed(trackerFile,
+                AlwaysFailingJUnit4Test.class.getName(), "testWillFail"));
     }
 
     /**
      * Tests the execution of test that's expected to succeed
      */
     @Test
-    public void testSuccessfulTests() {
-        buildRule.executeTarget("test-success");
+    public void testSuccessfulTests() throws Exception {
+        final String targetName = "test-success";
+        final Path trackerFile = setupTrackerProperty(targetName);
+        buildRule.executeTarget(targetName);
+        // make sure the right test(s) were run
+        Assert.assertTrue("JUnit4SampleTest test was expected to be run", wasTestRun(trackerFile, JUnit4SampleTest.class.getName()));
+        Assert.assertTrue("JUnit4SampleTest#testFoo was expected to succeed", verifySuccess(trackerFile,
+                JUnit4SampleTest.class.getName(), "testFoo"));
     }
 
     /**
      * Tests execution of a test which is configured to execute only a particular set of test methods
      */
     @Test
-    public void testSpecificMethodTest() {
-        buildRule.executeTarget("test-one-specific-method");
-        buildRule.executeTarget("test-multiple-specific-methods");
+    public void testSpecificMethodTest() throws Exception {
+        final String targetSpecificMethod = "test-one-specific-method";
+        final Path tracker1 = setupTrackerProperty(targetSpecificMethod);
+        buildRule.executeTarget(targetSpecificMethod);
+        // verify only that specific method was run
+        Assert.assertTrue("testBar was expected to be run", wasTestRun(tracker1, JUnit4SampleTest.class.getName(),
+                "testBar"));
+        Assert.assertFalse("testFoo wasn't expected to be run", wasTestRun(tracker1, JUnit4SampleTest.class.getName(),
+                "testFoo"));
+
+
+        final String targetMultipleMethods = "test-multiple-specific-methods";
+        final Path tracker2 = setupTrackerProperty(targetMultipleMethods);
+        buildRule.executeTarget(targetMultipleMethods);
+        Assert.assertTrue("testFooBar was expected to be run", wasTestRun(tracker2, JUnit4SampleTest.class.getName(),
+                "testFooBar"));
+        Assert.assertTrue("testFoo was expected to be run", wasTestRun(tracker2, JUnit4SampleTest.class.getName(),
+                "testFoo"));
+        Assert.assertFalse("testBar wasn't expected to be run", wasTestRun(tracker2, JUnit4SampleTest.class.getName(),
+                "testBar"));
     }
 
     /**
      * Tests the execution of more than one {@code &lt;test&gt;} elements in the {@code &lt;junitlauncher&gt;} task
      */
     @Test
-    public void testMultipleIndividualTests() {
-        buildRule.executeTarget("test-multiple-individual");
+    public void testMultipleIndividualTests() throws Exception {
+        final String targetName = "test-multiple-individual";
+        final Path trackerFile1 = setupTrackerProperty(targetName + "-1");
+        final Path trackerFile2 = setupTrackerProperty(targetName + "-2");
+        buildRule.executeTarget(targetName);
+
+        Assert.assertTrue("AlwaysFailingJUnit4Test#testWillFail was expected to be run", wasTestRun(trackerFile1,
+                AlwaysFailingJUnit4Test.class.getName(), "testWillFail"));
+        Assert.assertTrue("JUnit4SampleTest#testFoo was expected to be run", wasTestRun(trackerFile2,
+                JUnit4SampleTest.class.getName(), "testFoo"));
     }
 
     /**
@@ -122,16 +150,47 @@ public class JUnitLauncherTaskTest {
      * of the {@code &lt;junitlauncher&gt;} task
      */
     @Test
-    public void testTestClasses() {
-        buildRule.executeTarget("test-batch");
+    public void testTestClasses() throws Exception {
+        final String targetName = "test-batch";
+        final Path trackerFile = setupTrackerProperty(targetName);
+        buildRule.executeTarget(targetName);
+
+        Assert.assertTrue("JUnit4SampleTest#testFoo was expected to succeed", verifySuccess(trackerFile,
+                JUnit4SampleTest.class.getName(), "testFoo"));
+        Assert.assertTrue("AlwaysFailingJUnit4Test#testWillFail was expected to fail", verifyFailed(trackerFile,
+                AlwaysFailingJUnit4Test.class.getName(), "testWillFail"));
+        Assert.assertTrue("JupiterSampleTest#testSucceeds was expected to succeed", verifySuccess(trackerFile,
+                JupiterSampleTest.class.getName(), "testSucceeds"));
+        Assert.assertTrue("JupiterSampleTest#testFails was expected to succeed", verifyFailed(trackerFile,
+                JupiterSampleTest.class.getName(), "testFails"));
+        Assert.assertTrue("JupiterSampleTest#testSkipped was expected to be skipped", verifySkipped(trackerFile,
+                JupiterSampleTest.class.getName(), "testSkipped"));
+        Assert.assertFalse("ForkedTest wasn't expected to be run", wasTestRun(trackerFile, ForkedTest.class.getName()));
     }
 
     /**
      * Tests the execution of a forked test
      */
     @Test
-    public void testBasicFork() {
-        buildRule.executeTarget("test-basic-fork");
+    public void testBasicFork() throws Exception {
+        final String targetName = "test-basic-fork";
+        final Path trackerFile = setupTrackerProperty(targetName);
+        // setup a dummy and incorrect value of a sysproperty that's used in the test
+        // being forked
+        System.setProperty(ForkedTest.SYS_PROP_ONE, "dummy");
+        buildRule.executeTarget(targetName);
+        // verify that our JVM's sysprop value didn't get changed
+        Assert.assertEquals("System property " + ForkedTest.SYS_PROP_ONE + " was unexpected updated",
+                "dummy", System.getProperty(ForkedTest.SYS_PROP_ONE));
+
+        Assert.assertTrue("ForkedTest#testSysProp was expected to succeed", verifySuccess(trackerFile,
+                ForkedTest.class.getName(), "testSysProp"));
+    }
 
+    private Path setupTrackerProperty(final String targetName) {
+        final String filename = targetName + "-tracker.txt";
+        buildRule.getProject().setProperty(targetName + ".tracker", filename);
+        final String outputDir = buildRule.getProject().getProperty("output.dir");
+        return Paths.get(outputDir, filename);
     }
 }

src/tests/junit/org/example/junitlauncher/Tracker.java

@@ -0,0 +1,136 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.example.junitlauncher;
+
+import org.apache.tools.ant.taskdefs.optional.junitlauncher.TestExecutionContext;
+import org.apache.tools.ant.taskdefs.optional.junitlauncher.TestResultFormatter;
+import org.junit.platform.engine.TestExecutionResult;
+import org.junit.platform.engine.support.descriptor.ClassSource;
+import org.junit.platform.engine.support.descriptor.MethodSource;
+import org.junit.platform.launcher.TestIdentifier;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+
+/**
+ * Utility listener used in {@link org.apache.tools.ant.taskdefs.optional.junitlauncher.JUnitLauncherTaskTest}
+ * to verify that tests launched through junitlauncher task are executed as expected
+ */
+public class Tracker implements TestResultFormatter {
+
+    private static final String PREFIX_TEST_CLASS_STARTED = "started:test-class:";
+    private static final String PREFIX_TEST_METHOD_STARTED = "started:test-method:";
+    private static final String PREFIX_TEST_CLASS_SKIPPED = "skipped:test-class:";
+    private static final String PREFIX_TEST_METHOD_SKIPPED = "skipped:test-method:";
+
+    private PrintWriter writer;
+    private TestExecutionContext context;
+
+    @Override
+    public void setDestination(final OutputStream os) {
+        this.writer = new PrintWriter(os, true);
+    }
+
+    @Override
+    public void setContext(final TestExecutionContext context) {
+        this.context = context;
+    }
+
+    @Override
+    public void close() throws IOException {
+        this.writer.flush();
+    }
+
+    @Override
+    public void executionStarted(final TestIdentifier testIdentifier) {
+        testIdentifier.getSource().ifPresent((s) -> {
+            if (s instanceof MethodSource) {
+                writer.println(PREFIX_TEST_METHOD_STARTED + ((MethodSource) s).getClassName()
+                        + "#" + ((MethodSource) s).getMethodName());
+                return;
+            }
+            if (s instanceof ClassSource) {
+                writer.println(PREFIX_TEST_CLASS_STARTED + ((ClassSource) s).getClassName());
+                return;
+            }
+
+        });
+    }
+
+    @Override
+    public void executionFinished(final TestIdentifier testIdentifier, final TestExecutionResult testExecutionResult) {
+        testIdentifier.getSource().ifPresent((s) -> {
+            if (s instanceof MethodSource) {
+                writer.println(testExecutionResult.getStatus().name() + ":test-method:" + ((MethodSource) s).getClassName()
+                        + "#" + ((MethodSource) s).getMethodName());
+                return;
+            }
+            if (s instanceof ClassSource) {
+                writer.println(testExecutionResult.getStatus().name() + ":test-class:" + ((ClassSource) s).getClassName());
+                return;
+            }
+
+        });
+    }
+
+    @Override
+    public void executionSkipped(final TestIdentifier testIdentifier, final String reason) {
+        testIdentifier.getSource().ifPresent((s) -> {
+            if (s instanceof MethodSource) {
+                writer.println(PREFIX_TEST_METHOD_SKIPPED + ((MethodSource) s).getClassName() + "#"
+                        + ((MethodSource) s).getMethodName());
+                return;
+            }
+            if (s instanceof ClassSource) {
+                writer.println(PREFIX_TEST_CLASS_SKIPPED + ((ClassSource) s).getClassName());
+                return;
+            }
+
+        });
+    }
+
+    public static boolean wasTestRun(final Path trackerFile, final String className) throws IOException {
+        final List<String> lines = Files.readAllLines(trackerFile);
+        return lines.contains(PREFIX_TEST_CLASS_STARTED + className);
+    }
+
+    public static boolean wasTestRun(final Path trackerFile, final String className, final String methodName) throws IOException {
+        final List<String> lines = Files.readAllLines(trackerFile);
+        return lines.contains(PREFIX_TEST_METHOD_STARTED + className + "#" + methodName);
+    }
+
+    public static boolean verifyFailed(final Path trackerFile, final String className, final String methodName) throws IOException {
+        final List<String> lines = Files.readAllLines(trackerFile);
+        return lines.contains(TestExecutionResult.Status.FAILED + ":test-method:" + className + "#" + methodName);
+    }
+
+    public static boolean verifySuccess(final Path trackerFile, final String className, final String methodName) throws IOException {
+        final List<String> lines = Files.readAllLines(trackerFile);
+        return lines.contains(TestExecutionResult.Status.SUCCESSFUL + ":test-method:" + className + "#" + methodName);
+    }
+
+    public static boolean verifySkipped(final Path trackerFile, final String className, final String methodName) throws IOException {
+        final List<String> lines = Files.readAllLines(trackerFile);
+        return lines.contains(PREFIX_TEST_METHOD_SKIPPED + className + "#" + methodName);
+    }
+}

src/tests/junit/org/example/junitlauncher/vintage/ForkedTest.java

@@ -0,0 +1,37 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.example.junitlauncher.vintage;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class ForkedTest {
+
+    public static String SYS_PROP_ONE = "junitlauncher.test.sysprop.one";
+
+
+    @Test
+    public void testSysProp() {
+        Assert.assertEquals("Unexpected value for system property",
+                "forked", System.getProperty(SYS_PROP_ONE));
+    }
+}