New JUnit formatter: collects failing test cases (testXXX methods) for a rerun.

* works from command line
* its own JUnit test scenario fails (for - to me - unknown reason)

* BFT has new method 'assertOutputNotContaining' similar to 'assertOutputContaining'



git-svn-id: https://svn.apache.org/repos/asf/ant/core/trunk@563053 13f79535-47bb-0310-9956-ffa450edef68

src/etc/testcases/taskdefs/optional/junit.xml

@@ -143,4 +143,94 @@
       </batchtest>
     </junit>
   </target>
+
+  <target name="failureRecorder.prepare">
+    <property name="tmp.dir" value="out"/>
+    <mkdir dir="${tmp.dir}/org"/>
+    <echo file="${tmp.dir}/A.java">
+        import junit.framework.*;
+        public class A extends TestCase {
+            public A(String s) { super(s); }
+            public void test01() { System.out.println("A.test01"); }
+            public void test02() { System.out.println("A.test02"); fail(); }
+            public void test03() { System.out.println("A.test03"); fail(); }
+      }
+    </echo>
+    <echo file="${tmp.dir}/B.java">
+        import junit.framework.*;
+        public class B extends TestCase {
+            public B(String s) { super(s); }
+            public void test04() { System.out.println("B.test04"); fail(); }
+            public void test05() { System.out.println("B.test05"); }
+            public void test06() { System.out.println("B.test06"); }
+        }
+    </echo>
+    <echo file="${tmp.dir}/C.java">
+        import junit.framework.*;
+        public class C extends TestCase {
+            public C(String s) { super(s); }
+            public void test07() { System.out.println("C.test07"); }
+            public void test08() { System.out.println("C.test08"); }
+            public void test09() { System.out.println("C.test09"); }
+        }
+    </echo>
+    <echo file="${tmp.dir}/org/D.java">
+    	package org;
+        import junit.framework.*;
+        public class D extends TestCase {
+            public D(String s) { super(s); }
+            public void test10() { System.out.println("D.test10"); fail(); }
+        }
+    </echo>
+    <javac srcdir="${tmp.dir}" destdir="${tmp.dir}"/>
+  </target>
+
+  <target name="failureRecorder.internal">  
+    <property name="tmp.dir" value="out"/>
+    <!-- 
+    <delete>
+      <fileset dir="${tmp.dir}" includes="FailedTests*.class"/>
+    </delete>
+     -->
+    <!-- compile the FailedTests class if present --> 
+    <javac srcdir="${tmp.dir}" destdir="${tmp.dir}"/>
+    <available file="${tmp.dir}/FailedTests.class" property="hasFailingTests"/>
+    <junit haltonerror="false" haltonfailure="false">
+      <sysproperty key="ant.junit.failureCollector" value="${tmp.dir}/FailedTests"/>
+      <classpath>
+        <pathelement location="${tmp.dir}"/>
+      </classpath>
+      <batchtest todir="${tmp.dir}" unless="hasFailingTests">
+        <fileset dir="${tmp.dir}" includes="**/*.java" excludes="**/FailedTests.*"/>
+        <!-- for initial creation of the FailingTests.java -->
+        <formatter type="failure"/>
+        <!-- I want to see something ... -->
+        <formatter type="plain" usefile="false"/>
+      </batchtest>
+      <test name="FailedTests" if="hasFailingTests">
+      	<!-- update the FailingTests.java -->
+        <formatter type="failure"/>
+        <!-- again, I want to see something -->
+        <formatter type="plain" usefile="false"/>
+      </test>
+    </junit>
+  </target>
+  
+  <target name="failureRecorder.runtest">
+  	<ant target="failureRecorder.internal" antfile="junit.xml" inheritAll="false"/>
+  </target>
+  
+  <target name="failureRecorder.fixing">
+    <property name="tmp.dir" value="out"/>
+    <echo file="${tmp.dir}/A.java">
+        import junit.framework.*;
+        public class A extends TestCase {
+            public A(String s) { super(s); }
+            public void test01() { System.out.println("A.test01"); }
+            public void test02() { System.out.println("A.test02"); }
+            public void test03() { System.out.println("A.test03"); }
+      }
+    </echo>
+  </target>
+
 </project>

src/main/org/apache/tools/ant/taskdefs/optional/junit/FailureRecorder.java

@@ -0,0 +1,200 @@
+package org.apache.tools.ant.taskdefs.optional.junit;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.Iterator;
+
+import junit.framework.AssertionFailedError;
+import junit.framework.Test;
+
+import org.apache.tools.ant.BuildException;
+import org.apache.tools.ant.util.FileUtils;
+
+/**
+ * <p>Collects all failing test <i>cases</i> and creates a new JUnit test class containing 
+ * a suite() method which calls these failed tests.</p>
+ * <p>Having classes <i>A</i> ... <i>D</i> with each several testcases you could earn a new
+ * test class like
+ * <pre>
+ * // generated on: 2007.08.06 09:42:34,555
+ * import junit.framework.*;
+ * public class FailedTests extends TestCase {
+ *     public FailedTests(String s) {
+ *         super(s);
+ *     }
+ *     public static Test suite() {
+ *         TestSuite suite = new TestSuite();
+ *         suite.addTest( new B("test04") );
+ *         suite.addTest( new org.D("test10") );
+ *         return suite;
+ *     }
+ * }
+ * </pre> 
+ * 
+ * @since Ant 1.7.1
+ */
+/*
+ * Because each running test case gets its own formatter, we collect
+ * the failing test cases in a static list. Because we dont have a finalizer
+ * method in the formatters "lifecycle", we regenerate the new java source
+ * at each end of a test suite. The last run will contain all failed tests.
+ */
+public class FailureRecorder implements JUnitResultFormatter {
+    
+    /** 
+     * This is the name of a magic System property ({@value}). The value of this 
+     * <b>System</b> property should point to the location where to store the 
+     * generated class (without suffix).
+     * Default location and name is defined in DEFAULT_CLASS_LOCATION.
+     * @see #DEFAULT_CLASS_LOCATION
+     */
+    public static final String MAGIC_PROPERTY_CLASS_LOCATION = "ant.junit.failureCollector";
+
+    /** Default location and name for the generated JUnit class file. {@value} */
+    public static final String DEFAULT_CLASS_LOCATION = System.getProperty("java.io.tmpdir") + "FailedTests";
+    
+    /** Class names of failed tests without duplicates. */
+    private static HashSet/*<Test>*/ failedTests = new HashSet();
+    
+    /** A writer for writing the generated source to. */
+    private PrintWriter writer;
+    
+    /**
+     * Location and name of the generated JUnit class. 
+     * Lazy instantiated via getLocationName().
+     */
+    private static String locationName;
+    
+    //TODO: Dont set the locationName via System.getProperty - better
+    //      via Ant properties. But how to access these?
+    private String getLocationName() {
+        if (locationName == null) {
+            String propValue = System.getProperty(MAGIC_PROPERTY_CLASS_LOCATION);
+            locationName = (propValue != null) ? propValue : DEFAULT_CLASS_LOCATION;
+        }
+        return locationName;
+    }
+
+    /**
+     * After each test suite, the whole new JUnit class will be regenerated. 
+     * @see org.apache.tools.ant.taskdefs.optional.junit.JUnitResultFormatter#endTestSuite(org.apache.tools.ant.taskdefs.optional.junit.JUnitTest)
+     */
+    public void endTestSuite(JUnitTest suite) throws BuildException {
+        if (failedTests.isEmpty()) return;
+        try {
+            File sourceFile = new File(getLocationName() + ".java");
+            sourceFile.delete();
+            writer = new PrintWriter(new FileOutputStream(sourceFile));
+
+            createClassHeader();
+            createTestSuiteHeader();
+            for (Iterator iter = failedTests.iterator(); iter.hasNext();) {
+                Test test = (Test) iter.next();
+                if (test!=null) {
+                    createAddTestToSuite(test);
+                }
+            }
+            createTestSuiteFooter();
+            createClassFooter();
+            
+            FileUtils.close(writer);
+        } catch (FileNotFoundException e) {
+            e.printStackTrace();
+        }
+    }
+    
+    public void addError(Test test, Throwable throwable) {
+        failedTests.add(test);
+    }
+
+    public void addFailure(Test test, AssertionFailedError error) {
+        failedTests.add(test);
+    }
+
+    public void setOutput(OutputStream out) {
+        // not in use
+    }
+
+    public void setSystemError(String err) {
+        // not in use
+    }
+
+    public void setSystemOutput(String out) {
+        // not in use
+    }
+
+    public void startTestSuite(JUnitTest suite) throws BuildException {
+        // not in use
+    }
+
+    public void endTest(Test test) {
+        // not in use
+    }
+
+    public void startTest(Test test) {
+        // not in use
+    }
+    
+    // "Templates" for generating the JUnit class
+    
+    private void createClassHeader() {
+        String className = getLocationName().replace('\\', '/');
+        if (className.indexOf('/') > -1) {
+            className = className.substring(className.lastIndexOf('/')+1);
+        }
+        SimpleDateFormat sdf = new SimpleDateFormat("yyyy.MM.dd HH:mm:ss,SSS");
+        writer.print("// generated on: ");
+        writer.println(sdf.format(new Date()));
+        writer.println("import junit.framework.*;");
+        writer.print("public class ");
+        writer.print( className );
+        // If this class does not extend TC, Ant doesnt run these
+        writer.println(" extends TestCase {");
+        // no-arg constructor
+        writer.print("    public ");
+        writer.print(className);
+        writer.println("(String s) {");
+        writer.println("        super(s);");
+        writer.println("    }");
+    }
+    
+    private void createTestSuiteHeader() {
+        writer.println("    public static Test suite() {");
+        writer.println("        TestSuite suite = new TestSuite();");
+    }
+    
+    private void createAddTestToSuite(Test test) {
+        writer.print("        suite.addTest( new ");
+        writer.print( getClassName(test) );
+        writer.print("(\"");
+        writer.print( getMethodName(test) );
+        writer.println("\") );");
+    }
+
+    private void createTestSuiteFooter() {
+        writer.println("        return suite;");
+        writer.println("    }");
+    }
+    
+    private void createClassFooter() {
+        writer.println("}");
+    }
+    
+    // Helper methods
+    
+    private String getMethodName(Test test) {
+        String methodName = test.toString();
+        return methodName.substring(0, methodName.indexOf('('));
+    }
+
+    private String getClassName(Test test) {
+        return test.getClass().getName();
+    }
+    
+}

src/main/org/apache/tools/ant/taskdefs/optional/junit/FormatterElement.java

@@ -47,6 +47,7 @@ import org.apache.tools.ant.types.EnumeratedAttribute;
  * @see XMLJUnitResultFormatter
  * @see BriefJUnitResultFormatter
  * @see PlainJUnitResultFormatter
+ * @see FailureRecorder
  * @see JUnitResultFormatter
  */
 public class FormatterElement {
@@ -68,6 +69,9 @@ public class FormatterElement {
     /** plain formatter class */
     public static final String PLAIN_FORMATTER_CLASS_NAME =
         "org.apache.tools.ant.taskdefs.optional.junit.PlainJUnitResultFormatter";
+    /** failure recorder class */
+    public static final String FAILURE_RECORDER_CLASS_NAME =
+        "org.apache.tools.ant.taskdefs.optional.junit.FailureRecorder";
 
     /**
      * <p> Quick way to use a standard formatter.
@@ -77,6 +81,7 @@ public class FormatterElement {
      * <li> The <code>xml</code> type uses a <code>XMLJUnitResultFormatter</code>.
      * <li> The <code>brief</code> type uses a <code>BriefJUnitResultFormatter</code>.
      * <li> The <code>plain</code> type (the default) uses a <code>PlainJUnitResultFormatter</code>.
+     * <li> The <code>failure</code> type uses a <code>FailureRecorder</code>.
      * </ul>
      *
      * <p> Sets <code>classname</code> attribute - so you can't use that
@@ -84,13 +89,18 @@ public class FormatterElement {
      * @param type the enumerated value to use.
      */
     public void setType(TypeAttribute type) {
+        //TODO: Besseren Zugriffsalgorithums: TypeAttribut.getClassname()
         if ("xml".equals(type.getValue())) {
             setClassname(XML_FORMATTER_CLASS_NAME);
         } else {
             if ("brief".equals(type.getValue())) {
                 setClassname(BRIEF_FORMATTER_CLASS_NAME);
-            } else { // must be plain, ensured by TypeAttribute
-                setClassname(PLAIN_FORMATTER_CLASS_NAME);
+            } else {
+                if ("failure".equals(type.getValue())) {
+                    setClassname(FAILURE_RECORDER_CLASS_NAME);
+                } else { // must be plain, ensured by TypeAttribute
+                    setClassname(PLAIN_FORMATTER_CLASS_NAME);
+                }
             }
         }
     }
@@ -268,14 +278,14 @@ public class FormatterElement {
     }
 
     /**
-     * <p> Enumerated attribute with the values "plain", "xml" and "brief".
+     * <p> Enumerated attribute with the values "plain", "xml", "brief" and "failure".
      *
      * <p> Use to enumerate options for <code>type</code> attribute.
      */
     public static class TypeAttribute extends EnumeratedAttribute {
         /** {@inheritDoc}. */
         public String[] getValues() {
-            return new String[] {"plain", "xml", "brief"};
+            return new String[] {"plain", "xml", "brief", "failure"};
         }
     }
 }

src/main/org/apache/tools/ant/taskdefs/optional/junit/JUnitTask.java

@@ -163,7 +163,7 @@ public class JUnitTask extends Task {
     private JUnitTaskMirror delegate;
 
     /** A boolean on whether to get the forked path for ant classes */
-    private boolean   forkedPathChecked = false;
+    private boolean forkedPathChecked = false;
 
     //   Attributes for basetest
     private boolean haltOnError = false;

src/tests/junit/org/apache/tools/ant/BuildFileTest.java

@@ -18,11 +18,11 @@
 
 package org.apache.tools.ant;
 
-import junit.framework.TestCase;
 import java.io.File;
 import java.io.PrintStream;
 import java.net.URL;
-import java.util.Hashtable;
+
+import junit.framework.TestCase;
 
 /**
  * A BuildFileTest is a TestCase which executes targets from an Ant buildfile
@@ -117,18 +117,35 @@ public abstract class BuildFileTest extends TestCase {
 
     /**
      * Assert that the given substring is in the output messages.
+     * @param message Print this message if the test fails. Defaults to 
+     *                a meaningful text if <tt>null</tt> is passed.  
      * @since Ant1.7
      */
-    public void assertOutputContaining(String substring) {
+    public void assertOutputContaining(String message, String substring) {
         String realOutput = getOutput();
-        assertTrue("expecting output to contain \"" + substring
-                   + "\" output was \"" + realOutput + "\"",
-                   realOutput.indexOf(substring) >= 0);
+        String realMessage = (message != null) 
+            ? message 
+            : "expecting output to contain \"" + substring + "\" output was \"" + realOutput + "\"";
+        assertTrue(realMessage, realOutput.indexOf(substring) >= 0);
     }
 
     /**
-     * Assert that the given message has been logged with a priority
-     * &lt;= INFO when running the given target.
+     * Assert that the given substring is not in the output messages.
+     * @param message Print this message if the test fails. Defaults to 
+     *                a meaningful text if <tt>null</tt> is passed.  
+     * @since Ant1.7
+     */
+    public void assertOutputNotContaining(String message, String substring) {
+        String realOutput = getOutput();
+        String realMessage = (message != null) 
+            ? message 
+            : "expecting output to contain \"" + substring + "\" output was \"" + realOutput + "\"";
+        assertFalse(realMessage, realOutput.indexOf(substring) >= 0);
+    }
+
+    /**
+     * Assert that the given message has been logged with a priority &lt;= INFO when running the
+     * given target.
      */
     public void expectLogContaining(String target, String log) {
         executeTarget(target);

src/tests/junit/org/apache/tools/ant/taskdefs/optional/junit/JUnitTaskTest.java

@@ -17,31 +17,31 @@
  */
 package org.apache.tools.ant.taskdefs.optional.junit;
 
-import org.apache.tools.ant.BuildFileTest;
 import java.io.BufferedReader;
+import java.io.File;
 import java.io.FileReader;
 import java.io.IOException;
 
+import org.apache.tools.ant.BuildFileTest;
+
 public class JUnitTaskTest extends BuildFileTest {
 
     /**
-     * Constructor for the JUnitTaskTest object
+     * Constructor for the JUnitTaskTest object.
      */
     public JUnitTaskTest(String name) {
         super(name);
     }
 
-
     /**
-     * The JUnit setup method
+     * The JUnit setup method.
      */
     public void setUp() {
         configureProject("src/etc/testcases/taskdefs/optional/junit.xml");
     }
 
-
     /**
-     * The teardown method for JUnit
+     * The teardown method for JUnit.
      */
     public void tearDown() {
         executeTarget("cleanup");
@@ -86,7 +86,83 @@ public class JUnitTaskTest extends BuildFileTest {
     public void testBatchTestForkOnceExtension() {
         assertResultFilesExist("testBatchTestForkOnceExtension", ".foo");
     }
-
+    
+    /* Bugzilla Report 42984 */
+    //TODO This scenario works from command line, but not from JUnit ...
+    //     See the _run.bat attachement of the bug.
+    public void _testFailureRecorder() {
+        File testDir = new File(getProjectDir(), "out");
+        File collectorFile = new File(getProjectDir(), "out/FailedTests.java");
+
+        // ensure that there is a clean test environment
+        assertFalse("Test directory must not exist before the test preparation.", 
+                testDir.exists());
+        assertFalse("The collector file must not exist before the test preparation.", 
+                collectorFile.exists());
+
+        // prepare the test environment
+        executeTarget("failureRecorder.prepare");
+        assertTrue("Test directory was not created.", testDir.exists());
+        assertTrue("There should be one class.", (new File(testDir, "A.class")).exists());
+        assertFalse("The collector file " + collectorFile.getAbsolutePath() 
+                + " should not exist before the 1st run.", collectorFile.exists());
+
+        // 1st junit run: should do all tests - failing and not failing tests
+        executeTarget("failureRecorder.runtest");
+        assertTrue("The collector file " + collectorFile.getAbsolutePath() 
+                + " should exist after the 1st run.", collectorFile.exists());
+        // the passing test cases
+        assertOutputContaining("1st run: should run A.test01", "A.test01");
+        assertOutputContaining("1st run: should run B.test05", "B.test05");
+        assertOutputContaining("1st run: should run B.test06", "B.test06");
+        assertOutputContaining("1st run: should run C.test07", "C.test07");
+        assertOutputContaining("1st run: should run C.test08", "C.test08");
+        assertOutputContaining("1st run: should run C.test09", "C.test09");
+        // the failing test cases
+        assertOutputContaining("1st run: should run A.test02", "A.test02");
+        assertOutputContaining("1st run: should run A.test03", "A.test03");
+        assertOutputContaining("1st run: should run B.test04", "B.test04");
+        assertOutputContaining("1st run: should run D.test10", "D.test10");
+
+        // 2nd junit run: should do only failing tests
+        executeTarget("failureRecorder.runtest");
+        assertTrue("The collector file " + collectorFile.getAbsolutePath() 
+                + " should exist after the 2nd run.", collectorFile.exists());
+        // the passing test cases
+        assertOutputNotContaining("2nd run: should not run A.test01", "A.test01");
+        assertOutputNotContaining("2nd run: should not run A.test05", "B.test05");
+        assertOutputNotContaining("2nd run: should not run B.test06", "B.test06");
+        assertOutputNotContaining("2nd run: should not run C.test07", "C.test07");
+        assertOutputNotContaining("2nd run: should not run C.test08", "C.test08");
+        assertOutputNotContaining("2nd run: should not run C.test09", "C.test09");
+        // the failing test cases
+        assertOutputContaining("2nd run: should run A.test02", "A.test02");
+        assertOutputContaining("2nd run: should run A.test03", "A.test03");
+        assertOutputContaining("2nd run: should run B.test04", "B.test04");
+        assertOutputContaining("2nd run: should run D.test10", "D.test10");
+        
+        // "fix" errors in class A
+        executeTarget("failureRecorder.fixing");
+        
+        // 3rd run: four running tests with two errors
+        executeTarget("failureRecorder.runtest");
+        assertTrue("The collector file " + collectorFile.getAbsolutePath() 
+                + " should exist after the 3rd run.", collectorFile.exists());
+        assertOutputContaining("3rd run: should run A.test02", "A.test02");
+        assertOutputContaining("3rd run: should run A.test03", "A.test03");
+        assertOutputContaining("3rd run: should run B.test04", "B.test04");
+        assertOutputContaining("3rd run: should run D.test10", "D.test10");
+        
+        // 4rd run: two running tests with errors
+        executeTarget("failureRecorder.runtest");
+        assertTrue("The collector file " + collectorFile.getAbsolutePath() 
+                + " should exist after the 4th run.", collectorFile.exists());
+        assertOutputNotContaining("4th run: should not run A.test02", "A.test02");
+        assertOutputNotContaining("4th run: should not run A.test03", "A.test03");
+        assertOutputContaining("4th run: should run B.test04", "B.test04");
+        assertOutputContaining("4th run: should run D.test10", "D.test10");
+    }
+    
     public void testBatchTestForkOnceCustomFormatter() {
         assertResultFilesExist("testBatchTestForkOnceCustomFormatter", "foo");
     }
@@ -155,5 +231,4 @@ public class JUnitTaskTest extends BuildFileTest {
         assertEquals(search, line);
     }
 
-}
-
+}