Conform tokenizer-only U+0000 NUL handling to spec

This change brings the tokenizer’s handling of U+0000 NUL characters in
the DATA state and the CDATA section state into conformance with the
requirements in the HTML spec — for the case where only tokenization is
being performed, without tree construction; that is, the case where the
tokenizer() method is called, rather than parse() or parseFragment().

Specifically, the tokenization steps defined in the spec require that
when a U+0000 NUL is consumed in the DATA state or in the CDATA section
state, the parser must then emit a U+0000 NUL. But when performing tree
construction, the spec requires that when a U+0000 NUL is consumed, the
parser must instead emit a U+FFFD REPLACEMENT CHARACTER.

Without this change, the parser always emits a U+FFFD REPLACEMENT
CHARACTER — even when only tokenization is being performed. That causes
us to fail a number of tests in html5lib-tests suite.

For more background on the relevant behavior, see the following:

* https://www.w3.org/Bugs/Public/show_bug.cgi?id=9659
* whatwg/html@d98f83e
* 9b9c263

Relates to #35

Author: sideshowbarker

src/nu/validator/htmlparser/common/TokenHandler.java

@@ -144,6 +144,17 @@ public void characters(@Const @NoLength char[] buf, int start, int length)
      */
     public void zeroOriginatingReplacementCharacter() throws SAXException;
 
+    /**
+     * Emits:
+     *
+     *  * U+0000 if only tokenization is being performed
+     *  * U+FFFD if tree construction is being performed also
+     *
+     * @throws SAXException
+     *             if something went wrong
+     */
+    public void zeroOrReplacementCharacter() throws SAXException;
+
     /**
      * The end-of-file token.
      * 

src/nu/validator/htmlparser/impl/Tokenizer.java

@@ -1578,7 +1578,7 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                 break dataloop; // FALL THROUGH continue
                             // stateloop;
                             case '\u0000':
-                                emitReplacementCharacter(buf, pos);
+                                maybeEmitReplacementCharacter(buf, pos);
                                 continue;
                             case '\r':
                                 emitCarriageReturn(buf, pos);
@@ -3081,7 +3081,7 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                 state = transition(state, Tokenizer.CDATA_RSQB, reconsume, pos);
                                 break cdatasectionloop; // FALL THROUGH
                             case '\u0000':
-                                emitReplacementCharacter(buf, pos);
+                                maybeEmitReplacementCharacter(buf, pos);
                                 continue;
                             case '\r':
                                 emitCarriageReturn(buf, pos);
@@ -6177,6 +6177,13 @@ private void emitReplacementCharacter(@NoLength char[] buf, int pos)
         cstart = pos + 1;
     }
 
+    private void maybeEmitReplacementCharacter(@NoLength char[] buf, int pos)
+            throws SAXException {
+        flushChars(buf, pos);
+        tokenHandler.zeroOrReplacementCharacter();
+        cstart = pos + 1;
+    }
+
     private void emitPlaintextReplacementCharacter(@NoLength char[] buf, int pos)
             throws SAXException {
         flushChars(buf, pos);

src/nu/validator/htmlparser/impl/TreeBuilder.java

@@ -1241,6 +1241,13 @@ public void zeroOriginatingReplacementCharacter() throws SAXException {
         }
     }
 
+    /**
+     * @see nu.validator.htmlparser.common.TokenHandler#zeroOrReplacementCharacter()
+     */
+    public void zeroOrReplacementCharacter() throws SAXException {
+        zeroOriginatingReplacementCharacter();
+    }
+
     public final void eof() throws SAXException {
         flushCharacters();
         // Note: Can't attach error messages to EOF in C++ yet

test-src/nu/validator/htmlparser/test/JSONArrayTokenHandler.java

@@ -54,6 +54,8 @@ public class JSONArrayTokenHandler implements TokenHandler, ErrorHandler {
 
     private static final char[] REPLACEMENT_CHARACTER = { '\uFFFD' };
 
+    private static final char[] NULL = { '\u0000' };
+
     private final StringBuilder builder = new StringBuilder();
 
     private JSONArray array = null;
@@ -174,6 +176,11 @@ public void endTokenization() throws SAXException {
         builder.append(REPLACEMENT_CHARACTER, 0, 1);
     }
 
+    @Override public void zeroOrReplacementCharacter()
+            throws SAXException {
+        builder.append(NULL, 0, 1);
+    }
+
     @Override public boolean cdataSectionAllowed() throws SAXException {
         return false;
     }

test-src/nu/validator/htmlparser/test/TokenPrinter.java

@@ -200,6 +200,11 @@ public void endTokenization() throws SAXException {
         }
     }
 
+    @Override public void zeroOrReplacementCharacter()
+            throws SAXException {
+        zeroOriginatingReplacementCharacter();
+    }
+
     @Override public boolean cdataSectionAllowed() throws SAXException {
         return false;
     }