Index: analysis/common/build.xml
===================================================================
--- analysis/common/build.xml	(revision 1712294)
+++ analysis/common/build.xml	(working copy)
@@ -32,7 +32,7 @@
   <property name="snowball.programs.dir" location="src/java/org/tartarus/snowball/ext"/>  
   
   <target name="jflex" depends="-install-jflex,clean-jflex,-jflex-StandardAnalyzer,-jflex-UAX29URLEmailTokenizer,
-                                -jflex-wiki-tokenizer,-jflex-HTMLStripCharFilter"/>
+                                -jflex-wiki-tokenizer,-jflex-UnicodeWhitespaceTokenizer,-jflex-HTMLStripCharFilter"/>
 
   <target name="-jflex-HTMLStripCharFilter"
           depends="init,generate-jflex-html-char-entities">
@@ -70,7 +70,12 @@
     <run-jflex-and-disable-buffer-expansion
         dir="src/java/org/apache/lucene/analysis/standard" name="UAX29URLEmailTokenizerImpl"/>
   </target>
-  
+
+  <target name="-jflex-UnicodeWhitespaceTokenizer" depends="init,-install-jflex">
+    <run-jflex-and-disable-buffer-expansion
+        dir="src/java/org/apache/lucene/analysis/whitespace" name="UnicodeWhitespaceTokenizerImpl"/>
+  </target>
+
   <macrodef name="run-jflex">
     <attribute name="dir"/>
     <attribute name="name"/>
Index: analysis/common/src/java/org/apache/lucene/analysis/whitespace/UnicodeWhitespaceTokenizer.java
===================================================================
--- analysis/common/src/java/org/apache/lucene/analysis/whitespace/UnicodeWhitespaceTokenizer.java	(revision 0)
+++ analysis/common/src/java/org/apache/lucene/analysis/whitespace/UnicodeWhitespaceTokenizer.java	(working copy)
@@ -0,0 +1,128 @@
+package org.apache.lucene.analysis.whitespace;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.util.AttributeFactory;
+
+/** A grammar-based tokenizer constructed with JFlex.
+ * <p>
+ * A tokenizer that divides text at Unicode whitespace.
+ * Adjacent sequences of non-Whitespace characters form tokens.
+ * This tokenizer uses the Unicode White_Space property.
+ * See http://www.unicode.org/Public/UCD/latest/ucd/PropList.txt
+ */
+public final class UnicodeWhitespaceTokenizer extends Tokenizer {
+  /** A private instance of the JFlex-constructed scanner */
+  private UnicodeWhitespaceTokenizerImpl scanner;
+
+  public static final int MAX_TOKEN_LENGTH_LIMIT = 1024 * 1024;
+
+  private int maxTokenLength = UnicodeWhitespaceTokenizerFactory.DEFAULT_MAX_TOKEN_LENGTH;
+
+  /**
+   * Set the max allowed token length.  No tokens longer than this are emitted.
+   *
+   * @throws IllegalArgumentException if the given length is outside of the
+   *  range [1, {@value #MAX_TOKEN_LENGTH_LIMIT}].
+   */
+  public void setMaxTokenLength(int length) {
+    if (length < 1) {
+      throw new IllegalArgumentException("maxTokenLength must be greater than zero");
+    } else if (length > MAX_TOKEN_LENGTH_LIMIT) {
+      throw new IllegalArgumentException("maxTokenLength may not exceed " + MAX_TOKEN_LENGTH_LIMIT);
+    }
+    if (length != maxTokenLength) {
+      maxTokenLength = length;
+      scanner.setBufferSize(length);
+    }
+  }
+
+  /** @see #setMaxTokenLength */
+  public int getMaxTokenLength() {
+    return maxTokenLength;
+  }
+
+  /**
+   * Creates a new instance of the {@link org.apache.lucene.analysis.whitespace.UnicodeWhitespaceTokenizer}.
+   * Attaches the <code>input</code> to the newly created JFlex scanner.
+   */
+  public UnicodeWhitespaceTokenizer() {
+    init();
+  }
+
+  /**
+   * Creates a new UnicodeWhitespaceTokenizer with a given {@link org.apache.lucene.util.AttributeFactory}
+   */
+  public UnicodeWhitespaceTokenizer(AttributeFactory factory) {
+    super(factory);
+    init();
+  }
+
+  private void init() {
+    this.scanner = new UnicodeWhitespaceTokenizerImpl(input);
+  }
+
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+  private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
+  private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
+
+  /*
+   * (non-Javadoc)
+   *
+   * @see org.apache.lucene.analysis.TokenStream#next()
+   */
+  @Override
+  public final boolean incrementToken() throws IOException {
+    clearAttributes();
+
+    if (scanner.incrementToken()) {
+      scanner.getText(termAtt);
+      final int start = scanner.yychar();
+      offsetAtt.setOffset(correctOffset(start), correctOffset(start + termAtt.length()));
+      return true;
+    } else {
+      return false;
+    }
+  }
+
+  @Override
+  public final void end() throws IOException {
+    super.end();
+    // set final offset
+    int finalOffset = correctOffset(scanner.yychar() + scanner.yylength());
+    offsetAtt.setOffset(finalOffset, finalOffset);
+  }
+
+  @Override
+  public void close() throws IOException {
+    super.close();
+    scanner.yyreset(input);
+  }
+
+  @Override
+  public void reset() throws IOException {
+    super.reset();
+    scanner.yyreset(input);
+  }
+}

Property changes on: analysis/common/src/java/org/apache/lucene/analysis/whitespace/UnicodeWhitespaceTokenizer.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: analysis/common/src/java/org/apache/lucene/analysis/whitespace/UnicodeWhitespaceTokenizerFactory.java
===================================================================
--- analysis/common/src/java/org/apache/lucene/analysis/whitespace/UnicodeWhitespaceTokenizerFactory.java	(revision 0)
+++ analysis/common/src/java/org/apache/lucene/analysis/whitespace/UnicodeWhitespaceTokenizerFactory.java	(working copy)
@@ -0,0 +1,53 @@
+package org.apache.lucene.analysis.whitespace;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.apache.lucene.util.AttributeFactory;
+
+import java.util.Map;
+
+/**
+ * Factory for {@link UnicodeWhitespaceTokenizer}.
+ * <pre class="prettyprint">
+ * &lt;fieldType name="text_uni_space" class="solr.TextField" positionIncrementGap="100"&gt;
+ *   &lt;analyzer&gt;
+ *     &lt;tokenizer class="solr.UnicodeWhitespaceTokenizerFactory" maxTokenLength="255"/&gt;
+ *   &lt;/analyzer&gt;
+ * &lt;/fieldType&gt;</pre>
+ */
+public class UnicodeWhitespaceTokenizerFactory extends TokenizerFactory {
+  public static final int DEFAULT_MAX_TOKEN_LENGTH = 255;
+  private final int maxTokenLength;
+
+  /** Creates a new UnicodeWhitespaceTokenizerFactory */
+  public UnicodeWhitespaceTokenizerFactory(Map<String,String> args) {
+    super(args);
+    maxTokenLength = getInt(args, "maxTokenLength", DEFAULT_MAX_TOKEN_LENGTH);
+    if (!args.isEmpty()) {
+      throw new IllegalArgumentException("Unknown parameters: " + args);
+    }
+  }
+
+  @Override
+  public UnicodeWhitespaceTokenizer create(AttributeFactory factory) {
+    UnicodeWhitespaceTokenizer tokenizer = new UnicodeWhitespaceTokenizer(factory);
+    tokenizer.setMaxTokenLength(maxTokenLength);
+    return tokenizer;
+  }
+}

Property changes on: analysis/common/src/java/org/apache/lucene/analysis/whitespace/UnicodeWhitespaceTokenizerFactory.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: analysis/common/src/java/org/apache/lucene/analysis/whitespace/UnicodeWhitespaceTokenizerImpl.java
===================================================================
--- analysis/common/src/java/org/apache/lucene/analysis/whitespace/UnicodeWhitespaceTokenizerImpl.java	(revision 0)
+++ analysis/common/src/java/org/apache/lucene/analysis/whitespace/UnicodeWhitespaceTokenizerImpl.java	(working copy)
@@ -0,0 +1,593 @@
+/* The following code was generated by JFlex 1.6.0 */
+
+package org.apache.lucene.analysis.whitespace;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+
+/**
+ * This class implements returns tokens that match \P{White_Space}, that is,
+ * consisting of a sequence of one or more characters that do not have the
+ * White_Space property.
+ * See http://www.unicode.org/Public/UCD/latest/ucd/PropList.txt
+ */
+@SuppressWarnings("fallthrough")
+
+public final class UnicodeWhitespaceTokenizerImpl {
+
+  /** This character denotes the end of file */
+  public static final int YYEOF = -1;
+
+  /** initial size of the lookahead buffer */
+  private int ZZ_BUFFERSIZE = 255;
+
+  /** lexical states */
+  public static final int YYINITIAL = 0;
+
+  /**
+   * ZZ_LEXSTATE[l] is the state in the DFA for the lexical state l
+   * ZZ_LEXSTATE[l+1] is the state in the DFA for the lexical state l
+   *                  at the beginning of a line
+   * l is of the form l = 2*k, k a non negative integer
+   */
+  private static final int ZZ_LEXSTATE[] = { 
+     0, 0
+  };
+
+  /** 
+   * Translates characters to character classes
+   */
+  private static final String ZZ_CMAP_PACKED = 
+    "\11\0\5\1\22\0\1\1\144\0\1\1\32\0\1\1\u15df\0\1\1"+
+    "\u097f\0\13\1\35\0\2\1\5\0\1\1\57\0\1\1\u0fa0\0\1\1"+
+    "\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\uffff\0\ud00f\0";
+
+  /** 
+   * Translates characters to character classes
+   */
+  private static final char [] ZZ_CMAP = zzUnpackCMap(ZZ_CMAP_PACKED);
+
+  /** 
+   * Translates DFA states to action switch labels.
+   */
+  private static final int [] ZZ_ACTION = zzUnpackAction();
+
+  private static final String ZZ_ACTION_PACKED_0 =
+    "\1\0\1\1\1\2";
+
+  private static int [] zzUnpackAction() {
+    int [] result = new int[3];
+    int offset = 0;
+    offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result);
+    return result;
+  }
+
+  private static int zzUnpackAction(String packed, int offset, int [] result) {
+    int i = 0;       /* index in packed string  */
+    int j = offset;  /* index in unpacked array */
+    int l = packed.length();
+    while (i < l) {
+      int count = packed.charAt(i++);
+      int value = packed.charAt(i++);
+      do result[j++] = value; while (--count > 0);
+    }
+    return j;
+  }
+
+
+  /** 
+   * Translates a state to a row index in the transition table
+   */
+  private static final int [] ZZ_ROWMAP = zzUnpackRowMap();
+
+  private static final String ZZ_ROWMAP_PACKED_0 =
+    "\0\0\0\2\0\4";
+
+  private static int [] zzUnpackRowMap() {
+    int [] result = new int[3];
+    int offset = 0;
+    offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result);
+    return result;
+  }
+
+  private static int zzUnpackRowMap(String packed, int offset, int [] result) {
+    int i = 0;  /* index in packed string  */
+    int j = offset;  /* index in unpacked array */
+    int l = packed.length();
+    while (i < l) {
+      int high = packed.charAt(i++) << 16;
+      result[j++] = high | packed.charAt(i++);
+    }
+    return j;
+  }
+
+  /** 
+   * The transition table of the DFA
+   */
+  private static final int [] ZZ_TRANS = zzUnpackTrans();
+
+  private static final String ZZ_TRANS_PACKED_0 =
+    "\1\2\1\3\1\2\3\0";
+
+  private static int [] zzUnpackTrans() {
+    int [] result = new int[6];
+    int offset = 0;
+    offset = zzUnpackTrans(ZZ_TRANS_PACKED_0, offset, result);
+    return result;
+  }
+
+  private static int zzUnpackTrans(String packed, int offset, int [] result) {
+    int i = 0;       /* index in packed string  */
+    int j = offset;  /* index in unpacked array */
+    int l = packed.length();
+    while (i < l) {
+      int count = packed.charAt(i++);
+      int value = packed.charAt(i++);
+      value--;
+      do result[j++] = value; while (--count > 0);
+    }
+    return j;
+  }
+
+
+  /* error codes */
+  private static final int ZZ_UNKNOWN_ERROR = 0;
+  private static final int ZZ_NO_MATCH = 1;
+  private static final int ZZ_PUSHBACK_2BIG = 2;
+
+  /* error messages for the codes above */
+  private static final String ZZ_ERROR_MSG[] = {
+    "Unkown internal scanner error",
+    "Error: could not match input",
+    "Error: pushback value was too large"
+  };
+
+  /**
+   * ZZ_ATTRIBUTE[aState] contains the attributes of state <code>aState</code>
+   */
+  private static final int [] ZZ_ATTRIBUTE = zzUnpackAttribute();
+
+  private static final String ZZ_ATTRIBUTE_PACKED_0 =
+    "\1\0\1\1\1\11";
+
+  private static int [] zzUnpackAttribute() {
+    int [] result = new int[3];
+    int offset = 0;
+    offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result);
+    return result;
+  }
+
+  private static int zzUnpackAttribute(String packed, int offset, int [] result) {
+    int i = 0;       /* index in packed string  */
+    int j = offset;  /* index in unpacked array */
+    int l = packed.length();
+    while (i < l) {
+      int count = packed.charAt(i++);
+      int value = packed.charAt(i++);
+      do result[j++] = value; while (--count > 0);
+    }
+    return j;
+  }
+
+  /** the input device */
+  private java.io.Reader zzReader;
+
+  /** the current state of the DFA */
+  private int zzState;
+
+  /** the current lexical state */
+  private int zzLexicalState = YYINITIAL;
+
+  /** this buffer contains the current text to be matched and is
+      the source of the yytext() string */
+  private char zzBuffer[] = new char[ZZ_BUFFERSIZE];
+
+  /** the textposition at the last accepting state */
+  private int zzMarkedPos;
+
+  /** the current text position in the buffer */
+  private int zzCurrentPos;
+
+  /** startRead marks the beginning of the yytext() string in the buffer */
+  private int zzStartRead;
+
+  /** endRead marks the last character in the buffer, that has been read
+      from input */
+  private int zzEndRead;
+
+  /** number of newlines encountered up to the start of the matched text */
+  private int yyline;
+
+  /** the number of characters up to the start of the matched text */
+  private int yychar;
+
+  /**
+   * the number of characters from the last newline up to the start of the 
+   * matched text
+   */
+  private int yycolumn;
+
+  /** 
+   * zzAtBOL == true <=> the scanner is currently at the beginning of a line
+   */
+  private boolean zzAtBOL = true;
+
+  /** zzAtEOF == true <=> the scanner is at the EOF */
+  private boolean zzAtEOF;
+
+  /** denotes if the user-EOF-code has already been executed */
+  private boolean zzEOFDone;
+  
+  /** 
+   * The number of occupied positions in zzBuffer beyond zzEndRead.
+   * When a lead/high surrogate has been read from the input stream
+   * into the final zzBuffer position, this will have a value of 1;
+   * otherwise, it will have a value of 0.
+   */
+  private int zzFinalHighSurrogate = 0;
+
+  /* user code: */
+  public final int yychar() {
+    return yychar;
+  }
+
+  /**
+   * Fills CharTermAttribute with the current token text.
+   */
+  public final void getText(CharTermAttribute t) {
+    t.copyBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
+  }
+
+  /**
+   * Sets the scanner buffer size in chars
+   */
+   public final void setBufferSize(int numChars) {
+     ZZ_BUFFERSIZE = numChars;
+     char[] newZzBuffer = new char[ZZ_BUFFERSIZE];
+     System.arraycopy(zzBuffer, 0, newZzBuffer, 0, Math.min(zzBuffer.length, ZZ_BUFFERSIZE));
+     zzBuffer = newZzBuffer;
+   }
+
+
+  /**
+   * Creates a new scanner
+   *
+   * @param   in  the java.io.Reader to read input from.
+   */
+  public UnicodeWhitespaceTokenizerImpl(java.io.Reader in) {
+    this.zzReader = in;
+  }
+
+
+  /** 
+   * Unpacks the compressed character translation table.
+   *
+   * @param packed   the packed character translation table
+   * @return         the unpacked character translation table
+   */
+  private static char [] zzUnpackCMap(String packed) {
+    char [] map = new char[0x110000];
+    int i = 0;  /* index in packed string  */
+    int j = 0;  /* index in unpacked array */
+    while (i < 74) {
+      int  count = packed.charAt(i++);
+      char value = packed.charAt(i++);
+      do map[j++] = value; while (--count > 0);
+    }
+    return map;
+  }
+
+
+  /**
+   * Refills the input buffer.
+   *
+   * @return      <code>false</code>, iff there was new input.
+   * 
+   * @exception   java.io.IOException  if any I/O-Error occurs
+   */
+  private boolean zzRefill() throws java.io.IOException {
+
+    /* first: make room (if you can) */
+    if (zzStartRead > 0) {
+      zzEndRead += zzFinalHighSurrogate;
+      zzFinalHighSurrogate = 0;
+      System.arraycopy(zzBuffer, zzStartRead,
+                       zzBuffer, 0,
+                       zzEndRead-zzStartRead);
+
+      /* translate stored positions */
+      zzEndRead-= zzStartRead;
+      zzCurrentPos-= zzStartRead;
+      zzMarkedPos-= zzStartRead;
+      zzStartRead = 0;
+    }
+
+
+    /* fill the buffer with new input */
+    int requested = zzBuffer.length - zzEndRead - zzFinalHighSurrogate;           
+    int totalRead = 0;
+    while (totalRead < requested) {
+      int numRead = zzReader.read(zzBuffer, zzEndRead + totalRead, requested - totalRead);
+      if (numRead == -1) {
+        break;
+      }
+      totalRead += numRead;
+    }
+
+    if (totalRead > 0) {
+      zzEndRead += totalRead;
+      if (totalRead == requested) { /* possibly more input available */
+        if (Character.isHighSurrogate(zzBuffer[zzEndRead - 1])) {
+          --zzEndRead;
+          zzFinalHighSurrogate = 1;
+          if (totalRead == 1) { return true; }
+        }
+      }
+      return false;
+    }
+
+    // totalRead = 0: End of stream
+    return true;
+  }
+
+    
+  /**
+   * Closes the input stream.
+   */
+  public final void yyclose() throws java.io.IOException {
+    zzAtEOF = true;            /* indicate end of file */
+    zzEndRead = zzStartRead;  /* invalidate buffer    */
+
+    if (zzReader != null)
+      zzReader.close();
+  }
+
+
+  /**
+   * Resets the scanner to read from a new input stream.
+   * Does not close the old reader.
+   *
+   * All internal variables are reset, the old input stream 
+   * <b>cannot</b> be reused (internal buffer is discarded and lost).
+   * Lexical state is set to <tt>ZZ_INITIAL</tt>.
+   *
+   * Internal scan buffer is resized down to its initial length, if it has grown.
+   *
+   * @param reader   the new input stream 
+   */
+  public final void yyreset(java.io.Reader reader) {
+    zzReader = reader;
+    zzAtBOL  = true;
+    zzAtEOF  = false;
+    zzEOFDone = false;
+    zzEndRead = zzStartRead = 0;
+    zzCurrentPos = zzMarkedPos = 0;
+    zzFinalHighSurrogate = 0;
+    yyline = yychar = yycolumn = 0;
+    zzLexicalState = YYINITIAL;
+    if (zzBuffer.length > ZZ_BUFFERSIZE)
+      zzBuffer = new char[ZZ_BUFFERSIZE];
+  }
+
+
+  /**
+   * Returns the current lexical state.
+   */
+  public final int yystate() {
+    return zzLexicalState;
+  }
+
+
+  /**
+   * Enters a new lexical state
+   *
+   * @param newState the new lexical state
+   */
+  public final void yybegin(int newState) {
+    zzLexicalState = newState;
+  }
+
+
+  /**
+   * Returns the text matched by the current regular expression.
+   */
+  public final String yytext() {
+    return new String( zzBuffer, zzStartRead, zzMarkedPos-zzStartRead );
+  }
+
+
+  /**
+   * Returns the character at position <tt>pos</tt> from the 
+   * matched text. 
+   * 
+   * It is equivalent to yytext().charAt(pos), but faster
+   *
+   * @param pos the position of the character to fetch. 
+   *            A value from 0 to yylength()-1.
+   *
+   * @return the character at position pos
+   */
+  public final char yycharat(int pos) {
+    return zzBuffer[zzStartRead+pos];
+  }
+
+
+  /**
+   * Returns the length of the matched text region.
+   */
+  public final int yylength() {
+    return zzMarkedPos-zzStartRead;
+  }
+
+
+  /**
+   * Reports an error that occured while scanning.
+   *
+   * In a wellformed scanner (no or only correct usage of 
+   * yypushback(int) and a match-all fallback rule) this method 
+   * will only be called with things that "Can't Possibly Happen".
+   * If this method is called, something is seriously wrong
+   * (e.g. a JFlex bug producing a faulty scanner etc.).
+   *
+   * Usual syntax/scanner level error handling should be done
+   * in error fallback rules.
+   *
+   * @param   errorCode  the code of the errormessage to display
+   */
+  private void zzScanError(int errorCode) {
+    String message;
+    try {
+      message = ZZ_ERROR_MSG[errorCode];
+    }
+    catch (ArrayIndexOutOfBoundsException e) {
+      message = ZZ_ERROR_MSG[ZZ_UNKNOWN_ERROR];
+    }
+
+    throw new Error(message);
+  } 
+
+
+  /**
+   * Pushes the specified amount of characters back into the input stream.
+   *
+   * They will be read again by then next call of the scanning method
+   *
+   * @param number  the number of characters to be read again.
+   *                This number must not be greater than yylength()!
+   */
+  public void yypushback(int number)  {
+    if ( number > yylength() )
+      zzScanError(ZZ_PUSHBACK_2BIG);
+
+    zzMarkedPos -= number;
+  }
+
+
+  /**
+   * Resumes scanning until the next regular expression is matched,
+   * the end of input is encountered or an I/O-Error occurs.
+   *
+   * @return      the next token
+   * @exception   java.io.IOException  if any I/O-Error occurs
+   */
+  public boolean incrementToken() throws java.io.IOException {
+    int zzInput;
+    int zzAction;
+
+    // cached fields:
+    int zzCurrentPosL;
+    int zzMarkedPosL;
+    int zzEndReadL = zzEndRead;
+    char [] zzBufferL = zzBuffer;
+    char [] zzCMapL = ZZ_CMAP;
+
+    int [] zzTransL = ZZ_TRANS;
+    int [] zzRowMapL = ZZ_ROWMAP;
+    int [] zzAttrL = ZZ_ATTRIBUTE;
+
+    while (true) {
+      zzMarkedPosL = zzMarkedPos;
+
+      yychar+= zzMarkedPosL-zzStartRead;
+
+      zzAction = -1;
+
+      zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL;
+  
+      zzState = ZZ_LEXSTATE[zzLexicalState];
+
+      // set up zzAction for empty match case:
+      int zzAttributes = zzAttrL[zzState];
+      if ( (zzAttributes & 1) == 1 ) {
+        zzAction = zzState;
+      }
+
+
+      zzForAction: {
+        while (true) {
+    
+          if (zzCurrentPosL < zzEndReadL) {
+            zzInput = Character.codePointAt(zzBufferL, zzCurrentPosL, zzEndReadL);
+            zzCurrentPosL += Character.charCount(zzInput);
+          }
+          else if (zzAtEOF) {
+            zzInput = YYEOF;
+            break zzForAction;
+          }
+          else {
+            // store back cached positions
+            zzCurrentPos  = zzCurrentPosL;
+            zzMarkedPos   = zzMarkedPosL;
+            boolean eof = zzRefill();
+            // get translated positions and possibly new buffer
+            zzCurrentPosL  = zzCurrentPos;
+            zzMarkedPosL   = zzMarkedPos;
+            zzBufferL      = zzBuffer;
+            zzEndReadL     = zzEndRead;
+            if (eof) {
+              zzInput = YYEOF;
+              break zzForAction;
+            }
+            else {
+              zzInput = Character.codePointAt(zzBufferL, zzCurrentPosL, zzEndReadL);
+              zzCurrentPosL += Character.charCount(zzInput);
+            }
+          }
+          int zzNext = zzTransL[ zzRowMapL[zzState] + zzCMapL[zzInput] ];
+          if (zzNext == -1) break zzForAction;
+          zzState = zzNext;
+
+          zzAttributes = zzAttrL[zzState];
+          if ( (zzAttributes & 1) == 1 ) {
+            zzAction = zzState;
+            zzMarkedPosL = zzCurrentPosL;
+            if ( (zzAttributes & 8) == 8 ) break zzForAction;
+          }
+
+        }
+      }
+
+      // store back cached position
+      zzMarkedPos = zzMarkedPosL;
+
+      switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) {
+        case 1: 
+          { return true;
+          }
+        case 3: break;
+        case 2: 
+          { /* ignore chars matching \p{White_Space} */
+          }
+        case 4: break;
+        default: 
+          if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
+            zzAtEOF = true;
+              {
+                return false;
+              }
+          } 
+          else {
+            zzScanError(ZZ_NO_MATCH);
+          }
+      }
+    }
+  }
+
+
+}

Property changes on: analysis/common/src/java/org/apache/lucene/analysis/whitespace/UnicodeWhitespaceTokenizerImpl.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: analysis/common/src/java/org/apache/lucene/analysis/whitespace/UnicodeWhitespaceTokenizerImpl.jflex
===================================================================
--- analysis/common/src/java/org/apache/lucene/analysis/whitespace/UnicodeWhitespaceTokenizerImpl.jflex	(revision 0)
+++ analysis/common/src/java/org/apache/lucene/analysis/whitespace/UnicodeWhitespaceTokenizerImpl.jflex	(working copy)
@@ -0,0 +1,68 @@
+package org.apache.lucene.analysis.whitespace;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+
+/**
+ * This class implements returns tokens that match \P{White_Space}, that is,
+ * consisting of a sequence of one or more characters that do not have the
+ * White_Space property.
+ * See http://www.unicode.org/Public/UCD/latest/ucd/PropList.txt
+ */
+@SuppressWarnings("fallthrough")
+%%
+
+%unicode 6.3
+%type boolean
+%final
+%public
+%class UnicodeWhitespaceTokenizerImpl
+%function incrementToken
+%char
+%buffer 255
+
+%{
+  public final int yychar() {
+    return yychar;
+  }
+
+  /**
+   * Fills CharTermAttribute with the current token text.
+   */
+  public final void getText(CharTermAttribute t) {
+    t.copyBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
+  }
+
+  /**
+   * Sets the scanner buffer size in chars
+   */
+   public final void setBufferSize(int numChars) {
+     ZZ_BUFFERSIZE = numChars;
+     char[] newZzBuffer = new char[ZZ_BUFFERSIZE];
+     System.arraycopy(zzBuffer, 0, newZzBuffer, 0, Math.min(zzBuffer.length, ZZ_BUFFERSIZE));
+     zzBuffer = newZzBuffer;
+   }
+%}
+
+%%
+<<EOF>> { return false; }
+
+\P{White_Space}+ { return true; }
+
+[^] { /* ignore chars not matching \P{White_Space} */ }
Index: analysis/common/src/java/org/apache/lucene/analysis/whitespace/package-info.java
===================================================================
--- analysis/common/src/java/org/apache/lucene/analysis/whitespace/package-info.java	(revision 0)
+++ analysis/common/src/java/org/apache/lucene/analysis/whitespace/package-info.java	(working copy)
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Whitespace tokenizers.
+ *
+ * <p>The <code>org.apache.lucene.analysis.whitespace</code> package contains a
+ * fast grammar-based tokenizer constructed with JFlex:</p>
+ * <ul>
+ *     <li>{@link org.apache.lucene.analysis.whitespace.UnicodeWhitespaceTokenizer}:
+ *         Returns tokens that are sequences of \P{White_Space} characters, i.e.,
+ *         that do not match the Unicode White_Space property.  See
+ *         http://www.unicode.org/Public/UCD/latest/ucd/PropList.txt
+ *     </li>
+ * </ul>
+ */
+package org.apache.lucene.analysis.whitespace;

Property changes on: analysis/common/src/java/org/apache/lucene/analysis/whitespace/package-info.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenizerFactory
===================================================================
--- analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenizerFactory	(revision 1712294)
+++ analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenizerFactory	(working copy)
@@ -25,4 +25,5 @@
 org.apache.lucene.analysis.standard.StandardTokenizerFactory
 org.apache.lucene.analysis.standard.UAX29URLEmailTokenizerFactory
 org.apache.lucene.analysis.th.ThaiTokenizerFactory
+org.apache.lucene.analysis.whitespace.UnicodeWhitespaceTokenizerFactory
 org.apache.lucene.analysis.wikipedia.WikipediaTokenizerFactory
Index: analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java
===================================================================
--- analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java	(revision 1712294)
+++ analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java	(working copy)
@@ -17,8 +17,17 @@
  * limitations under the License.
  */
 
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.FileWriter;
 import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
 import java.io.StringReader;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.BaseTokenStreamTestCase;
@@ -30,6 +39,7 @@
 import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.TestUtil;
 
 public class TestAnalyzers extends BaseTokenStreamTestCase {
 
@@ -227,7 +237,75 @@
     assertTokenStreamContents(tokenizer, new String[] { "Tokenizer",
         "\ud801\udc1ctest" });
   }
-  
+
+  public void testUnicode() throws IOException {
+    Random r = new Random(11);
+    WhitespaceTokenizer tokenizer = new WhitespaceTokenizer();
+    CharTermAttribute charTermAttribute = tokenizer.addAttribute(CharTermAttribute.class);
+    StringBuilder builder = new StringBuilder();
+    long count = 0L;
+    String[] strings = new String[10000];
+    for (int i = 0 ; i < 10000 ; ++i) {
+      String str = TestUtil.randomUnicodeString(r, 10000);
+      //str = str.replaceAll("[\u0085\u00A0\u2007\u202F\u001C-\u001F\u180E]", " ");
+      strings[i] = str;
+    }
+    long start = System.nanoTime();
+    for (int i = 0 ; i < 10000 ; ++i) {
+      tokenizer.setReader(new StringReader(strings[i]));
+      tokenizer.reset();
+      builder.setLength(0);
+      while (tokenizer.incrementToken()) {
+        ++count;
+      }
+      tokenizer.end();
+      tokenizer.close();
+    }
+    long stop = System.nanoTime();
+    long millis = TimeUnit.NANOSECONDS.toMillis(stop - start);
+    System.err.println("Duration: " + millis + "ms   Count: " + count + "   Tokens/sec: " + ((double)count / millis * 1000.));
+  }
+
+
+  public void testReuters() throws IOException {
+    StringBuilder builder = new StringBuilder();
+    FileInputStream inputStream = new FileInputStream("/Users/sarowe/all.reuters.txt");
+    Reader reader = new InputStreamReader(inputStream, StandardCharsets.UTF_8);
+    BufferedReader bufferedReader = new BufferedReader(reader);
+    String str;
+    while (null != (str = bufferedReader.readLine())) {
+      builder.append(str).append("\n");
+    }
+    bufferedReader.close();
+    str = builder.toString();
+    long count = 0L;
+    {
+      WhitespaceTokenizer tokenizer = new WhitespaceTokenizer();
+      tokenizer.setReader(new StringReader(str));
+      tokenizer.reset();
+      while (tokenizer.incrementToken()) {
+        ++count;
+      }
+    }
+    count = 0L;
+    long start = System.nanoTime();
+    for (int i = 0 ; i < 10 ; ++i) {
+      WhitespaceTokenizer tokenizer = new WhitespaceTokenizer();
+      tokenizer.setReader(new StringReader(str));
+      tokenizer.reset();
+      while (tokenizer.incrementToken()) {
+        ++count;
+      }
+      tokenizer.end();
+      tokenizer.close();
+    }
+    long stop = System.nanoTime();
+    long millis = TimeUnit.NANOSECONDS.toMillis(stop - start);
+    System.err.println("Duration: " + millis + "ms   Count: " + count + "   Tokens/sec: " + ((double)count / millis * 1000.));
+  }
+
+
+
   /** blast some random strings through the analyzer */
   public void testRandomStrings() throws Exception {
     Analyzer analyzers[] = new Analyzer[] { new WhitespaceAnalyzer(), new SimpleAnalyzer(), new StopAnalyzer() };
Index: analysis/common/src/test/org/apache/lucene/analysis/whitespace/TestUnicodeWhitespaceTokenizer.java
===================================================================
--- analysis/common/src/test/org/apache/lucene/analysis/whitespace/TestUnicodeWhitespaceTokenizer.java	(revision 0)
+++ analysis/common/src/test/org/apache/lucene/analysis/whitespace/TestUnicodeWhitespaceTokenizer.java	(working copy)
@@ -0,0 +1,121 @@
+package org.apache.lucene.analysis.whitespace;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+    
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.io.StringReader;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
+
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.util.TestUtil;
+
+public class TestUnicodeWhitespaceTokenizer extends BaseTokenStreamTestCase {
+  
+  // clone of test from WhitespaceTokenizer
+  public void testSimple() throws IOException {
+    StringReader reader = new StringReader("Tokenizer \ud801\udc1ctest");
+    UnicodeWhitespaceTokenizer tokenizer = new UnicodeWhitespaceTokenizer();
+    tokenizer.setReader(reader);
+    assertTokenStreamContents(tokenizer, new String[] { "Tokenizer",
+        "\ud801\udc1ctest" });
+  }
+  
+  public void testNBSP() throws IOException {
+    StringReader reader = new StringReader("Tokenizer\u00A0test");
+    UnicodeWhitespaceTokenizer tokenizer = new UnicodeWhitespaceTokenizer();
+    tokenizer.setReader(reader);
+    assertTokenStreamContents(tokenizer, new String[] { "Tokenizer",
+        "test" });
+  }
+
+  public void testUnicode() throws IOException {
+    Random r = new Random(11);
+    UnicodeWhitespaceTokenizer tokenizer = new UnicodeWhitespaceTokenizer();
+    CharTermAttribute charTermAttribute = tokenizer.addAttribute(CharTermAttribute.class);
+    StringBuilder builder = new StringBuilder();
+    long count = 0L;
+    String[] strings = new String[10000];
+    for (int i = 0 ; i < 10000 ; ++i) {
+      String str = TestUtil.randomUnicodeString(r, 10000);
+      //str = str.replaceAll("[\u0085\u00A0\u2007\u202F\u001C-\u001F\u180E]", " ");
+      strings[i] = str;
+    }
+    long start = System.nanoTime();
+    for (int i = 0 ; i < 10000 ; ++i) {
+      tokenizer.setReader(new StringReader(strings[i]));
+      tokenizer.reset();
+      builder.setLength(0);
+      while (tokenizer.incrementToken()) {
+        ++count;
+      }
+      tokenizer.end();
+      tokenizer.close();
+    }
+    long stop = System.nanoTime();
+    long millis = TimeUnit.NANOSECONDS.toMillis(stop - start);
+    System.err.println("Duration: " + millis + "ms   Count: " + count + "   Tokens/sec: " + ((double)count / millis * 1000.));
+  }
+
+  public void testReuters() throws IOException {
+    StringBuilder builder = new StringBuilder();
+    FileInputStream inputStream = new FileInputStream("/Users/sarowe/all.reuters.txt");
+    Reader reader = new InputStreamReader(inputStream, StandardCharsets.UTF_8);
+    BufferedReader bufferedReader = new BufferedReader(reader);
+    String str;
+    while (null != (str = bufferedReader.readLine())) {
+      builder.append(str).append("\n");
+    }
+    bufferedReader.close();
+    str = builder.toString();
+    long count = 0L;
+    {
+      UnicodeWhitespaceTokenizer tokenizer = new UnicodeWhitespaceTokenizer();
+      tokenizer.setReader(new StringReader(str));
+      tokenizer.reset();
+      while (tokenizer.incrementToken()) {
+        ++count;
+      }
+    }
+    count = 0L;
+    long start = System.nanoTime();
+    for (int i = 0 ; i < 10 ; ++i) {
+      UnicodeWhitespaceTokenizer tokenizer = new UnicodeWhitespaceTokenizer();
+      tokenizer.setReader(new StringReader(str));
+      tokenizer.reset();
+      while (tokenizer.incrementToken()) {
+        ++count;
+      }
+      tokenizer.end();
+      tokenizer.close();
+    }
+    long stop = System.nanoTime();
+    long millis = TimeUnit.NANOSECONDS.toMillis(stop - start);
+    System.err.println("Duration: " + millis + "ms   Count: " + count + "   Tokens/sec: " + ((double)count / millis * 1000.));
+  }
+}
\ No newline at end of file

Property changes on: analysis/common/src/test/org/apache/lucene/analysis/whitespace/TestUnicodeWhitespaceTokenizer.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUWhitespaceTokenizer.java
===================================================================
--- analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUWhitespaceTokenizer.java	(revision 0)
+++ analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUWhitespaceTokenizer.java	(working copy)
@@ -0,0 +1,56 @@
+package org.apache.lucene.analysis.icu;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.util.CharTokenizer;
+import org.apache.lucene.util.AttributeFactory;
+
+import com.ibm.icu.lang.UCharacter;
+
+/**
+ * A tokenizer that divides text at Unicode whitespace.
+ * Adjacent sequences of non-Whitespace characters form tokens.
+ * This tokenizer uses {@link UCharacter#isUWhiteSpace(int)}.
+ */
+public final class ICUWhitespaceTokenizer extends CharTokenizer {
+  
+  /**
+   * Construct a new ICUWhitespaceTokenizer.
+   */
+  public ICUWhitespaceTokenizer() {
+  }
+
+  /**
+   * Construct a new ICUWhitespaceTokenizer using a given
+   * {@link org.apache.lucene.util.AttributeFactory}.
+   *
+   * @param factory
+   *          the attribute factory to use for this {@link Tokenizer}
+   */
+  public ICUWhitespaceTokenizer(AttributeFactory factory) {
+    super(factory);
+  }
+  
+  /** Collects only characters which do not satisfy
+   * {@link UCharacter#isUWhiteSpace(int)}.*/
+  @Override
+  protected boolean isTokenChar(int c) {
+    return !UCharacter.isUWhiteSpace(c);
+  }
+}

Property changes on: analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUWhitespaceTokenizer.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUWhitespaceTokenizerFactory.java
===================================================================
--- analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUWhitespaceTokenizerFactory.java	(revision 0)
+++ analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUWhitespaceTokenizerFactory.java	(working copy)
@@ -0,0 +1,48 @@
+package org.apache.lucene.analysis.icu;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.apache.lucene.util.AttributeFactory;
+
+import java.util.Map;
+
+/**
+ * Factory for {@link ICUWhitespaceTokenizer}. 
+ * <pre class="prettyprint">
+ * &lt;fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100"&gt;
+ *   &lt;analyzer&gt;
+ *     &lt;tokenizer class="solr.ICUWhitespaceTokenizerFactory"/&gt;
+ *   &lt;/analyzer&gt;
+ * &lt;/fieldType&gt;</pre>
+ */
+public class ICUWhitespaceTokenizerFactory extends TokenizerFactory {
+
+  /** Creates a new ICUWhitespaceTokenizerFactory */
+  public ICUWhitespaceTokenizerFactory(Map<String,String> args) {
+    super(args);
+    if (!args.isEmpty()) {
+      throw new IllegalArgumentException("Unknown parameters: " + args);
+    }
+  }
+
+  @Override
+  public ICUWhitespaceTokenizer create(AttributeFactory factory) {
+    return new ICUWhitespaceTokenizer(factory);
+  }
+}

Property changes on: analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUWhitespaceTokenizerFactory.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: analysis/icu/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenizerFactory
===================================================================
--- analysis/icu/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenizerFactory	(revision 1712294)
+++ analysis/icu/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenizerFactory	(working copy)
@@ -14,3 +14,4 @@
 #  limitations under the License.
 
 org.apache.lucene.analysis.icu.segmentation.ICUTokenizerFactory
+org.apache.lucene.analysis.icu.ICUWhitespaceTokenizerFactory
Index: analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUWhitespaceTokenizer.java
===================================================================
--- analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUWhitespaceTokenizer.java	(revision 0)
+++ analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUWhitespaceTokenizer.java	(working copy)
@@ -0,0 +1,119 @@
+package org.apache.lucene.analysis.icu;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.io.StringReader;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.util.TestUtil;
+
+public class TestICUWhitespaceTokenizer extends BaseTokenStreamTestCase {
+
+  // clone of test from WhitespaceTokenizer
+  public void testSimple() throws IOException {
+    StringReader reader = new StringReader("Tokenizer \ud801\udc1ctest");
+    ICUWhitespaceTokenizer tokenizer = new ICUWhitespaceTokenizer();
+    tokenizer.setReader(reader);
+    assertTokenStreamContents(tokenizer, new String[] { "Tokenizer",
+    "\ud801\udc1ctest" });
+  }
+  
+  public void testNBSP() throws IOException {
+    StringReader reader = new StringReader("Tokenizer\u00A0test");
+    ICUWhitespaceTokenizer tokenizer = new ICUWhitespaceTokenizer();
+    tokenizer.setReader(reader);
+    assertTokenStreamContents(tokenizer, new String[] { "Tokenizer",
+    "test" });
+  }
+
+  public void testUnicode() throws IOException {
+    Random r = new Random(11);
+    ICUWhitespaceTokenizer tokenizer = new ICUWhitespaceTokenizer();
+    CharTermAttribute charTermAttribute = tokenizer.addAttribute(CharTermAttribute.class);
+    StringBuilder builder = new StringBuilder();
+    long count = 0L;
+    String[] strings = new String[10000];
+    for (int i = 0 ; i < 10000 ; ++i) {
+      String str = TestUtil.randomUnicodeString(r, 10000);
+      //str = str.replaceAll("[\u0085\u00A0\u2007\u202F\u001C-\u001F\u180E]", " ");
+      strings[i] = str;
+    }
+    long start = System.nanoTime();
+    for (int i = 0 ; i < 10000 ; ++i) {
+      tokenizer.setReader(new StringReader(strings[i]));
+      tokenizer.reset();
+      builder.setLength(0);
+      while (tokenizer.incrementToken()) {
+        ++count;
+      }
+      tokenizer.end();
+      tokenizer.close();
+    }
+    long stop = System.nanoTime();
+    long millis = TimeUnit.NANOSECONDS.toMillis(stop - start);
+    System.err.println("Duration: " + millis + "ms   Count: " + count + "   Tokens/sec: " + ((double)count / millis * 1000.));
+  }
+
+  public void testReuters() throws IOException {
+    StringBuilder builder = new StringBuilder();
+    FileInputStream inputStream = new FileInputStream("/Users/sarowe/all.reuters.txt");
+    Reader reader = new InputStreamReader(inputStream, StandardCharsets.UTF_8);
+    BufferedReader bufferedReader = new BufferedReader(reader);
+    String str;
+    while (null != (str = bufferedReader.readLine())) {
+      builder.append(str).append("\n");
+    }
+    bufferedReader.close();
+    str = builder.toString();
+    long count = 0L;
+    {
+      ICUWhitespaceTokenizer tokenizer = new ICUWhitespaceTokenizer();
+      tokenizer.setReader(new StringReader(str));
+      tokenizer.reset();
+      while (tokenizer.incrementToken()) {
+        ++count;
+      }
+    }
+    count = 0L;
+    long start = System.nanoTime();
+    for (int i = 0 ; i < 10 ; ++i) {
+      ICUWhitespaceTokenizer tokenizer = new ICUWhitespaceTokenizer();
+      tokenizer.setReader(new StringReader(str));
+      tokenizer.reset();
+      while (tokenizer.incrementToken()) {
+        ++count;
+      }
+      tokenizer.end();
+      tokenizer.close();
+    }
+    long stop = System.nanoTime();
+    long millis = TimeUnit.NANOSECONDS.toMillis(stop - start);
+    System.err.println("Duration: " + millis + "ms   Count: " + count + "   Tokens/sec: " + ((double)count / millis * 1000.));
+  }
+}

Property changes on: analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUWhitespaceTokenizer.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: benchmark/build.xml
===================================================================
--- benchmark/build.xml	(revision 1712294)
+++ benchmark/build.xml	(working copy)
@@ -175,6 +175,7 @@
       <pathelement path="${queries.jar}"/>
       <pathelement path="${codecs.jar}"/>
       <pathelement path="${join.jar}"/>
+      <pathelement path="${analyzers-icu.jar}"/>
       <path refid="base.classpath"/>
       <fileset dir="lib"/>
     </path>
@@ -277,7 +278,7 @@
       <echo>Benchmark output in JIRA table format is in file: ${shingle.jira.output.file}</echo>
     </target>
 
-    <target name="init" depends="module-build.init,jar-memory,jar-highlighter,jar-analyzers-common,jar-queryparser,jar-facet,jar-spatial,jar-codecs,jar-join"/>
+    <target name="init" depends="module-build.init,jar-memory,jar-highlighter,jar-analyzers-common,jar-queryparser,jar-facet,jar-spatial,jar-codecs,jar-join,jar-analyzers-icu"/>
   
     <target name="compile-test" depends="copy-alg-files-for-testing,module-build.compile-test"/>
     <target name="copy-alg-files-for-testing" description="copy .alg files as resources for testing">
Index: benchmark/conf/wstok.alg
===================================================================
--- benchmark/conf/wstok.alg	(revision 0)
+++ benchmark/conf/wstok.alg	(working copy)
@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This alg compares the performance of the original Character.isWhitespace()-based
+
+content.source=org.apache.lucene.benchmark.byTask.feeds.ReutersContentSource
+doc.tokenized=false
+doc.body.tokenized=true
+docs.dir=reuters-out
+
+-AnalyzerFactory(name:WhitespaceTokenizer, WhitespaceTokenizer)
+
+-AnalyzerFactory(name:UnicodeWhitespaceTokenizer, UnicodeWhitespaceTokenizer)
+
+-AnalyzerFactory(name:ICUWhitespaceTokenizer, ICUWhitespaceTokenizer)
+
+{ "Rounds"
+
+    -NewAnalyzer(ICUWhitespaceTokenizer)
+    -ResetInputs
+    { "[ICU] ICUWhitespaceTokenizer" { ReadTokens > : 20000 }
+
+    -NewAnalyzer(WhitespaceTokenizer)
+    -ResetInputs
+    { "[Character.isWhitespace()] WhitespaceTokenizer" { ReadTokens > : 20000 }
+
+    -NewAnalyzer(UnicodeWhitespaceTokenizer)
+    -ResetInputs
+    { "[JFlex] UnicodeWhitespaceTokenizer" { ReadTokens > : 20000 }
+
+    NewRound
+} : 5
+RepSumByNameRound
Index: benchmark/src/java/org/apache/lucene/benchmark/utils/ExtractReuters.java
===================================================================
--- benchmark/src/java/org/apache/lucene/benchmark/utils/ExtractReuters.java	(revision 1712294)
+++ benchmark/src/java/org/apache/lucene/benchmark/utils/ExtractReuters.java	(working copy)
@@ -47,6 +47,7 @@
 
   public void extract() throws IOException {
     long count = 0;
+    Files.createDirectories(outputDir);
     try (DirectoryStream<Path> stream = Files.newDirectoryStream(reutersDir, "*.sgm")) {
       for (Path sgmFile : stream) {
         extractFile(sgmFile);
@@ -70,7 +71,7 @@
    * Override if you wish to change what is extracted
    */
   protected void extractFile(Path sgmFile) {
-    try (BufferedReader reader = Files.newBufferedReader(sgmFile, StandardCharsets.UTF_8)) {
+    try (BufferedReader reader = Files.newBufferedReader(sgmFile, StandardCharsets.ISO_8859_1)) {
       StringBuilder buffer = new StringBuilder(1024);
       StringBuilder outBuffer = new StringBuilder(1024);
 
Index: core/src/java/org/apache/lucene/search/PhraseQuery.java
===================================================================
--- core/src/java/org/apache/lucene/search/PhraseQuery.java	(revision 1712294)
+++ core/src/java/org/apache/lucene/search/PhraseQuery.java	(working copy)
@@ -263,7 +263,7 @@
   @Override
   public Query rewrite(IndexReader reader) throws IOException {
     if (terms.length == 0) {
-      return new MatchNoDocsQuery();
+      return   new MatchNoDocsQuery();
     } else if (terms.length == 1) {
       return new TermQuery(terms[0]);
     } else if (positions[0] != 0) {
